def test_eventteams_update(self):
self.teams_auth.put()
team_list = ['frc254', 'frc971', 'frc604']
request_body = json.dumps(team_list)
request_path = '/api/trusted/v1/event/2014casj/team_list/update'
sig = md5.new('{}{}{}'.format('321tEsTsEcReT', request_path, request_body)).hexdigest()
response = self.testapp.post(request_path, request_body, headers={'X-TBA-Auth-Id': 'tEsT_id_0', 'X-TBA-Auth-Sig': sig}, expect_errors=True)
self.assertEqual(response.status_code, 200)
db_eventteams = EventTeam.query(EventTeam.event == self.event.key).fetch(None)
self.assertEqual(len(db_eventteams), 3)
self.assertTrue('2014casj_frc254' in [et.key.id() for et in db_eventteams])
self.assertTrue('2014casj_frc971' in [et.key.id() for et in db_eventteams])
self.assertTrue('2014casj_frc604' in [et.key.id() for et in db_eventteams])
team_list = ['frc254', 'frc100']
request_body = json.dumps(team_list)
sig = md5.new('{}{}{}'.format('321tEsTsEcReT', request_path, request_body)).hexdigest()
response = self.testapp.post(request_path, request_body, headers={'X-TBA-Auth-Id': 'tEsT_id_0', 'X-TBA-Auth-Sig': sig}, expect_errors=True)
self.assertEqual(response.status_code, 200)
db_eventteams = EventTeam.query(EventTeam.event == self.event.key).fetch(None)
self.assertEqual(len(db_eventteams), 2)
self.assertTrue('2014casj_frc254' in [et.key.id() for et in db_eventteams])
self.assertTrue('2014casj_frc100' in [et.key.id() for et in db_eventteams])
def get_events_and_matches_async():
if return_valid_years:
event_team_keys_query = EventTeam.query(EventTeam.team == team.key)
else:
event_team_keys_query = EventTeam.query(EventTeam.team == team.key, EventTeam.year == year)
event_team_keys = yield event_team_keys_query.fetch_async(1000, keys_only=True)
event_teams = yield ndb.get_multi_async(event_team_keys)
event_keys = []
for event_team in event_teams:
if return_valid_years:
valid_years.add(event_team.year) # valid_years is a "global" variable (defined below). Doing this removes the complexity of having to propagate the years up through the tasklet call chain.
if not return_valid_years or event_team.year == year:
event_keys.append(event_team.event)
events, matches = yield ndb.get_multi_async(event_keys), get_matches_async(event_keys)
raise ndb.Return((events, matches))
def team_updated(affected_refs):
team_keys = _filter(affected_refs['key'])
event_team_keys_future = EventTeam.query(EventTeam.team.IN([team_key for team_key in team_keys])).fetch_async(None, keys_only=True)
district_team_keys_future = DistrictTeam.query(DistrictTeam.team.IN([team_key for team_key in team_keys])).fetch_async(None, keys_only=True)
queries_and_keys = []
for team_key in team_keys:
queries_and_keys.append((TeamQuery(team_key.id())))
page_num = _get_team_page_num(team_key.id())
queries_and_keys.append((TeamListQuery(page_num)))
for et_key in event_team_keys_future.get_result():
year = int(et_key.id()[:4])
event_key = et_key.id().split('_')[0]
page_num = _get_team_page_num(et_key.id().split('_')[1])
queries_and_keys.append((TeamListYearQuery(year, page_num)))
queries_and_keys.append((EventTeamsQuery(event_key)))
queries_and_keys.append((EventEventTeamsQuery(event_key)))
for dt_key in district_team_keys_future.get_result():
district_key = dt_key.id().split('_')[0]
queries_and_keys.append((DistrictTeamsQuery(district_key)))
return queries_and_keys
def get_event_details_cache_keys_and_controllers(cls, affected_refs):
"""
Gets cache keys and controllers that references this EventDetails
TODO: Currently inefficient and also clears event APIv2 endpoints,
since alliances are served under the event.
APIv3 will break alliances out.
"""
event_details_keys = affected_refs['key']
event_keys = set()
years = set()
event_district_abbrevs = set()
for event_details_key in event_details_keys:
event_key = ndb.Key(Event, event_details_key.id())
event_keys.add(event_key)
event = event_key.get()
years.add(event.year)
event_district_abbrevs.add(event.event_district_abbrev)
event_team_keys_future = EventTeam.query(EventTeam.event.IN([event_key for event_key in event_keys])).fetch_async(None, keys_only=True)
team_keys = set()
for et_key in event_team_keys_future.get_result():
team_key_name = et_key.id().split('_')[1]
team_keys.add(ndb.Key(Team, team_key_name))
return cls._get_events_cache_keys_and_controllers(event_keys) + \
cls._get_event_district_points_cache_keys_and_controllers(event_keys) + \
cls._get_eventlist_cache_keys_and_controllers(years) + \
cls._get_team_events_cache_keys_and_controllers(team_keys, years) + \
cls._get_districtlist_cache_keys_and_controllers(years) + \
cls._get_district_events_cache_keys_and_controllers(event_district_abbrevs, years) + \
cls._get_district_rankings_cache_keys_and_controllers(event_district_abbrevs, years)
def addTeamDetails(self, team_dict, year):
"""
Consume a Team dict, and return it with a year's Events filtered and Matches added
"""
# TODO Matches should live under Events - gregmarra 1 feb 2011
# TODO Filter Events by year - gregmarra 1 feb 2011
memcache_key = "api_team_details_%s_%s" % (team_dict["key"], year)
matches_list = memcache.get(memcache_key)
if matches_list is None:
matches = list()
team = Team.get_by_id(team_dict["key"])
for e in [a.event.get() for a in EventTeam.query(EventTeam.team == team.key).fetch(1000) if a.year == year]:
match_list = Match.query(Match.event == event.key, Match.team_key_names == team.key_name).fetch(500)
matches.extend(match_list)
matches_list = list()
for match in matches:
match_dict = dict()
match_dict["key"] = match.key_name
match_dict["event"] = match.event
match_dict["comp_level"] = match.comp_level
match_dict["set_number"] = match.set_number
match_dict["match_number"] = match.match_number
match_dict["team_keys"] = match.team_key_names
match_dict["alliances"] = json.loads(match.alliances_json)
matches_list.append(match_dict)
#TODO: Reduce caching time before 2013 season. 2592000 is one month -gregmarra
if tba_config.CONFIG["memcache"]: memcache.set(memcache_key, matches_list, 2592000)
team_dict["matches"] = matches_list
return team_dict
def get_event_cache_keys_and_controllers(cls, affected_refs):
"""
Gets cache keys and controllers that references this event
"""
event_keys = affected_refs["key"]
years = affected_refs["year"]
event_district_abbrevs = affected_refs["event_district_abbrev"]
event_team_keys_future = EventTeam.query(
EventTeam.event.IN([event_key for event_key in event_keys])
).fetch_async(None, keys_only=True)
team_keys = set()
for et_key in event_team_keys_future.get_result():
team_key_name = et_key.id().split("_")[1]
team_keys.add(ndb.Key(Team, team_key_name))
return (
cls._get_events_cache_keys_and_controllers(event_keys)
+ cls._get_event_district_points_cache_keys_and_controllers(event_keys)
+ cls._get_eventlist_cache_keys_and_controllers(years)
+ cls._get_team_events_cache_keys_and_controllers(team_keys, years)
+ cls._get_districtlist_cache_keys_and_controllers(years)
+ cls._get_district_events_cache_keys_and_controllers(event_district_abbrevs, years)
+ cls._get_district_rankings_cache_keys_and_controllers(event_district_abbrevs, years)
)
def update(self, event_key):
"""
Updates EventTeams for an event.
Returns a tuple of (teams, event_teams, event_team_keys_to_delete)
An EventTeam is valid iff the team:
a) played a match at the event,
b) the team received an award at the event,
c) or the event has not yet occurred.
"""
event = Event.get_by_id(event_key)
# Add teams from Matches and Awards
team_ids = set()
match_key_futures = Match.query(
Match.event == event.key).fetch_async(1000, keys_only=True)
award_key_futures = Award.query(
Award.event == event.key).fetch_async(1000, keys_only=True)
match_futures = ndb.get_multi_async(match_key_futures.get_result())
award_futures = ndb.get_multi_async(award_key_futures.get_result())
for match_future in match_futures:
match = match_future.get_result()
for team in match.team_key_names:
team_ids.add(team)
for award_future in award_futures:
award = award_future.get_result()
for team_key in award.team_list:
team_ids.add(team_key.id())
# Create or update EventTeams
teams = [Team(id=team_id,
team_number=int(team_id[3:]))
for team_id in team_ids]
if teams:
event_teams = [EventTeam(id=event_key + "_" + team.key.id(),
event=event.key,
team=team.key,
year=event.year)
for team in teams]
else:
event_teams = None
# Delete EventTeams for teams who did not participate in the event
# Only runs if event is over
existing_event_teams_keys = EventTeam.query(
EventTeam.event == event.key).fetch(1000, keys_only=True)
existing_event_teams = ndb.get_multi(existing_event_teams_keys)
existing_team_ids = set()
for et in existing_event_teams:
existing_team_ids.add(et.team.id())
et_keys_to_delete = set()
if event.end_date is not None and event.end_date < datetime.datetime.now():
for team_id in existing_team_ids.difference(team_ids):
et_key_name = "{}_{}".format(event.key_name, team_id)
et_keys_to_delete.add(ndb.Key(EventTeam, et_key_name))
ndb.delete_multi(et_keys_to_delete)
return teams, event_teams, et_keys_to_delete
def event_updated(affected_refs):
event_keys = filter(None, affected_refs["key"])
years = filter(None, affected_refs["year"])
event_district_keys = filter(None, affected_refs["event_district_key"])
event_team_keys_future = EventTeam.query(EventTeam.event.IN([event_key for event_key in event_keys])).fetch_async(
None, keys_only=True
)
queries_and_keys = []
for event_key in event_keys:
queries_and_keys.append((EventQuery(event_key.id())))
for year in years:
queries_and_keys.append((EventListQuery(year)))
for event_district_key in event_district_keys:
queries_and_keys.append((DistrictEventsQuery(event_district_key)))
for et_key in event_team_keys_future.get_result():
team_key = et_key.id().split("_")[1]
year = int(et_key.id()[:4])
queries_and_keys.append((TeamEventsQuery(team_key)))
queries_and_keys.append((TeamYearEventsQuery(team_key, year)))
return queries_and_keys
def event_updated(affected_refs):
event_keys = _filter(affected_refs['key'])
years = _filter(affected_refs['year'])
event_district_keys = _filter(affected_refs['district_key'])
event_team_keys_future = EventTeam.query(EventTeam.event.IN([event_key for event_key in event_keys])).fetch_async(None, keys_only=True)
events_future = ndb.get_multi_async(event_keys)
queries_and_keys = []
for event_key in event_keys:
queries_and_keys.append((EventQuery(event_key.id())))
queries_and_keys.append(EventDivisionsQuery(event_key.id()))
for year in years:
queries_and_keys.append((EventListQuery(year)))
for event_district_key in event_district_keys:
queries_and_keys.append((DistrictEventsQuery(event_district_key.id())))
if event_keys:
for et_key in event_team_keys_future.get_result():
team_key = et_key.id().split('_')[1]
year = int(et_key.id()[:4])
queries_and_keys.append((TeamEventsQuery(team_key)))
queries_and_keys.append((TeamYearEventsQuery(team_key, year)))
queries_and_keys.append((TeamYearEventTeamsQuery(team_key, year)))
events_with_parents = filter(lambda e: e.get_result() is not None and e.get_result().parent_event is not None, events_future)
parent_keys = set([e.get_result().parent_event for e in events_with_parents])
for parent_key in parent_keys:
queries_and_keys.append((EventDivisionsQuery(parent_key.id())))
return queries_and_keys
def getTeamInfo(self, team_key):
"""
Return a Team dict with basic information.
"""
memcache_key = "api_team_info_%s" % team_key
team_dict = memcache.get(memcache_key)
if team_dict is None:
team = Team.get_by_id(team_key)
if team is not None:
team_dict = dict()
team_dict["key"] = team.key_name
team_dict["team_number"] = team.team_number
team_dict["name"] = team.name
team_dict["nickname"] = team.nickname
team_dict["website"] = team.website
team_dict["location"] = team.location
event_teams = EventTeam.query(EventTeam.team == team.key,\
EventTeam.year == datetime.now().year)\
.fetch(1000, projection=[EventTeam.event])
team_dict["events"] = [event_team.event.id() for event_team in event_teams]
try:
team_dict["location"] = team.location
team_dict["locality"] = team.locality
team_dict["region"] = team.region
team_dict["country_name"] = team.country_name
except Exception, e:
logging.warning("Failed to include Address for api_team_info_%s: %s" % (team_key, e))
#TODO: Reduce caching time before 2013 season. 2592000 is one month -gregmarra
if tba_config.CONFIG["memcache"]: memcache.set(memcache_key, team_dict, 2592000)
else:
raise IndexError
def _render(self, team_number):
team = Team.get_by_id("frc" + team_number)
if not team:
return self.redirect("/error/404")
event_team_keys_future = EventTeam.query(EventTeam.team == team.key).fetch_async(1000, keys_only=True)
award_keys_future = Award.query(Award.team == team.key).fetch_async(1000, keys_only=True)
event_teams_futures = ndb.get_multi_async(event_team_keys_future.get_result())
awards_futures = ndb.get_multi_async(award_keys_future.get_result())
event_keys = [event_team_future.get_result().event for event_team_future in event_teams_futures]
events_futures = ndb.get_multi_async(event_keys)
awards_by_event = {}
for award_future in awards_futures:
award = award_future.get_result()
if award.event.id() not in awards_by_event:
awards_by_event[award.event.id()] = [award]
else:
awards_by_event[award.event.id()].append(award)
event_awards = []
current_event = None
matches_upcoming = None
short_cache = False
for event_future in events_futures:
event = event_future.get_result()
if event.now:
current_event = event
team_matches_future = Match.query(Match.event == event.key, Match.team_key_names == team.key_name)\
.fetch_async(500, keys_only=True)
matches = ndb.get_multi(team_matches_future.get_result())
matches_upcoming = MatchHelper.upcomingMatches(matches)
if event.within_a_day:
short_cache = True
if event.key_name in awards_by_event:
sorted_awards = AwardHelper.organizeAwards(awards_by_event[event.key_name])['list']
else:
sorted_awards = []
event_awards.append((event, sorted_awards))
event_awards = sorted(event_awards, key=lambda (e, _): e.start_date if e.start_date else datetime.datetime(e.year, 12, 31))
years = sorted(set([et.get_result().year for et in event_teams_futures if et.get_result().year != None]))
template_values = {'team': team,
'event_awards': event_awards,
'years': years,
'current_event': current_event,
'matches_upcoming': matches_upcoming}
if short_cache:
self._cache_expiration = self.SHORT_CACHE_EXPIRATION
path = os.path.join(os.path.dirname(__file__), '../templates/team_history.html')
return template.render(path, template_values)
def media_updated(affected_refs):
reference_keys = _filter(affected_refs['references'])
years = _filter(affected_refs['year'])
media_tags = _filter(affected_refs['media_tag_enum'])
team_keys = filter(lambda x: x.kind() == 'Team', reference_keys)
event_team_keys_future = EventTeam.query(EventTeam.team.IN(team_keys)).fetch_async(None, keys_only=True) if team_keys else None
queries_and_keys = []
for reference_key in reference_keys:
if reference_key.kind() == 'Team':
for year in years:
queries_and_keys.append((TeamYearMediaQuery(reference_key.id(), year)))
for media_tag in media_tags:
queries_and_keys.append((TeamYearTagMediasQuery(reference_key.id(), media_tag, year)))
for media_tag in media_tags:
queries_and_keys.append((TeamTagMediasQuery(reference_key.id(), media_tag)))
queries_and_keys.append((TeamSocialMediaQuery(reference_key.id())))
if reference_key.kind() == 'Event':
queries_and_keys.append((EventMediasQuery(reference_key.id())))
if event_team_keys_future:
for event_team_key in event_team_keys_future.get_result():
event_key = event_team_key.id().split('_')[0]
year = int(event_key[:4])
if year in years:
queries_and_keys.append(EventTeamsMediasQuery(event_key))
queries_and_keys.append(EventTeamsPreferredMediasQuery(event_key))
return queries_and_keys
def get(self, team_number):
self._require_admin()
team = Team.get_by_id("frc" + team_number)
event_teams = EventTeam.query(EventTeam.team == team.key).fetch(500)
team_medias = Media.query(Media.references == team.key).fetch(500)
robots = Robot.query(Robot.team == team.key).fetch()
district_teams = DistrictTeam.query(DistrictTeam.team == team.key).fetch()
team_medias_by_year = {}
for media in team_medias:
if media.year in team_medias_by_year:
team_medias_by_year[media.year].append(media)
else:
team_medias_by_year[media.year] = [media]
self.template_values.update({
'event_teams': event_teams,
'team': team,
'team_medias_by_year': team_medias_by_year,
'robots': robots,
'district_teams': district_teams,
})
path = os.path.join(os.path.dirname(__file__), '../../templates/admin/team_details.html')
self.response.out.write(template.render(path, self.template_values))
def get(self, team_number):
self._require_admin()
team = Team.get_by_id("frc" + team_number)
if not team:
self.abort(404)
event_teams = EventTeam.query(EventTeam.team == team.key).fetch(500)
team_medias = Media.query(Media.references == team.key).fetch(500)
robots = Robot.query(Robot.team == team.key).fetch()
district_teams = DistrictTeam.query(DistrictTeam.team == team.key).fetch()
years_participated = sorted(TeamParticipationQuery(team.key_name).fetch())
team_medias_by_year = {}
for media in team_medias:
if media.year in team_medias_by_year:
team_medias_by_year[media.year].append(media)
else:
team_medias_by_year[media.year] = [media]
media_years = sorted(team_medias_by_year.keys(), reverse=True)
self.template_values.update({
'event_teams': event_teams,
'team': team,
'team_media_years': media_years,
'team_medias_by_year': team_medias_by_year,
'robots': robots,
'district_teams': district_teams,
'years_participated': years_participated,
})
path = os.path.join(os.path.dirname(__file__), '../../templates/admin/team_details.html')
self.response.out.write(template.render(path, self.template_values))
def prepTeams(self):
# TODO there is a way to do this with yields such that this would be a
# generator function that would yield, and if two sets of ndb fetches
# went by would cleanly do itself without forcing a fetch.
# -gregmarra 20121007
from models.event_team import EventTeam
if self._teams_future is None:
self._event_teams_future = EventTeam.query(EventTeam.event == self.key).fetch_async(500)
def _render(self, team_key):
event_team_keys = EventTeam.query(EventTeam.team == ndb.Key(Team, team_key)).fetch(None, keys_only=True)
years_participated = set()
for event_team_key in event_team_keys:
years_participated.add(int(event_team_key.id()[:4]))
years_participated = sorted(list(years_participated))
return json.dumps(years_participated, ensure_ascii=True)
def _query_async(self):
team_key = self._query_args[0]
year = self._query_args[1]
event_teams = yield EventTeam.query(
EventTeam.team == ndb.Key(Team, team_key),
EventTeam.year == year).fetch_async()
event_keys = map(lambda event_team: event_team.event, event_teams)
events = yield ndb.get_multi_async(event_keys)
raise ndb.Return(events)
def getTeamData(self, event):
#reader = csv.reader(open(file,"rb"))
event_teams = EventTeam.query(EventTeam.event == event.key).fetch(500)
team_keys = [event_team.team for event_team in event_teams]
for num, team_key in enumerate(team_keys):
OprHelper.teamdata.append([])
OprHelper.teamdata[num].append(num) # teamid
OprHelper.teamdata[num].append(int(team_key[3:])) # teamnumber
def get(self, event_key):
self._require_admin()
event = Event.get_by_id(event_key)
if not event:
self.abort(404)
return
existing_event_team_keys = set(EventTeam.query(EventTeam.event == event.key).fetch(1000, keys_only=True))
EventTeamManipulator.delete_keys(existing_event_team_keys)
self.response.out.write("Deleted {} EventTeams from {}".format(len(existing_event_team_keys), event_key))
def getAwards(self, event):
awards = []
if event.event_type_enum == EventType.CMP_DIVISION and event.year >= 2015: # 8 subdivisions from 2015+ have awards listed under 4 divisions
event_team_keys = EventTeam.query(EventTeam.event == event.key).fetch(keys_only=True)
valid_team_nums = set([int(etk.id().split('_')[1][3:]) for etk in event_team_keys])
awards += self._parse(self.FMS_API_AWARDS_URL_PATTERN % (event.year, self._get_event_short(self.SUBDIV_TO_DIV[event.event_short])), FMSAPIAwardsParser(event, valid_team_nums))
awards += self._parse(self.FMS_API_AWARDS_URL_PATTERN % (event.year, self._get_event_short(event.event_short)), FMSAPIAwardsParser(event))
return awards
def get(self, year, first_eid):
df = DatafeedUsfirst()
df_legacy = DatafeedUsfirstLegacy()
event = df.getEventDetails(first_eid)
if not event:
logging.warning("getEventDetails with DatafeedUsfirst for event id {} failed. Retrying with DatafeedUsfirstLegacy.".format(first_eid))
event = df_legacy.getEventDetails(int(year), first_eid)
if self.request.get('event_district_enum'):
event.event_district_enum = int(self.request.get('event_district_enum'))
event = EventManipulator.createOrUpdate(event)
teams = df.getEventTeams(int(year), first_eid)
if not teams:
logging.warning("getEventTeams with DatafeedUsfirst for event id {} failed. Retrying with DatafeedUsfirstLegacy.".format(first_eid))
teams = df_legacy.getEventTeams(int(year), first_eid)
if not teams:
logging.warning("getEventTeams with DatafeedUsfirstLegacy for event id {} failed.".format(first_eid))
teams = []
teams = TeamManipulator.createOrUpdate(teams)
if teams:
if type(teams) is not list:
teams = [teams]
event_teams = [EventTeam(
id=event.key.id() + "_" + team.key.id(),
event=event.key,
team=team.key,
year=event.year)
for team in teams]
# Delete eventteams of teams that unregister from an event
if event.future:
existing_event_team_keys = set(EventTeam.query(EventTeam.event == event.key).fetch(1000, keys_only=True))
event_team_keys = set([et.key for et in event_teams])
et_keys_to_delete = existing_event_team_keys.difference(event_team_keys)
EventTeamManipulator.delete_keys(et_keys_to_delete)
event_teams = EventTeamManipulator.createOrUpdate(event_teams)
if type(event_teams) is not list:
event_teams = [event_teams]
else:
event_teams = []
template_values = {
'event': event,
'event_teams': event_teams,
}
path = os.path.join(os.path.dirname(__file__), '../templates/datafeeds/usfirst_event_details_get.html')
self.response.out.write(template.render(path, template_values))
def get(self, event_key):
event = Event.get_by_id(event_key)
event_teams = EventTeam.query(EventTeam.event==event.key).fetch()
for event_team in event_teams:
status = EventTeamStatusHelper.generate_team_at_event_status(event_team.team.id(), event)
event_team.status = status
FirebasePusher.update_event_team_status(event_key, event_team.team.id(), status)
EventTeamManipulator.createOrUpdate(event_teams)
if 'X-Appengine-Taskname' not in self.request.headers: # Only write out if not in taskqueue
self.response.out.write("Finished calculating event team statuses for: {}".format(event_key))
def _render(self, team_key, year=None):
self._set_team(team_key)
event_team_keys = EventTeam.query(EventTeam.team == self.team.key, EventTeam.year == self.year).fetch(1000, keys_only=True)
event_teams = ndb.get_multi(event_team_keys)
event_keys = [event_team.event for event_team in event_teams]
events = ndb.get_multi(event_keys)
events = [ModelToDict.eventConverter(event) for event in events]
return json.dumps(events, ensure_ascii=True)
def _query_async(self):
event_key = self._query_args[0]
year = int(event_key[:4])
event_team_keys = yield EventTeam.query(EventTeam.event == ndb.Key(Event, event_key)).fetch_async(keys_only=True)
if not event_team_keys:
raise ndb.Return([])
team_keys = map(lambda event_team_key: ndb.Key(Team, event_team_key.id().split('_')[1]), event_team_keys)
medias = yield Media.query(
Media.preferred_references.IN(team_keys),
Media.year == year).fetch_async()
raise ndb.Return(medias)
def test_repair(self):
event_team = EventTeam.get_by_id("2011ct_frc177")
self.assertEqual(event_team.year, None)
broken_event_teams = EventTeam.query(EventTeam.year == None).fetch()
self.assertGreater(len(broken_event_teams), 0)
fixed_event_teams = EventTeamRepairer.repair(broken_event_teams)
fixed_event_teams = EventTeamManipulator.createOrUpdate(fixed_event_teams)
event_team = EventTeam.get_by_id("2011ct_frc177")
self.assertEqual(event_team.year, 2011)
def get(self, event_key):
df = DatafeedFMSAPI('v2.0')
event = Event.get_by_id(event_key)
models = df.getEventTeams(event_key)
teams = []
district_teams = []
robots = []
for group in models:
# models is a list of tuples (team, districtTeam, robot)
if isinstance(group[0], Team):
teams.append(group[0])
if isinstance(group[1], DistrictTeam):
district_teams.append(group[1])
if isinstance(group[2], Robot):
robots.append(group[2])
# Write new models
teams = TeamManipulator.createOrUpdate(teams)
district_teams = DistrictTeamManipulator.createOrUpdate(district_teams)
robots = RobotManipulator.createOrUpdate(robots)
if not teams:
# No teams found registered for this event
teams = []
# Build EventTeams
event_teams = [EventTeam(
id=event.key_name + "_" + team.key_name,
event=event.key,
team=team.key,
year=event.year)
for team in teams]
# Delete eventteams of teams that are no longer registered
if event.future:
existing_event_team_keys = set(EventTeam.query(EventTeam.event == event.key).fetch(1000, keys_only=True))
event_team_keys = set([et.key for et in event_teams])
et_keys_to_delete = existing_event_team_keys.difference(event_team_keys)
EventTeamManipulator.delete_keys(et_keys_to_delete)
event_teams = EventTeamManipulator.createOrUpdate(event_teams)
if type(event_teams) is not list:
event_teams = [event_teams]
template_values = {
'event': event,
'event_teams': event_teams,
}
path = os.path.join(os.path.dirname(__file__), '../templates/datafeeds/usfirst_event_details_get.html')
self.response.out.write(template.render(path, template_values))
def get(self, team_number):
self._require_admin()
team = Team.get_by_id("frc" + team_number)
event_teams = EventTeam.query(EventTeam.team == team.key).fetch(500)
self.template_values.update({
'event_teams': event_teams,
'team': team,
})
path = os.path.join(os.path.dirname(__file__), '../../templates/admin/team_details.html')
self.response.out.write(template.render(path, self.template_values))
def _render(self, district_abbrev, year=None, explicit_year=False):
district_type = DistrictType.abbrevs[district_abbrev]
event_keys = Event.query(Event.year == year, Event.event_district_enum == district_type).fetch(None, keys_only=True)
if not event_keys:
self.abort(404)
# needed for valid_years
all_cmp_event_keys_future = Event.query(Event.event_district_enum == district_type, Event.event_type_enum == EventType.DISTRICT_CMP).fetch_async(None, keys_only=True)
# needed for valid_districts
district_cmp_keys_future = Event.query(Event.year == year, Event.event_type_enum == EventType.DISTRICT_CMP).fetch_async(None, keys_only=True) # to compute valid_districts
event_futures = ndb.get_multi_async(event_keys)
event_team_keys_future = EventTeam.query(EventTeam.event.IN(event_keys)).fetch_async(None, keys_only=True)
if year == 2014: # TODO: only 2014 has accurate rankings calculations
team_futures = ndb.get_multi_async(set([ndb.Key(Team, et_key.id().split('_')[1]) for et_key in event_team_keys_future.get_result()]))
events = [event_future.get_result() for event_future in event_futures]
EventHelper.sort_events(events)
district_cmp_futures = ndb.get_multi_async(district_cmp_keys_future.get_result())
if year == 2014: # TODO: only 2014 has accurate rankings calculations
team_totals = DistrictHelper.calculate_rankings(events, team_futures, year)
else:
team_totals = None
valid_districts = set()
for district_cmp_future in district_cmp_futures:
district_cmp = district_cmp_future.get_result()
cmp_dis_type = district_cmp.event_district_enum
if cmp_dis_type is None:
logging.warning("District event {} has unknown district type!".format(district_cmp.key.id()))
else:
valid_districts.add((DistrictType.type_names[cmp_dis_type], DistrictType.type_abbrevs[cmp_dis_type]))
valid_districts = sorted(valid_districts, key=lambda (name, _): name)
self.template_values.update({
'explicit_year': explicit_year,
'year': year,
'valid_years': sorted(set([int(event_key.id()[:4]) for event_key in all_cmp_event_keys_future.get_result()])),
'valid_districts': valid_districts,
'district_name': DistrictType.type_names[district_type],
'district_abbrev': district_abbrev,
'events': events,
'team_totals': team_totals,
})
path = os.path.join(os.path.dirname(__file__), '../templates/district_details.html')
return template.render(path, self.template_values)
def _query_async(self):
year = self._query_args[0]
page_num = self._query_args[1]
event_team_keys_future = EventTeam.query(EventTeam.year == year).fetch_async(keys_only=True)
teams_future = TeamListQuery(page_num).fetch_async()
year_team_keys = set()
for et_key in event_team_keys_future.get_result():
team_key = et_key.id().split('_')[1]
year_team_keys.add(team_key)
teams = filter(lambda team: team.key.id() in year_team_keys, teams_future.get_result())
raise ndb.Return(teams)
def _render(self, district_abbrev, year=None):
self._set_district(district_abbrev)
if self.year < 2009:
return json.dumps([], ensure_ascii=True)
event_keys = Event.query(Event.year == self.year, Event.event_district_enum == self.district).fetch(
None, keys_only=True
)
if not event_keys:
return json.dumps([], ensure_ascii=True)
events = ndb.get_multi(event_keys)
event_futures = ndb.get_multi_async(event_keys)
event_team_keys_future = EventTeam.query(EventTeam.event.IN(event_keys)).fetch_async(None, keys_only=True)
team_futures = ndb.get_multi_async(
set([ndb.Key(Team, et_key.id().split("_")[1]) for et_key in event_team_keys_future.get_result()])
)
events = [event_future.get_result() for event_future in event_futures]
EventHelper.sort_events(events)
team_totals = DistrictHelper.calculate_rankings(events, team_futures, self.year)
rankings = []
current_rank = 1
for key, points in team_totals:
point_detail = {}
point_detail["rank"] = current_rank
point_detail["team_key"] = key
point_detail["event_points"] = {}
for event in points["event_points"]:
event_key = event[0].key_name
point_detail["event_points"][event_key] = event[1]
event_details = Event.get_by_id(event_key)
point_detail["event_points"][event[0].key_name]["district_cmp"] = (
True if event_details.event_type_enum == EventType.DISTRICT_CMP else False
)
if "rookie_bonus" in points:
point_detail["rookie_bonus"] = points["rookie_bonus"]
else:
point_detail["rookie_bonus"] = 0
point_detail["point_total"] = points["point_total"]
rankings.append(point_detail)
current_rank += 1
return json.dumps(rankings)
def event_updated(affected_refs):
event_keys = _filter(affected_refs['key'])
years = _filter(affected_refs['year'])
event_district_keys = _filter(affected_refs['district_key'])
event_team_keys_future = EventTeam.query(
EventTeam.event.IN([event_key for event_key in event_keys
])).fetch_async(None, keys_only=True)
events_future = ndb.get_multi_async(event_keys)
queries_and_keys = []
for event_key in event_keys:
queries_and_keys.append((EventQuery(event_key.id())))
queries_and_keys.append(EventDivisionsQuery(event_key.id()))
for year in years:
queries_and_keys.append((EventListQuery(year)))
for event_district_key in event_district_keys:
queries_and_keys.append((DistrictEventsQuery(event_district_key.id())))
if event_keys:
for et_key in event_team_keys_future.get_result():
team_key = et_key.id().split('_')[1]
year = int(et_key.id()[:4])
queries_and_keys.append((TeamEventsQuery(team_key)))
queries_and_keys.append((TeamYearEventsQuery(team_key, year)))
queries_and_keys.append((TeamYearEventTeamsQuery(team_key, year)))
events_with_parents = filter(
lambda e: e.get_result() is not None and e.get_result().parent_event is
not None, events_future)
parent_keys = set(
[e.get_result().parent_event for e in events_with_parents])
for parent_key in parent_keys:
queries_and_keys.append((EventDivisionsQuery(parent_key.id())))
return queries_and_keys
def get(self, team_number):
self._require_admin()
team = Team.get_by_id("frc" + team_number)
event_teams = EventTeam.query(EventTeam.team == team.key).fetch(500)
team_medias = Media.query(Media.references == team.key).fetch(500)
team_medias_by_year = {}
for media in team_medias:
if media.year in team_medias_by_year:
team_medias_by_year[media.year].append(media)
else:
team_medias_by_year[media.year] = [media]
self.template_values.update({
'event_teams': event_teams,
'team': team,
'team_medias_by_year': team_medias_by_year,
})
path = os.path.join(os.path.dirname(__file__),
'../../templates/admin/team_details.html')
self.response.out.write(template.render(path, self.template_values))
def media_updated(affected_refs):
reference_keys = filter(None, affected_refs['references'])
years = filter(None, affected_refs['year'])
team_keys = filter(lambda x: x.kind() == 'Team', reference_keys)
event_team_keys_future = EventTeam.query(
EventTeam.team.IN(team_keys)).fetch_async(None, keys_only=True)
queries_and_keys = []
for year in years:
for reference_key in reference_keys:
if reference_key.kind() == 'Team':
queries_and_keys.append(
(TeamYearMediaQuery(reference_key.id(), year)))
for event_team_key in event_team_keys_future.get_result():
event_key = event_team_key.id().split('_')[0]
year = int(event_key[:4])
if year in years:
queries_and_keys.append(EventTeamsMediasQuery(event_key))
queries_and_keys.append(EventTeamsPreferredMediasQuery(event_key))
return queries_and_keys
def event_updated(affected_refs):
event_keys = filter(None, affected_refs['key'])
years = filter(None, affected_refs['year'])
event_district_keys = filter(None, affected_refs['event_district_key'])
event_team_keys_future = EventTeam.query(
EventTeam.event.IN([event_key for event_key in event_keys
])).fetch_async(None, keys_only=True)
queries_and_keys = []
for year in years:
queries_and_keys.append((EventListQuery(year)))
for event_district_key in event_district_keys:
queries_and_keys.append((DistrictEventsQuery(event_district_key)))
for et_key in event_team_keys_future.get_result():
team_key = et_key.id().split('_')[1]
year = int(et_key.id()[:4])
queries_and_keys.append((TeamEventsQuery(team_key)))
queries_and_keys.append((TeamYearEventsQuery(team_key, year)))
return queries_and_keys
def team_updated(affected_refs):
team_keys = filter(None, affected_refs['key'])
event_team_keys_future = EventTeam.query(EventTeam.team.IN([team_key for team_key in team_keys])).fetch_async(None, keys_only=True)
district_team_keys_future = DistrictTeam.query(DistrictTeam.team.IN([team_key for team_key in team_keys])).fetch_async(None, keys_only=True)
queries_and_keys = []
for team_key in team_keys:
page_num = _get_team_page_num(team_key.id())
queries_and_keys.append((TeamListQuery(page_num)))
for et_key in event_team_keys_future.get_result():
year = int(et_key.id()[:4])
event_key = et_key.id().split('_')[0]
page_num = _get_team_page_num(et_key.id().split('_')[1])
queries_and_keys.append((TeamListYearQuery(year, page_num)))
queries_and_keys.append((EventTeamsQuery(event_key)))
for dt_key in district_team_keys_future.get_result():
district_key = dt_key.id().split('_')[0]
queries_and_keys.append((DistrictTeamsQuery(district_key)))
return queries_and_keys
def get_event_cache_keys_and_controllers(cls, affected_refs):
"""
Gets cache keys and controllers that references this event
"""
event_keys = affected_refs['key']
years = affected_refs['year']
event_district_abbrevs = map(lambda x: x.id()[4:], filter(None, affected_refs['district_key']))
event_team_keys_future = EventTeam.query(EventTeam.event.IN([event_key for event_key in event_keys])).fetch_async(None, keys_only=True)
team_keys = set()
for et_key in event_team_keys_future.get_result():
team_key_name = et_key.id().split('_')[1]
team_keys.add(ndb.Key(Team, team_key_name))
return cls._get_events_cache_keys_and_controllers(event_keys) + \
cls._get_event_district_points_cache_keys_and_controllers(event_keys) + \
cls._get_eventlist_cache_keys_and_controllers(years) + \
cls._get_team_events_cache_keys_and_controllers(team_keys, years) + \
cls._get_districtlist_cache_keys_and_controllers(years) + \
cls._get_district_events_cache_keys_and_controllers(event_district_abbrevs, years) + \
cls._get_district_rankings_cache_keys_and_controllers(event_district_abbrevs, years) + \
cls._queries_to_cache_keys_and_controllers(get_affected_queries.event_updated(affected_refs))
def get_event_details_cache_keys_and_controllers(cls, affected_refs):
"""
Gets cache keys and controllers that references this EventDetails
TODO: Currently inefficient and also clears event APIv2 endpoints,
since alliances are served under the event.
APIv3 will break alliances out.
"""
event_details_keys = affected_refs['key']
event_keys = set()
years = set()
event_district_abbrevs = set()
for event_details_key in event_details_keys:
event_key = ndb.Key(Event, event_details_key.id())
event_keys.add(event_key)
event = event_key.get()
years.add(event.year)
event_district_abbrevs.add(event.event_district_abbrev)
event_team_keys_future = EventTeam.query(
EventTeam.event.IN([event_key for event_key in event_keys
])).fetch_async(None, keys_only=True)
team_keys = set()
for et_key in event_team_keys_future.get_result():
team_key_name = et_key.id().split('_')[1]
team_keys.add(ndb.Key(Team, team_key_name))
return cls._get_events_cache_keys_and_controllers(event_keys) + \
cls._get_event_district_points_cache_keys_and_controllers(event_keys) + \
cls._get_eventlist_cache_keys_and_controllers(years) + \
cls._get_team_events_cache_keys_and_controllers(team_keys, years) + \
cls._get_districtlist_cache_keys_and_controllers(years) + \
cls._get_district_events_cache_keys_and_controllers(event_district_abbrevs, years) + \
cls._get_district_rankings_cache_keys_and_controllers(event_district_abbrevs, years) + \
cls._queries_to_cache_keys_and_controllers(get_affected_queries.event_details_updated(affected_refs))
def get_team_cache_keys_and_controllers(cls, affected_refs):
"""
Gets cache keys and controllers that references this team
"""
team_keys = affected_refs['key']
event_team_keys_future = EventTeam.query(EventTeam.team.IN([team_key for team_key in team_keys])).fetch_async(None, keys_only=True)
district_team_keys_future = DistrictTeam.query(DistrictTeam.team.IN([team_key for team_key in team_keys])).fetch_async(None, keys_only=True)
event_keys = set()
for et_key in event_team_keys_future.get_result():
event_key_name = et_key.id().split('_')[0]
event_keys.add(ndb.Key(Event, event_key_name))
district_keys = set()
for dt_key in district_team_keys_future.get_result():
district_key_name = dt_key.id().split('_')[0]
district_keys.add(ndb.Key(District, district_key_name))
return cls._get_teams_cache_keys_and_controllers(team_keys) + \
cls._get_eventteams_cache_keys_and_controllers(event_keys) + \
cls._get_teamlist_cache_keys_and_controllers(team_keys) + \
cls._get_districtteams_cache_keys_and_controllers(district_keys, team_keys) + \
cls._queries_to_cache_keys_and_controllers(get_affected_queries.team_updated(affected_refs))
def get(self, event_key):
df = DatafeedFMSAPI('v2.0')
df2 = DatafeedFIRSTElasticSearch()
event = Event.get_by_id(event_key)
# Update event
updated_event = df2.getEventDetails(event)
if updated_event:
event = EventManipulator.createOrUpdate(updated_event)
models = df.getEventTeams(event_key)
teams = []
district_teams = []
robots = []
for group in models:
# models is a list of tuples (team, districtTeam, robot)
if isinstance(group[0], Team):
teams.append(group[0])
if isinstance(group[1], DistrictTeam):
district_teams.append(group[1])
if isinstance(group[2], Robot):
robots.append(group[2])
# Merge teams
teams = TeamManipulator.mergeModels(teams, df2.getEventTeams(event))
# Write new models
if teams:
teams = TeamManipulator.createOrUpdate(teams)
district_teams = DistrictTeamManipulator.createOrUpdate(district_teams)
robots = RobotManipulator.createOrUpdate(robots)
if not teams:
# No teams found registered for this event
teams = []
if type(teams) is not list:
teams = [teams]
# Build EventTeams
event_teams = [EventTeam(
id=event.key_name + "_" + team.key_name,
event=event.key,
team=team.key,
year=event.year)
for team in teams]
# Delete eventteams of teams that are no longer registered
if event_teams != []:
existing_event_team_keys = set(EventTeam.query(EventTeam.event == event.key).fetch(1000, keys_only=True))
event_team_keys = set([et.key for et in event_teams])
et_keys_to_delete = existing_event_team_keys.difference(event_team_keys)
EventTeamManipulator.delete_keys(et_keys_to_delete)
event_teams = EventTeamManipulator.createOrUpdate(event_teams)
if type(event_teams) is not list:
event_teams = [event_teams]
template_values = {
'event': event,
'event_teams': event_teams,
}
if 'X-Appengine-Taskname' not in self.request.headers: # Only write out if not in taskqueue
path = os.path.join(os.path.dirname(__file__), '../templates/datafeeds/usfirst_event_details_get.html')
self.response.out.write(template.render(path, template_values))
def _render(self, *args, **kw):
week_events = EventHelper.getWeekEvents()
special_webcasts = FirebasePusher.get_special_webcasts()
events = {}
for event in week_events:
events[event.key.id()] = event
# Calculate popular teams
# Get cached team keys
event_team_keys = memcache.get_multi(events.keys(),
namespace='event-team-keys')
# Get uncached team keys
to_query = set(events.keys()).difference(event_team_keys.keys())
event_teams_futures = [
(event_key,
EventTeam.query(
EventTeam.event == ndb.Key(Event, event_key)).fetch_async(
projection=[EventTeam.team])) for event_key in to_query
]
# Merge cached and uncached team keys
for event_key, event_teams in event_teams_futures:
event_team_keys[event_key] = [
et.team.id() for et in event_teams.get_result()
]
memcache.set_multi(event_team_keys,
60 * 60 * 24,
namespace='event-team-keys')
team_keys = []
team_events = {}
for event_key, event_team_keys in event_team_keys.items():
team_keys += event_team_keys
for team_key in event_team_keys:
team_events[team_key] = events[event_key]
# Get cached counts
team_favorite_counts = memcache.get_multi(
team_keys, namespace='team-favorite-counts')
# Get uncached counts
to_count = set(team_keys).difference(team_favorite_counts.keys())
count_futures = [
(team_key,
Favorite.query(Favorite.model_key == team_key).count_async())
for team_key in to_count
]
# Merge cached and uncached counts
for team_key, count_future in count_futures:
team_favorite_counts[team_key] = count_future.get_result()
memcache.set_multi(team_favorite_counts,
60 * 60 * 24,
namespace='team-favorite-counts')
# Sort to get top popular teams
popular_team_keys = []
for team_key, _ in sorted(team_favorite_counts.items(),
key=lambda tc: -tc[1])[:25]:
popular_team_keys.append(ndb.Key(Team, team_key))
popular_teams = sorted(ndb.get_multi(popular_team_keys),
key=lambda team: team.team_number)
popular_teams_events = []
for team in popular_teams:
popular_teams_events.append((team, team_events[team.key.id()]))
self.template_values.update({
"events":
week_events,
"any_webcast_online":
any(w.get('status') == 'online' for w in special_webcasts),
"special_webcasts":
special_webcasts,
"popular_teams_events":
popular_teams_events,
})
path = os.path.join(os.path.dirname(__file__),
'../templates/index_competitionseason.html')
return template.render(path, self.template_values)
def get(self, event_key):
df = DatafeedFMSAPI('v2.0')
df2 = DatafeedFIRSTElasticSearch()
event = Event.get_by_id(event_key)
# Update event
fmsapi_events, fmsapi_districts = df.getEventDetails(event_key)
elasticsearch_events = df2.getEventDetails(event)
updated_event = EventManipulator.mergeModels(fmsapi_events,
elasticsearch_events)
if updated_event:
event = EventManipulator.createOrUpdate(updated_event)
DistrictManipulator.createOrUpdate(fmsapi_districts)
models = df.getEventTeams(event_key)
teams = []
district_teams = []
robots = []
for group in models:
# models is a list of tuples (team, districtTeam, robot)
if isinstance(group[0], Team):
teams.append(group[0])
if isinstance(group[1], DistrictTeam):
district_teams.append(group[1])
if isinstance(group[2], Robot):
robots.append(group[2])
# Merge teams
teams = TeamManipulator.mergeModels(teams, df2.getEventTeams(event))
# Write new models
if teams and event.year == tba_config.MAX_YEAR: # Only update from latest year
teams = TeamManipulator.createOrUpdate(teams)
district_teams = DistrictTeamManipulator.createOrUpdate(district_teams)
robots = RobotManipulator.createOrUpdate(robots)
if not teams:
# No teams found registered for this event
teams = []
if type(teams) is not list:
teams = [teams]
# Build EventTeams
cmp_hack_sitevar = Sitevar.get_or_insert('cmp_registration_hacks')
events_without_eventteams = cmp_hack_sitevar.contents.get('skip_eventteams', []) \
if cmp_hack_sitevar else []
skip_eventteams = event_key in events_without_eventteams
event_teams = [
EventTeam(id=event.key_name + "_" + team.key_name,
event=event.key,
team=team.key,
year=event.year) for team in teams
] if not skip_eventteams else []
# Delete eventteams of teams that are no longer registered
if event_teams and not skip_eventteams:
existing_event_team_keys = set(
EventTeam.query(EventTeam.event == event.key).fetch(
1000, keys_only=True))
event_team_keys = set([et.key for et in event_teams])
et_keys_to_delete = existing_event_team_keys.difference(
event_team_keys)
EventTeamManipulator.delete_keys(et_keys_to_delete)
event_teams = EventTeamManipulator.createOrUpdate(event_teams)
if type(event_teams) is not list:
event_teams = [event_teams]
if event.year in {2018, 2019, 2020}:
avatars, keys_to_delete = df.getEventTeamAvatars(event.key_name)
if avatars:
MediaManipulator.createOrUpdate(avatars)
MediaManipulator.delete_keys(keys_to_delete)
template_values = {
'event': event,
'event_teams': event_teams,
}
if 'X-Appengine-Taskname' not in self.request.headers: # Only write out if not in taskqueue
path = os.path.join(
os.path.dirname(__file__),
'../templates/datafeeds/usfirst_event_details_get.html')
self.response.out.write(template.render(path, template_values))
def render_team_history(cls, handler, team, is_canonical):
event_team_keys_future = EventTeam.query(
EventTeam.team == team.key).fetch_async(1000, keys_only=True)
award_keys_future = Award.query(
Award.team_list == team.key).fetch_async(1000, keys_only=True)
event_teams_futures = ndb.get_multi_async(
event_team_keys_future.get_result())
awards_futures = ndb.get_multi_async(award_keys_future.get_result())
event_keys = [
event_team_future.get_result().event
for event_team_future in event_teams_futures
]
events_futures = ndb.get_multi_async(event_keys)
awards_by_event = {}
for award_future in awards_futures:
award = award_future.get_result()
if award.event.id() not in awards_by_event:
awards_by_event[award.event.id()] = [award]
else:
awards_by_event[award.event.id()].append(award)
event_awards = []
current_event = None
matches_upcoming = None
short_cache = False
for event_future in events_futures:
event = event_future.get_result()
if event.now:
current_event = event
team_matches_future = Match.query(Match.event == event.key, Match.team_key_names == team.key_name)\
.fetch_async(500, keys_only=True)
matches = ndb.get_multi(team_matches_future.get_result())
matches_upcoming = MatchHelper.upcomingMatches(matches)
if event.within_a_day:
short_cache = True
if event.key_name in awards_by_event:
sorted_awards = AwardHelper.organizeAwards(
awards_by_event[event.key_name])
else:
sorted_awards = []
event_awards.append((event, sorted_awards))
event_awards = sorted(
event_awards,
key=lambda (e, _): e.start_date
if e.start_date else datetime.datetime(e.year, 12, 31))
years = sorted(
set([
et.get_result().year for et in event_teams_futures
if et.get_result().year is not None
]))
handler.template_values.update({
'is_canonical': is_canonical,
'team': team,
'event_awards': event_awards,
'years': years,
'current_event': current_event,
'matches_upcoming': matches_upcoming
})
if short_cache:
handler._cache_expiration = handler.SHORT_CACHE_EXPIRATION
path = os.path.join(os.path.dirname(__file__),
'../templates/team_history.html')
return template.render(path, handler.template_values)
def getPopularTeamsEvents(self, events):
events_by_key = {}
for event in events:
events_by_key[event.key.id()] = event
# Calculate popular teams
# Get cached team keys
event_team_keys = memcache.get_multi(events_by_key.keys(),
namespace='event-team-keys')
# Get uncached team keys
to_query = set(events_by_key.keys()).difference(event_team_keys.keys())
event_teams_futures = [
(event_key,
EventTeam.query(
EventTeam.event == ndb.Key(Event, event_key)).fetch_async(
projection=[EventTeam.team])) for event_key in to_query
]
# Merge cached and uncached team keys
for event_key, event_teams in event_teams_futures:
event_team_keys[event_key] = [
et.team.id() for et in event_teams.get_result()
]
memcache.set_multi(event_team_keys,
60 * 60 * 24,
namespace='event-team-keys')
team_keys = []
team_events = {}
for event_key, event_team_keys in event_team_keys.items():
team_keys += event_team_keys
for team_key in event_team_keys:
team_events[team_key] = events_by_key[event_key]
# Get cached counts
team_favorite_counts = memcache.get_multi(
team_keys, namespace='team-favorite-counts')
# Get uncached counts
to_count = set(team_keys).difference(team_favorite_counts.keys())
count_futures = [
(team_key,
Favorite.query(Favorite.model_key == team_key).count_async())
for team_key in to_count
]
# Merge cached and uncached counts
for team_key, count_future in count_futures:
team_favorite_counts[team_key] = count_future.get_result()
memcache.set_multi(team_favorite_counts,
60 * 60 * 24,
namespace='team-favorite-counts')
# Sort to get top popular teams
popular_team_keys = []
for team_key, _ in sorted(team_favorite_counts.items(),
key=lambda tc: -tc[1])[:25]:
popular_team_keys.append(ndb.Key(Team, team_key))
popular_teams = sorted(ndb.get_multi(popular_team_keys),
key=lambda team: team.team_number)
popular_teams_events = []
for team in popular_teams:
popular_teams_events.append((team, team_events[team.key.id()]))
return popular_teams_events
def _query_async(self):
team_key = self._query_args[0]
event_teams = yield EventTeam.query(EventTeam.team == ndb.Key(Team, team_key)).fetch_async()
event_keys = map(lambda event_team: event_team.event, event_teams)
events = yield ndb.get_multi_async(event_keys)
raise ndb.Return(events)
def _render(self, district_abbrev, year=None, explicit_year=False):
district_type = DistrictType.abbrevs[district_abbrev]
event_keys = Event.query(Event.year == year, Event.event_district_enum == district_type).fetch(None, keys_only=True)
if not event_keys:
self.abort(404)
# needed for district teams
district_key = '{}{}'.format(year, district_abbrev)
district_teams_future = DistrictTeamsQuery(district_key).fetch_async()
# needed for valid_years
all_cmp_event_keys_future = Event.query(Event.event_district_enum == district_type, Event.event_type_enum == EventType.DISTRICT_CMP).fetch_async(None, keys_only=True)
# needed for valid_districts
district_cmp_keys_future = Event.query(Event.year == year, Event.event_type_enum == EventType.DISTRICT_CMP).fetch_async(None, keys_only=True) # to compute valid_districts
# Needed for active team statuses
live_events = EventHelper.getWeekEvents()
live_eventteams_futures = []
for event in live_events:
live_eventteams_futures.append(EventTeamsQuery(event.key_name).fetch_async())
event_futures = ndb.get_multi_async(event_keys)
event_team_keys_future = EventTeam.query(EventTeam.event.IN(event_keys)).fetch_async(None, keys_only=True)
team_futures = ndb.get_multi_async(set([ndb.Key(Team, et_key.id().split('_')[1]) for et_key in event_team_keys_future.get_result()]))
events = [event_future.get_result() for event_future in event_futures]
EventHelper.sort_events(events)
district_cmp_futures = ndb.get_multi_async(district_cmp_keys_future.get_result())
team_totals = DistrictHelper.calculate_rankings(events, team_futures, year)
valid_districts = set()
for district_cmp_future in district_cmp_futures:
district_cmp = district_cmp_future.get_result()
cmp_dis_type = district_cmp.event_district_enum
if cmp_dis_type is None:
logging.warning("District event {} has unknown district type!".format(district_cmp.key.id()))
else:
valid_districts.add((DistrictType.type_names[cmp_dis_type], DistrictType.type_abbrevs[cmp_dis_type]))
valid_districts = sorted(valid_districts, key=lambda (name, _): name)
teams = TeamHelper.sortTeams(district_teams_future.get_result())
num_teams = len(teams)
middle_value = num_teams / 2
if num_teams % 2 != 0:
middle_value += 1
teams_a, teams_b = teams[:middle_value], teams[middle_value:]
# Currently Competing Team Status
live_events_with_teams = EventTeamStatusHelper.buildEventTeamStatus(live_events, live_eventteams_futures, teams)
live_events_with_teams.sort(key=lambda x: x[0].name)
self.template_values.update({
'explicit_year': explicit_year,
'year': year,
'valid_years': sorted(set([int(event_key.id()[:4]) for event_key in all_cmp_event_keys_future.get_result()])),
'valid_districts': valid_districts,
'district_name': DistrictType.type_names[district_type],
'district_abbrev': district_abbrev,
'events': events,
'team_totals': team_totals,
'teams_a': teams_a,
'teams_b': teams_b,
'live_events_with_teams': live_events_with_teams,
})
path = os.path.join(os.path.dirname(__file__), '../templates/district_details.html')
return template.render(path, self.template_values)
def _render(self):
year, location, range_limit, search_type, page = self._get_params()
num_results = 0
results = []
distances = []
if location:
lat_lon, _ = LocationHelper.get_lat_lon(location, geocode=True)
if lat_lon:
lat, lon = lat_lon
dist_expr = 'distance(location, geopoint({}, {}))'.format(
lat, lon)
if search_type == 'teams':
query_string = '{} < {}'.format(
dist_expr, range_limit * self.METERS_PER_MILE)
else:
query_string = '{} < {} AND year={}'.format(
dist_expr, range_limit * self.METERS_PER_MILE, year)
offset = self.PAGE_SIZE * page
query = search.Query(
query_string=query_string,
options=search.QueryOptions(
limit=self.PAGE_SIZE,
offset=offset,
sort_options=search.SortOptions(expressions=[
search.SortExpression(
expression=dist_expr,
direction=search.SortExpression.ASCENDING)
]),
returned_expressions=[
search.FieldExpression(name='distance',
expression=dist_expr)
],
))
if search_type == 'teams':
search_index = search.Index(name="teamLocation")
else:
search_index = search.Index(name="eventLocation")
docs = search_index.search(query)
num_results = docs.number_found
distances = {}
keys = []
event_team_count_futures = {}
for result in docs.results:
distances[result.doc_id] = result.expressions[
0].value / self.METERS_PER_MILE
if search_type == 'teams':
event_team_count_futures[
result.doc_id] = EventTeam.query(
EventTeam.team == ndb.Key(
'Team', result.doc_id),
EventTeam.year == year).count_async(
limit=1, keys_only=True)
keys.append(ndb.Key('Team', result.doc_id))
else:
keys.append(ndb.Key('Event', result.doc_id))
result_futures = ndb.get_multi_async(keys)
if search_type == 'teams':
results = []
for result_future, team_key in zip(result_futures, keys):
if event_team_count_futures[
team_key.id()].get_result() != 0:
results.append(result_future.get_result())
else:
results = [
result_future.get_result()
for result_future in result_futures
]
self.template_values.update({
'valid_years': self.VALID_YEARS,
'valid_ranges': self.VALID_RANGES,
'page_size': self.PAGE_SIZE,
'page': page,
'year': year,
'location': location,
'range_limit': range_limit,
'search_type': search_type,
'num_results': num_results,
'results': results,
'distances': distances,
})
return jinja2_engine.render('nearby.html', self.template_values)
def get(self, year, first_eid):
df = DatafeedUsfirst()
df_legacy = DatafeedUsfirstLegacy()
event = df.getEventDetails(first_eid)
if not event:
logging.warning(
"getEventDetails with DatafeedUsfirst for event id {} failed. Retrying with DatafeedUsfirstLegacy."
.format(first_eid))
event = df_legacy.getEventDetails(int(year), first_eid)
if self.request.get('event_district_enum'):
event.event_district_enum = int(
self.request.get('event_district_enum'))
event = EventManipulator.createOrUpdate(event)
teams = df.getEventTeams(int(year), first_eid)
if not teams:
logging.warning(
"getEventTeams with DatafeedUsfirst for event id {} failed. Retrying with DatafeedUsfirstLegacy."
.format(first_eid))
teams = df_legacy.getEventTeams(int(year), first_eid)
if not teams:
logging.warning(
"getEventTeams with DatafeedUsfirstLegacy for event id {} failed."
.format(first_eid))
teams = []
teams = TeamManipulator.createOrUpdate(teams)
if teams:
if type(teams) is not list:
teams = [teams]
event_teams = [
EventTeam(id=event.key.id() + "_" + team.key.id(),
event=event.key,
team=team.key,
year=event.year) for team in teams
]
# Delete eventteams of teams that unregister from an event
if event.future:
existing_event_team_keys = set(
EventTeam.query(EventTeam.event == event.key).fetch(
1000, keys_only=True))
event_team_keys = set([et.key for et in event_teams])
et_keys_to_delete = existing_event_team_keys.difference(
event_team_keys)
EventTeamManipulator.delete_keys(et_keys_to_delete)
event_teams = EventTeamManipulator.createOrUpdate(event_teams)
if type(event_teams) is not list:
event_teams = [event_teams]
else:
event_teams = []
template_values = {
'event': event,
'event_teams': event_teams,
}
path = os.path.join(
os.path.dirname(__file__),
'../templates/datafeeds/usfirst_event_details_get.html')
self.response.out.write(template.render(path, template_values))
def _query_async(self):
event_key = self._query_args[0]
event_teams = yield EventTeam.query(
EventTeam.event == ndb.Key(Event, event_key)).fetch_async()
raise ndb.Return(event_teams)
def update(self, event_key):
"""
Updates EventTeams for an event.
Returns a tuple of (teams, event_teams, event_team_keys_to_delete)
An EventTeam is valid iff the team:
a) played a match at the event,
b) the team received an award at the event,
c) the event has not yet occurred,
d) or the event is not from the current year. (This is to make sure we don't delete old data we may no longer be able to scrape)
"""
event = Event.get_by_id(event_key)
cur_year = datetime.datetime.now().year
# Add teams from Matches and Awards
team_ids = set()
match_key_futures = Match.query(
Match.event == event.key).fetch_async(1000, keys_only=True)
award_key_futures = Award.query(
Award.event == event.key).fetch_async(1000, keys_only=True)
match_futures = ndb.get_multi_async(match_key_futures.get_result())
award_futures = ndb.get_multi_async(award_key_futures.get_result())
for match_future in match_futures:
match = match_future.get_result()
for team in match.team_key_names:
team_ids.add(team)
for award_future in award_futures:
award = award_future.get_result()
for team_key in award.team_list:
team_ids.add(team_key.id())
# Create or update EventTeams
teams = [Team(id=team_id,
team_number=int(team_id[3:]))
for team_id in team_ids]
if teams:
event_teams = [EventTeam(id=event_key + "_" + team.key.id(),
event=event.key,
team=team.key,
year=event.year)
for team in teams]
else:
event_teams = None
# Delete EventTeams for teams who did not participate in the event
# Only runs if event is over
existing_event_teams_keys = EventTeam.query(
EventTeam.event == event.key).fetch(1000, keys_only=True)
existing_event_teams = ndb.get_multi(existing_event_teams_keys)
existing_team_ids = set()
for et in existing_event_teams:
existing_team_ids.add(et.team.id())
et_keys_to_delete = set()
if event.year == cur_year and event.end_date is not None and event.end_date < datetime.datetime.now():
for team_id in existing_team_ids.difference(team_ids):
et_key_name = "{}_{}".format(event.key_name, team_id)
et_keys_to_delete.add(ndb.Key(EventTeam, et_key_name))
return teams, event_teams, et_keys_to_delete
