def test_eventteam_updated(self):
affected_refs = {
'event': {ndb.Key(Event, '2015casj'),
ndb.Key(Event, '2015cama')},
'team': {ndb.Key(Team, 'frc254'),
ndb.Key(Team, 'frc604')},
'year': {2014, 2015}
}
cache_keys = [
q.cache_key
for q in get_affected_queries.eventteam_updated(affected_refs)
]
self.assertEqual(len(cache_keys), 14)
self.assertTrue(TeamEventsQuery('frc254').cache_key in cache_keys)
self.assertTrue(TeamEventsQuery('frc604').cache_key in cache_keys)
self.assertTrue(
TeamParticipationQuery('frc254').cache_key in cache_keys)
self.assertTrue(
TeamParticipationQuery('frc604').cache_key in cache_keys)
self.assertTrue(
TeamYearEventsQuery('frc254', 2014).cache_key in cache_keys)
self.assertTrue(
TeamYearEventsQuery('frc254', 2015).cache_key in cache_keys)
self.assertTrue(
TeamYearEventsQuery('frc604', 2014).cache_key in cache_keys)
self.assertTrue(
TeamYearEventsQuery('frc604', 2015).cache_key in cache_keys)
self.assertTrue(TeamListYearQuery(2014, 0).cache_key in cache_keys)
self.assertTrue(TeamListYearQuery(2014, 1).cache_key in cache_keys)
self.assertTrue(TeamListYearQuery(2015, 0).cache_key in cache_keys)
self.assertTrue(TeamListYearQuery(2015, 1).cache_key in cache_keys)
self.assertTrue(EventTeamsQuery('2015casj').cache_key in cache_keys)
self.assertTrue(EventTeamsQuery('2015cama').cache_key in cache_keys)
def test_event_updated(self):
affected_refs = {
'key': {ndb.Key(Event, '2015casj'),
ndb.Key(Event, '2015cama')},
'year': {2014, 2015},
'district_key':
{ndb.Key(District, '2015fim'),
ndb.Key(District, '2014mar')}
}
cache_keys = [
q.cache_key
for q in get_affected_queries.event_updated(affected_refs)
]
self.assertEqual(len(cache_keys), 10)
self.assertTrue(EventQuery('2015casj').cache_key in cache_keys)
self.assertTrue(EventQuery('2015cama').cache_key in cache_keys)
self.assertTrue(EventListQuery(2014).cache_key in cache_keys)
self.assertTrue(EventListQuery(2015).cache_key in cache_keys)
self.assertTrue(DistrictEventsQuery('2015fim').cache_key in cache_keys)
self.assertTrue(DistrictEventsQuery('2014mar').cache_key in cache_keys)
self.assertTrue(TeamEventsQuery('frc254').cache_key in cache_keys)
self.assertTrue(TeamEventsQuery('frc604').cache_key in cache_keys)
self.assertTrue(
TeamYearEventsQuery('frc254', 2015).cache_key in cache_keys)
self.assertTrue(
TeamYearEventsQuery('frc604', 2015).cache_key in cache_keys)
def event_updated(affected_refs):
event_keys = _filter(affected_refs['key'])
years = _filter(affected_refs['year'])
event_district_keys = _filter(affected_refs['district_key'])
event_team_keys_future = EventTeam.query(EventTeam.event.IN([event_key for event_key in event_keys])).fetch_async(None, keys_only=True)
events_future = ndb.get_multi_async(event_keys)
queries_and_keys = []
for event_key in event_keys:
queries_and_keys.append((EventQuery(event_key.id())))
queries_and_keys.append(EventDivisionsQuery(event_key.id()))
for year in years:
queries_and_keys.append((EventListQuery(year)))
for event_district_key in event_district_keys:
queries_and_keys.append((DistrictEventsQuery(event_district_key.id())))
for et_key in event_team_keys_future.get_result():
team_key = et_key.id().split('_')[1]
year = int(et_key.id()[:4])
queries_and_keys.append((TeamEventsQuery(team_key)))
queries_and_keys.append((TeamYearEventsQuery(team_key, year)))
events_with_parents = filter(lambda e: e.get_result() is not None and e.get_result().parent_event is not None, events_future)
parent_keys = set([e.get_result().parent_event for e in events_with_parents])
for parent_key in parent_keys:
queries_and_keys.append((EventDivisionsQuery(parent_key.id())))
return queries_and_keys
def _render(self, team_key):
self._set_team(team_key)
events = TeamEventsQuery(self.team_key).fetch()
event_list = [ModelToDict.eventConverter(event) for event in events]
return json.dumps(event_list, ensure_ascii=True)
def test_district_updated(self):
affected_refs = {
'key': {ndb.Key(District, '2016ne')},
'year': {2015, 2016},
'abbreviation': {'ne', 'chs'}
}
cache_keys = [
q.cache_key
for q in get_affected_queries.district_updated(affected_refs)
]
self.assertEqual(len(cache_keys), 11)
self.assertTrue(DistrictsInYearQuery(2015).cache_key in cache_keys)
self.assertTrue(DistrictsInYearQuery(2016).cache_key in cache_keys)
self.assertTrue(DistrictHistoryQuery('ne').cache_key in cache_keys)
self.assertTrue(DistrictHistoryQuery('chs').cache_key in cache_keys)
self.assertTrue(DistrictQuery('2016ne').cache_key in cache_keys)
self.assertTrue(TeamDistrictsQuery('frc604').cache_key in cache_keys)
# Necessary because APIv3 Event models include the District model
self.assertTrue(EventQuery('2016necmp').cache_key in cache_keys)
self.assertTrue(EventListQuery(2016).cache_key in cache_keys)
self.assertTrue(DistrictEventsQuery('2016ne').cache_key in cache_keys)
self.assertTrue(TeamEventsQuery('frc125').cache_key in cache_keys)
self.assertTrue(
TeamYearEventsQuery('frc125', 2016).cache_key in cache_keys)
def _render(self, team_key, year=None, model_type=None):
if year:
events, self._last_modified = TeamYearEventsQuery(team_key, int(year)).fetch(dict_version=3, return_updated=True)
else:
events, self._last_modified = TeamEventsQuery(team_key).fetch(dict_version=3, return_updated=True)
if model_type is not None:
events = filter_event_properties(events, model_type)
return json.dumps(events, ensure_ascii=True, indent=2, sort_keys=True)
def validate_team(team_number):
"""
Returns:
Team object if the team exists and is currently competing
String with the appropriate error otherwise
"""
team_key = 'frc{}'.format(team_number)
team = Team.get_by_id(team_key)
if not team:
return "Team {} does not exist.".format(team_number)
team_events_future = TeamEventsQuery(team_key).fetch_async()
current_event = None
for event in team_events_future.get_result():
if event.now:
current_event = event
if not current_event:
return "Team {} is not currently competing.".format(team_number)
return team, current_event
def validate_team(user_str, team_number):
"""
Returns:
Team object if the team exists and is currently competing
String with the appropriate error otherwise
"""
team_key = 'frc{}'.format(team_number)
team = Team.get_by_id(team_key)
if not team:
return "{}Team {} does not exist.".format(user_str, team_number)
team_events_future = TeamEventsQuery(team_key).fetch_async()
current_event = None
for event in team_events_future.get_result():
if event.now:
current_event = event
if not current_event:
return "{}Team {} is not currently competing.".format(user_str, team_number)
return team, current_event
def eventteam_updated(affected_refs):
event_keys = filter(None, affected_refs['event'])
team_keys = filter(None, affected_refs['team'])
years = filter(None, affected_refs['year'])
queries_and_keys = []
for team_key in team_keys:
queries_and_keys.append(TeamEventsQuery(team_key.id()))
queries_and_keys.append(TeamParticipationQuery(team_key.id()))
page_num = _get_team_page_num(team_key.id())
for year in years:
queries_and_keys.append(TeamYearEventsQuery(team_key.id(), year))
queries_and_keys.append(TeamListYearQuery(year, page_num))
for event_key in event_keys:
queries_and_keys.append(EventTeamsQuery(event_key.id()))
return queries_and_keys
def event_updated(affected_refs):
event_keys = filter(None, affected_refs['key'])
years = filter(None, affected_refs['year'])
event_district_keys = filter(None, affected_refs['event_district_key'])
event_team_keys_future = EventTeam.query(EventTeam.event.IN([event_key for event_key in event_keys])).fetch_async(None, keys_only=True)
queries_and_keys = []
for year in years:
queries_and_keys.append((EventListQuery(year)))
for event_district_key in event_district_keys:
queries_and_keys.append((DistrictEventsQuery(event_district_key)))
for et_key in event_team_keys_future.get_result():
team_key = et_key.id().split('_')[1]
year = int(et_key.id()[:4])
queries_and_keys.append((TeamEventsQuery(team_key)))
queries_and_keys.append((TeamYearEventsQuery(team_key, year)))
return queries_and_keys
def update_team_awards_index(cls, team):
awards_future = TeamAwardsQuery(team.key.id()).fetch_async()
events_future = TeamEventsQuery(team.key.id()).fetch_async()
medias_future = TeamMediaQuery(team.key.id()).fetch_async()
events_by_year = defaultdict(list)
for event in events_future.get_result():
events_by_year[event.year].append(event)
event.prep_details() # For rankings
awards_by_event = defaultdict(list)
for award in awards_future.get_result():
awards_by_event[award.event.id()].append(award)
medias_by_year = defaultdict(list)
for media in medias_future.get_result():
medias_by_year[media.year].append(media)
# General stuff that's the same for indexes
partial_fields = [
search.NumberField(name='number', value=team.team_number),
search.TextField(name='name', value=team.name),
search.TextField(name='nickname', value=team.nickname)
]
# field_counts = defaultdict(int)
for year, events in events_by_year.items():
year_matches_future = TeamYearMatchesQuery(team.key.id(),
year).fetch_async()
qual_seeds = defaultdict(int)
comp_levels = defaultdict(int)
year_awards = set()
for event in events:
if event.event_type_enum not in EventType.SEASON_EVENT_TYPES:
continue
if event.rankings:
for row in event.rankings:
if str(row[1]) == str(team.team_number):
seed = int(row[0])
for s in xrange(seed, 9):
qual_seeds[s] += 1
break
awards = awards_by_event.get(event.key.id(), [])
award_types = set([a.award_type_enum for a in awards])
for award_type in award_types:
if award_type in AwardType.SEARCHABLE:
year_awards.add(award_type)
if award_type == AwardType.WINNER:
comp_levels['win'] += 1
event_levels = defaultdict(set)
for match in year_matches_future.get_result():
if match.comp_level in {
'sf', 'f'
} and match.comp_level not in event_levels[match.event.id()]:
comp_levels[match.comp_level] += 1
event_levels[match.event.id()].add(match.comp_level)
has_cad = False
for media in medias_by_year[year]:
if media.media_type_enum in MediaType.robot_types:
has_cad = True
break
fields = partial_fields + [
search.AtomField(name='award', value=str(award))
for award in year_awards
] + [search.NumberField(name='year', value=year)]
if comp_levels:
for level, count in comp_levels.items():
fields += [
search.NumberField(name='comp_level_{}'.format(level),
value=count)
]
if qual_seeds:
for seed, count in qual_seeds.items():
fields += [
search.NumberField(name='seed_{}'.format(seed),
value=count)
]
if has_cad:
fields += [search.NumberField(name='has_cad', value=1)]
search.Index(name=cls.TEAM_AWARDS_INDEX).put(
search.Document(doc_id='{}_{}'.format(team.key.id(), year),
fields=fields))
def update_team_year_index(cls, team):
awards_future = TeamAwardsQuery(team.key.id()).fetch_async()
events_future = TeamEventsQuery(team.key.id()).fetch_async()
events_by_year = defaultdict(list)
for event in events_future.get_result():
events_by_year[event.year].append(event)
awards_by_year = defaultdict(list)
for award in awards_future.get_result():
awards_by_year[award.year].append(award)
# General stuff that's the same for all years
same_fields = [
search.NumberField(name='number', value=team.team_number),
search.TextField(name='name', value=team.name),
search.TextField(name='nickname', value=team.nickname)
]
if team.normalized_location and team.normalized_location.lat_lng:
same_fields += [
search.GeoField(name='location', value=search.GeoPoint(
team.normalized_location.lat_lng.lat,
team.normalized_location.lat_lng.lon))
]
# Construct overall and year specific fields
overall_fields = same_fields + [search.NumberField(name='year', value=0)]
overall_event_types = set()
overall_event_award_types = set()
overall_award_types = set()
overall_bb_count = 0
overall_divwin_count = 0
overall_cmpwin_count = 0
for year, events in events_by_year.items():
year_fields = same_fields + [search.NumberField(name='year', value=year)]
# Events
year_event_types = set()
for event in events:
# Allow searching by event type
if event.event_type_enum not in overall_event_types:
overall_fields += [search.AtomField(name='event_type', value=str(event.event_type_enum))]
overall_event_types.add(event.event_type_enum)
if event.event_type_enum not in year_event_types:
year_fields += [search.AtomField(name='event_type', value=str(event.event_type_enum))]
year_event_types.add(event.event_type_enum)
# Awards
year_event_award_types = set()
year_award_types = set()
year_bb_count = 0
year_divwin_count = 0
year_cmpwin_count = 0
for award in awards_by_year.get(year, []):
if award.event_type_enum not in EventType.SEASON_EVENT_TYPES:
continue
# Allow searching by award type and event type
ea_type = '{}_{}'.format(award.event_type_enum, award.award_type_enum)
if ea_type not in overall_event_award_types:
overall_fields += [search.AtomField(name='event_award_type', value=ea_type)]
overall_event_award_types.add(ea_type)
if ea_type not in year_event_award_types:
year_fields += [search.AtomField(name='event_award_type', value=ea_type)]
year_event_award_types.add(ea_type)
# Allow searching by award type
if award.award_type_enum not in overall_award_types:
overall_fields += [search.AtomField(name='award_type', value=str(award.award_type_enum))]
overall_award_types.add(award.award_type_enum)
if award.award_type_enum not in year_award_types:
year_fields += [search.AtomField(name='award_type', value=str(award.award_type_enum))]
year_award_types.add(award.award_type_enum)
# Allow searching/sorting by blue banners
if award.award_type_enum in AwardType.BLUE_BANNER_AWARDS:
overall_bb_count += 1
year_bb_count += 1
# Allow searching/sorting by div/cmp winner
if award.award_type_enum == AwardType.WINNER:
if award.event_type_enum == EventType.CMP_DIVISION:
overall_divwin_count += 1
year_divwin_count += 1
elif award.event_type_enum == EventType.CMP_FINALS:
overall_cmpwin_count += 1
year_cmpwin_count += 1
year_fields += [
search.NumberField(name='bb_count', value=year_bb_count),
search.NumberField(name='divwin_count', value=year_divwin_count),
search.NumberField(name='cmpwin_count', value=year_cmpwin_count),
]
# Put year index
search.Index(name=cls.TEAM_YEAR_INDEX).put(
search.Document(doc_id='{}_{}'.format(team.key.id(), year), fields=year_fields))
overall_fields += [
search.NumberField(name='bb_count', value=overall_bb_count),
search.NumberField(name='divwin_count', value=overall_divwin_count),
search.NumberField(name='cmpwin_count', value=overall_cmpwin_count),
]
# Put overall index
search.Index(name=cls.TEAM_YEAR_INDEX).put(
search.Document(doc_id='{}_0'.format(team.key.id()), fields=overall_fields))
def update_team_awards_index(cls, team):
awards_future = TeamAwardsQuery(team.key.id()).fetch_async()
events_future = TeamEventsQuery(team.key.id()).fetch_async()
medias_future = TeamMediaQuery(team.key.id()).fetch_async()
events_by_year = defaultdict(list)
for event in events_future.get_result():
events_by_year[event.year].append(event)
event.prep_details() # For rankings
awards_by_event = defaultdict(list)
for award in awards_future.get_result():
awards_by_event[award.event.id()].append(award)
medias_by_year = defaultdict(list)
for media in medias_future.get_result():
medias_by_year[media.year].append(media)
# General stuff that's the same for indexes
partial_fields = [
search.NumberField(name='number', value=team.team_number),
search.TextField(name='name', value=team.name),
search.TextField(name='nickname', value=team.nickname)
]
# field_counts = defaultdict(int)
for year, events in events_by_year.items():
year_matches_future = TeamYearMatchesQuery(team.key.id(), year).fetch_async()
qual_seeds = defaultdict(int)
comp_levels = defaultdict(int)
year_awards = set()
for event in events:
if event.event_type_enum not in EventType.SEASON_EVENT_TYPES:
continue
if event.rankings:
for row in event.rankings:
if str(row[1]) == str(team.team_number):
seed = int(row[0])
for s in xrange(seed, 9):
qual_seeds[s] += 1
break
awards = awards_by_event.get(event.key.id(), [])
award_types = set([a.award_type_enum for a in awards])
for award_type in award_types:
if award_type in AwardType.SEARCHABLE:
year_awards.add(award_type)
if award_type == AwardType.WINNER:
comp_levels['win'] += 1
event_levels = defaultdict(set)
for match in year_matches_future.get_result():
if match.comp_level in {'sf', 'f'} and match.comp_level not in event_levels[match.event.id()]:
comp_levels[match.comp_level] += 1
event_levels[match.event.id()].add(match.comp_level)
has_cad = False
for media in medias_by_year[year]:
if media.media_type_enum in MediaType.robot_types:
has_cad = True
break
fields = partial_fields + [
search.AtomField(name='award', value=str(award)) for award in year_awards
] + [search.NumberField(name='year', value=year)]
if comp_levels:
for level, count in comp_levels.items():
fields += [search.NumberField(name='comp_level_{}'.format(level), value=count)]
if qual_seeds:
for seed, count in qual_seeds.items():
fields += [search.NumberField(name='seed_{}'.format(seed), value=count)]
if has_cad:
fields += [
search.NumberField(name='has_cad', value=1)
]
search.Index(name=cls.TEAM_AWARDS_INDEX).put(
search.Document(doc_id='{}_{}'.format(team.key.id(), year), fields=fields))
def update_team_awards_index(cls, team):
awards_future = TeamAwardsQuery(team.key.id()).fetch_async()
events_future = TeamEventsQuery(team.key.id()).fetch_async()
medias_future = TeamMediaQuery(team.key.id()).fetch_async()
events_by_year = defaultdict(list)
for event in events_future.get_result():
events_by_year[event.year].append(event)
awards_by_event = defaultdict(list)
for award in awards_future.get_result():
awards_by_event[award.event.id()].append(award)
# General stuff that's the same for indexes
fields = [
search.NumberField(name='number', value=team.team_number),
search.TextField(name='name', value=team.name),
search.TextField(name='nickname', value=team.nickname)
]
field_counts = defaultdict(int)
for year, events in events_by_year.items():
season_awards = set()
season_awards_event = defaultdict(set)
for event in events:
if event.event_type_enum not in EventType.SEASON_EVENT_TYPES:
continue
awards = awards_by_event.get(event.key.id(), [])
award_types = set([a.award_type_enum for a in awards])
award_types = filter(lambda a: a in AwardType.SEARCHABLE, award_types)
# To search by event
for award_set in cls._construct_powerset(award_types):
set_name = cls._construct_set_name(award_set)
field_counts['e_{}'.format(set_name)] += 1
field_counts['e_{}_y{}'.format(set_name, event.year)] += 1
field_counts['e_{}_e{}'.format(set_name, event.event_type_enum)] += 1
field_counts['e_{}_e{}_y{}'.format(set_name, event.event_type_enum, event.year)] += 1
season_awards = season_awards.union(award_types)
season_awards_event[event.event_type_enum] = season_awards_event[event.event_type_enum].union(award_types)
# To search by year
for award_set in cls._construct_powerset(season_awards):
set_name = cls._construct_set_name(award_set)
field_counts['s_{}'.format(set_name)] += 1
field_counts['s_{}_y{}'.format(set_name, year)] += 1
for event_type, awards in season_awards_event.items():
for award_set in cls._construct_powerset(awards):
set_name = cls._construct_set_name(award_set)
field_counts['s_{}_e{}'.format(set_name, event_type)] += 1
field_counts['s_{}_e{}_y{}'.format(set_name, event_type, year)] += 1
fields += [search.NumberField(name=field, value=count) for field, count in field_counts.items()]
# Medias
robot_photo_years = set()
cad_model_years = set()
for media in medias_future.get_result():
if media.media_type_enum in MediaType.image_types and team.key in media.preferred_references:
robot_photo_years.add(media.year)
elif media.media_type_enum in MediaType.robot_types:
cad_model_years.add(media.year)
fields += [search.AtomField(name='robot_photo_year', value=str(year)) for year in robot_photo_years]
fields += [search.AtomField(name='cad_model_year', value=str(year)) for year in cad_model_years]
search.Index(name=cls.TEAM_AWARDS_INDEX).put(
search.Document(doc_id='{}'.format(team.key.id()), fields=fields))