def test_eventteam_updated(self): affected_refs = { 'event': {ndb.Key(Event, '2015casj'), ndb.Key(Event, '2015cama')}, 'team': {ndb.Key(Team, 'frc254'), ndb.Key(Team, 'frc604')}, 'year': {2014, 2015} } cache_keys = [ q.cache_key for q in get_affected_queries.eventteam_updated(affected_refs) ] self.assertEqual(len(cache_keys), 14) self.assertTrue(TeamEventsQuery('frc254').cache_key in cache_keys) self.assertTrue(TeamEventsQuery('frc604').cache_key in cache_keys) self.assertTrue( TeamParticipationQuery('frc254').cache_key in cache_keys) self.assertTrue( TeamParticipationQuery('frc604').cache_key in cache_keys) self.assertTrue( TeamYearEventsQuery('frc254', 2014).cache_key in cache_keys) self.assertTrue( TeamYearEventsQuery('frc254', 2015).cache_key in cache_keys) self.assertTrue( TeamYearEventsQuery('frc604', 2014).cache_key in cache_keys) self.assertTrue( TeamYearEventsQuery('frc604', 2015).cache_key in cache_keys) self.assertTrue(TeamListYearQuery(2014, 0).cache_key in cache_keys) self.assertTrue(TeamListYearQuery(2014, 1).cache_key in cache_keys) self.assertTrue(TeamListYearQuery(2015, 0).cache_key in cache_keys) self.assertTrue(TeamListYearQuery(2015, 1).cache_key in cache_keys) self.assertTrue(EventTeamsQuery('2015casj').cache_key in cache_keys) self.assertTrue(EventTeamsQuery('2015cama').cache_key in cache_keys)
def test_event_updated(self): affected_refs = { 'key': {ndb.Key(Event, '2015casj'), ndb.Key(Event, '2015cama')}, 'year': {2014, 2015}, 'district_key': {ndb.Key(District, '2015fim'), ndb.Key(District, '2014mar')} } cache_keys = [ q.cache_key for q in get_affected_queries.event_updated(affected_refs) ] self.assertEqual(len(cache_keys), 10) self.assertTrue(EventQuery('2015casj').cache_key in cache_keys) self.assertTrue(EventQuery('2015cama').cache_key in cache_keys) self.assertTrue(EventListQuery(2014).cache_key in cache_keys) self.assertTrue(EventListQuery(2015).cache_key in cache_keys) self.assertTrue(DistrictEventsQuery('2015fim').cache_key in cache_keys) self.assertTrue(DistrictEventsQuery('2014mar').cache_key in cache_keys) self.assertTrue(TeamEventsQuery('frc254').cache_key in cache_keys) self.assertTrue(TeamEventsQuery('frc604').cache_key in cache_keys) self.assertTrue( TeamYearEventsQuery('frc254', 2015).cache_key in cache_keys) self.assertTrue( TeamYearEventsQuery('frc604', 2015).cache_key in cache_keys)
def event_updated(affected_refs): event_keys = _filter(affected_refs['key']) years = _filter(affected_refs['year']) event_district_keys = _filter(affected_refs['district_key']) event_team_keys_future = EventTeam.query(EventTeam.event.IN([event_key for event_key in event_keys])).fetch_async(None, keys_only=True) events_future = ndb.get_multi_async(event_keys) queries_and_keys = [] for event_key in event_keys: queries_and_keys.append((EventQuery(event_key.id()))) queries_and_keys.append(EventDivisionsQuery(event_key.id())) for year in years: queries_and_keys.append((EventListQuery(year))) for event_district_key in event_district_keys: queries_and_keys.append((DistrictEventsQuery(event_district_key.id()))) for et_key in event_team_keys_future.get_result(): team_key = et_key.id().split('_')[1] year = int(et_key.id()[:4]) queries_and_keys.append((TeamEventsQuery(team_key))) queries_and_keys.append((TeamYearEventsQuery(team_key, year))) events_with_parents = filter(lambda e: e.get_result() is not None and e.get_result().parent_event is not None, events_future) parent_keys = set([e.get_result().parent_event for e in events_with_parents]) for parent_key in parent_keys: queries_and_keys.append((EventDivisionsQuery(parent_key.id()))) return queries_and_keys
def _render(self, team_key): self._set_team(team_key) events = TeamEventsQuery(self.team_key).fetch() event_list = [ModelToDict.eventConverter(event) for event in events] return json.dumps(event_list, ensure_ascii=True)
def test_district_updated(self): affected_refs = { 'key': {ndb.Key(District, '2016ne')}, 'year': {2015, 2016}, 'abbreviation': {'ne', 'chs'} } cache_keys = [ q.cache_key for q in get_affected_queries.district_updated(affected_refs) ] self.assertEqual(len(cache_keys), 11) self.assertTrue(DistrictsInYearQuery(2015).cache_key in cache_keys) self.assertTrue(DistrictsInYearQuery(2016).cache_key in cache_keys) self.assertTrue(DistrictHistoryQuery('ne').cache_key in cache_keys) self.assertTrue(DistrictHistoryQuery('chs').cache_key in cache_keys) self.assertTrue(DistrictQuery('2016ne').cache_key in cache_keys) self.assertTrue(TeamDistrictsQuery('frc604').cache_key in cache_keys) # Necessary because APIv3 Event models include the District model self.assertTrue(EventQuery('2016necmp').cache_key in cache_keys) self.assertTrue(EventListQuery(2016).cache_key in cache_keys) self.assertTrue(DistrictEventsQuery('2016ne').cache_key in cache_keys) self.assertTrue(TeamEventsQuery('frc125').cache_key in cache_keys) self.assertTrue( TeamYearEventsQuery('frc125', 2016).cache_key in cache_keys)
def _render(self, team_key, year=None, model_type=None): if year: events, self._last_modified = TeamYearEventsQuery(team_key, int(year)).fetch(dict_version=3, return_updated=True) else: events, self._last_modified = TeamEventsQuery(team_key).fetch(dict_version=3, return_updated=True) if model_type is not None: events = filter_event_properties(events, model_type) return json.dumps(events, ensure_ascii=True, indent=2, sort_keys=True)
def validate_team(team_number): """ Returns: Team object if the team exists and is currently competing String with the appropriate error otherwise """ team_key = 'frc{}'.format(team_number) team = Team.get_by_id(team_key) if not team: return "Team {} does not exist.".format(team_number) team_events_future = TeamEventsQuery(team_key).fetch_async() current_event = None for event in team_events_future.get_result(): if event.now: current_event = event if not current_event: return "Team {} is not currently competing.".format(team_number) return team, current_event
def validate_team(user_str, team_number): """ Returns: Team object if the team exists and is currently competing String with the appropriate error otherwise """ team_key = 'frc{}'.format(team_number) team = Team.get_by_id(team_key) if not team: return "{}Team {} does not exist.".format(user_str, team_number) team_events_future = TeamEventsQuery(team_key).fetch_async() current_event = None for event in team_events_future.get_result(): if event.now: current_event = event if not current_event: return "{}Team {} is not currently competing.".format(user_str, team_number) return team, current_event
def eventteam_updated(affected_refs): event_keys = filter(None, affected_refs['event']) team_keys = filter(None, affected_refs['team']) years = filter(None, affected_refs['year']) queries_and_keys = [] for team_key in team_keys: queries_and_keys.append(TeamEventsQuery(team_key.id())) queries_and_keys.append(TeamParticipationQuery(team_key.id())) page_num = _get_team_page_num(team_key.id()) for year in years: queries_and_keys.append(TeamYearEventsQuery(team_key.id(), year)) queries_and_keys.append(TeamListYearQuery(year, page_num)) for event_key in event_keys: queries_and_keys.append(EventTeamsQuery(event_key.id())) return queries_and_keys
def event_updated(affected_refs): event_keys = filter(None, affected_refs['key']) years = filter(None, affected_refs['year']) event_district_keys = filter(None, affected_refs['event_district_key']) event_team_keys_future = EventTeam.query(EventTeam.event.IN([event_key for event_key in event_keys])).fetch_async(None, keys_only=True) queries_and_keys = [] for year in years: queries_and_keys.append((EventListQuery(year))) for event_district_key in event_district_keys: queries_and_keys.append((DistrictEventsQuery(event_district_key))) for et_key in event_team_keys_future.get_result(): team_key = et_key.id().split('_')[1] year = int(et_key.id()[:4]) queries_and_keys.append((TeamEventsQuery(team_key))) queries_and_keys.append((TeamYearEventsQuery(team_key, year))) return queries_and_keys
def update_team_awards_index(cls, team): awards_future = TeamAwardsQuery(team.key.id()).fetch_async() events_future = TeamEventsQuery(team.key.id()).fetch_async() medias_future = TeamMediaQuery(team.key.id()).fetch_async() events_by_year = defaultdict(list) for event in events_future.get_result(): events_by_year[event.year].append(event) event.prep_details() # For rankings awards_by_event = defaultdict(list) for award in awards_future.get_result(): awards_by_event[award.event.id()].append(award) medias_by_year = defaultdict(list) for media in medias_future.get_result(): medias_by_year[media.year].append(media) # General stuff that's the same for indexes partial_fields = [ search.NumberField(name='number', value=team.team_number), search.TextField(name='name', value=team.name), search.TextField(name='nickname', value=team.nickname) ] # field_counts = defaultdict(int) for year, events in events_by_year.items(): year_matches_future = TeamYearMatchesQuery(team.key.id(), year).fetch_async() qual_seeds = defaultdict(int) comp_levels = defaultdict(int) year_awards = set() for event in events: if event.event_type_enum not in EventType.SEASON_EVENT_TYPES: continue if event.rankings: for row in event.rankings: if str(row[1]) == str(team.team_number): seed = int(row[0]) for s in xrange(seed, 9): qual_seeds[s] += 1 break awards = awards_by_event.get(event.key.id(), []) award_types = set([a.award_type_enum for a in awards]) for award_type in award_types: if award_type in AwardType.SEARCHABLE: year_awards.add(award_type) if award_type == AwardType.WINNER: comp_levels['win'] += 1 event_levels = defaultdict(set) for match in year_matches_future.get_result(): if match.comp_level in { 'sf', 'f' } and match.comp_level not in event_levels[match.event.id()]: comp_levels[match.comp_level] += 1 event_levels[match.event.id()].add(match.comp_level) has_cad = False for media in medias_by_year[year]: if media.media_type_enum in MediaType.robot_types: has_cad = True break fields = partial_fields + [ search.AtomField(name='award', value=str(award)) for award in year_awards ] + [search.NumberField(name='year', value=year)] if comp_levels: for level, count in comp_levels.items(): fields += [ search.NumberField(name='comp_level_{}'.format(level), value=count) ] if qual_seeds: for seed, count in qual_seeds.items(): fields += [ search.NumberField(name='seed_{}'.format(seed), value=count) ] if has_cad: fields += [search.NumberField(name='has_cad', value=1)] search.Index(name=cls.TEAM_AWARDS_INDEX).put( search.Document(doc_id='{}_{}'.format(team.key.id(), year), fields=fields))
def update_team_year_index(cls, team): awards_future = TeamAwardsQuery(team.key.id()).fetch_async() events_future = TeamEventsQuery(team.key.id()).fetch_async() events_by_year = defaultdict(list) for event in events_future.get_result(): events_by_year[event.year].append(event) awards_by_year = defaultdict(list) for award in awards_future.get_result(): awards_by_year[award.year].append(award) # General stuff that's the same for all years same_fields = [ search.NumberField(name='number', value=team.team_number), search.TextField(name='name', value=team.name), search.TextField(name='nickname', value=team.nickname) ] if team.normalized_location and team.normalized_location.lat_lng: same_fields += [ search.GeoField(name='location', value=search.GeoPoint( team.normalized_location.lat_lng.lat, team.normalized_location.lat_lng.lon)) ] # Construct overall and year specific fields overall_fields = same_fields + [search.NumberField(name='year', value=0)] overall_event_types = set() overall_event_award_types = set() overall_award_types = set() overall_bb_count = 0 overall_divwin_count = 0 overall_cmpwin_count = 0 for year, events in events_by_year.items(): year_fields = same_fields + [search.NumberField(name='year', value=year)] # Events year_event_types = set() for event in events: # Allow searching by event type if event.event_type_enum not in overall_event_types: overall_fields += [search.AtomField(name='event_type', value=str(event.event_type_enum))] overall_event_types.add(event.event_type_enum) if event.event_type_enum not in year_event_types: year_fields += [search.AtomField(name='event_type', value=str(event.event_type_enum))] year_event_types.add(event.event_type_enum) # Awards year_event_award_types = set() year_award_types = set() year_bb_count = 0 year_divwin_count = 0 year_cmpwin_count = 0 for award in awards_by_year.get(year, []): if award.event_type_enum not in EventType.SEASON_EVENT_TYPES: continue # Allow searching by award type and event type ea_type = '{}_{}'.format(award.event_type_enum, award.award_type_enum) if ea_type not in overall_event_award_types: overall_fields += [search.AtomField(name='event_award_type', value=ea_type)] overall_event_award_types.add(ea_type) if ea_type not in year_event_award_types: year_fields += [search.AtomField(name='event_award_type', value=ea_type)] year_event_award_types.add(ea_type) # Allow searching by award type if award.award_type_enum not in overall_award_types: overall_fields += [search.AtomField(name='award_type', value=str(award.award_type_enum))] overall_award_types.add(award.award_type_enum) if award.award_type_enum not in year_award_types: year_fields += [search.AtomField(name='award_type', value=str(award.award_type_enum))] year_award_types.add(award.award_type_enum) # Allow searching/sorting by blue banners if award.award_type_enum in AwardType.BLUE_BANNER_AWARDS: overall_bb_count += 1 year_bb_count += 1 # Allow searching/sorting by div/cmp winner if award.award_type_enum == AwardType.WINNER: if award.event_type_enum == EventType.CMP_DIVISION: overall_divwin_count += 1 year_divwin_count += 1 elif award.event_type_enum == EventType.CMP_FINALS: overall_cmpwin_count += 1 year_cmpwin_count += 1 year_fields += [ search.NumberField(name='bb_count', value=year_bb_count), search.NumberField(name='divwin_count', value=year_divwin_count), search.NumberField(name='cmpwin_count', value=year_cmpwin_count), ] # Put year index search.Index(name=cls.TEAM_YEAR_INDEX).put( search.Document(doc_id='{}_{}'.format(team.key.id(), year), fields=year_fields)) overall_fields += [ search.NumberField(name='bb_count', value=overall_bb_count), search.NumberField(name='divwin_count', value=overall_divwin_count), search.NumberField(name='cmpwin_count', value=overall_cmpwin_count), ] # Put overall index search.Index(name=cls.TEAM_YEAR_INDEX).put( search.Document(doc_id='{}_0'.format(team.key.id()), fields=overall_fields))
def update_team_awards_index(cls, team): awards_future = TeamAwardsQuery(team.key.id()).fetch_async() events_future = TeamEventsQuery(team.key.id()).fetch_async() medias_future = TeamMediaQuery(team.key.id()).fetch_async() events_by_year = defaultdict(list) for event in events_future.get_result(): events_by_year[event.year].append(event) event.prep_details() # For rankings awards_by_event = defaultdict(list) for award in awards_future.get_result(): awards_by_event[award.event.id()].append(award) medias_by_year = defaultdict(list) for media in medias_future.get_result(): medias_by_year[media.year].append(media) # General stuff that's the same for indexes partial_fields = [ search.NumberField(name='number', value=team.team_number), search.TextField(name='name', value=team.name), search.TextField(name='nickname', value=team.nickname) ] # field_counts = defaultdict(int) for year, events in events_by_year.items(): year_matches_future = TeamYearMatchesQuery(team.key.id(), year).fetch_async() qual_seeds = defaultdict(int) comp_levels = defaultdict(int) year_awards = set() for event in events: if event.event_type_enum not in EventType.SEASON_EVENT_TYPES: continue if event.rankings: for row in event.rankings: if str(row[1]) == str(team.team_number): seed = int(row[0]) for s in xrange(seed, 9): qual_seeds[s] += 1 break awards = awards_by_event.get(event.key.id(), []) award_types = set([a.award_type_enum for a in awards]) for award_type in award_types: if award_type in AwardType.SEARCHABLE: year_awards.add(award_type) if award_type == AwardType.WINNER: comp_levels['win'] += 1 event_levels = defaultdict(set) for match in year_matches_future.get_result(): if match.comp_level in {'sf', 'f'} and match.comp_level not in event_levels[match.event.id()]: comp_levels[match.comp_level] += 1 event_levels[match.event.id()].add(match.comp_level) has_cad = False for media in medias_by_year[year]: if media.media_type_enum in MediaType.robot_types: has_cad = True break fields = partial_fields + [ search.AtomField(name='award', value=str(award)) for award in year_awards ] + [search.NumberField(name='year', value=year)] if comp_levels: for level, count in comp_levels.items(): fields += [search.NumberField(name='comp_level_{}'.format(level), value=count)] if qual_seeds: for seed, count in qual_seeds.items(): fields += [search.NumberField(name='seed_{}'.format(seed), value=count)] if has_cad: fields += [ search.NumberField(name='has_cad', value=1) ] search.Index(name=cls.TEAM_AWARDS_INDEX).put( search.Document(doc_id='{}_{}'.format(team.key.id(), year), fields=fields))
def update_team_awards_index(cls, team): awards_future = TeamAwardsQuery(team.key.id()).fetch_async() events_future = TeamEventsQuery(team.key.id()).fetch_async() medias_future = TeamMediaQuery(team.key.id()).fetch_async() events_by_year = defaultdict(list) for event in events_future.get_result(): events_by_year[event.year].append(event) awards_by_event = defaultdict(list) for award in awards_future.get_result(): awards_by_event[award.event.id()].append(award) # General stuff that's the same for indexes fields = [ search.NumberField(name='number', value=team.team_number), search.TextField(name='name', value=team.name), search.TextField(name='nickname', value=team.nickname) ] field_counts = defaultdict(int) for year, events in events_by_year.items(): season_awards = set() season_awards_event = defaultdict(set) for event in events: if event.event_type_enum not in EventType.SEASON_EVENT_TYPES: continue awards = awards_by_event.get(event.key.id(), []) award_types = set([a.award_type_enum for a in awards]) award_types = filter(lambda a: a in AwardType.SEARCHABLE, award_types) # To search by event for award_set in cls._construct_powerset(award_types): set_name = cls._construct_set_name(award_set) field_counts['e_{}'.format(set_name)] += 1 field_counts['e_{}_y{}'.format(set_name, event.year)] += 1 field_counts['e_{}_e{}'.format(set_name, event.event_type_enum)] += 1 field_counts['e_{}_e{}_y{}'.format(set_name, event.event_type_enum, event.year)] += 1 season_awards = season_awards.union(award_types) season_awards_event[event.event_type_enum] = season_awards_event[event.event_type_enum].union(award_types) # To search by year for award_set in cls._construct_powerset(season_awards): set_name = cls._construct_set_name(award_set) field_counts['s_{}'.format(set_name)] += 1 field_counts['s_{}_y{}'.format(set_name, year)] += 1 for event_type, awards in season_awards_event.items(): for award_set in cls._construct_powerset(awards): set_name = cls._construct_set_name(award_set) field_counts['s_{}_e{}'.format(set_name, event_type)] += 1 field_counts['s_{}_e{}_y{}'.format(set_name, event_type, year)] += 1 fields += [search.NumberField(name=field, value=count) for field, count in field_counts.items()] # Medias robot_photo_years = set() cad_model_years = set() for media in medias_future.get_result(): if media.media_type_enum in MediaType.image_types and team.key in media.preferred_references: robot_photo_years.add(media.year) elif media.media_type_enum in MediaType.robot_types: cad_model_years.add(media.year) fields += [search.AtomField(name='robot_photo_year', value=str(year)) for year in robot_photo_years] fields += [search.AtomField(name='cad_model_year', value=str(year)) for year in cad_model_years] search.Index(name=cls.TEAM_AWARDS_INDEX).put( search.Document(doc_id='{}'.format(team.key.id()), fields=fields))