def test_district_updated(self): affected_refs = { 'key': {ndb.Key(District, '2016ne')}, 'year': {2015, 2016}, 'abbreviation': {'ne', 'chs'} } cache_keys = [ q.cache_key for q in get_affected_queries.district_updated(affected_refs) ] self.assertEqual(len(cache_keys), 11) self.assertTrue(DistrictsInYearQuery(2015).cache_key in cache_keys) self.assertTrue(DistrictsInYearQuery(2016).cache_key in cache_keys) self.assertTrue(DistrictHistoryQuery('ne').cache_key in cache_keys) self.assertTrue(DistrictHistoryQuery('chs').cache_key in cache_keys) self.assertTrue(DistrictQuery('2016ne').cache_key in cache_keys) self.assertTrue(TeamDistrictsQuery('frc604').cache_key in cache_keys) # Necessary because APIv3 Event models include the District model self.assertTrue(EventQuery('2016necmp').cache_key in cache_keys) self.assertTrue(EventListQuery(2016).cache_key in cache_keys) self.assertTrue(DistrictEventsQuery('2016ne').cache_key in cache_keys) self.assertTrue(TeamEventsQuery('frc125').cache_key in cache_keys) self.assertTrue( TeamYearEventsQuery('frc125', 2016).cache_key in cache_keys)
def test_district_updated(self): affected_refs = {'year': {2015, 2016}, 'abbreviation': {'ne', 'chs'}} cache_keys = [ q.cache_key for q in get_affected_queries.district_updated(affected_refs) ] self.assertEqual(len(cache_keys), 4) self.assertTrue(DistrictsInYearQuery(2015).cache_key in cache_keys) self.assertTrue(DistrictsInYearQuery(2016).cache_key in cache_keys) self.assertTrue(DistrictHistoryQuery('ne').cache_key in cache_keys) self.assertTrue(DistrictHistoryQuery('chs').cache_key in cache_keys)
def get(self, year): districts = DistrictsInYearQuery(int(year)).fetch() district_keys = [district.key.id() for district in districts] for district_key in district_keys: taskqueue.add(url='/tasks/math/do/district_rankings_calc/{}'.format(district_key), method='GET') self.response.out.write("Enqueued for: {}".format(district_keys))
def _render(self, year): district_list, self._last_modified = DistrictsInYearQuery( int(year)).fetch(dict_version=3, return_updated=True) return json.dumps(district_list, ensure_ascii=True, indent=True, sort_keys=True)
def district_updated(affected_refs): years = _filter(affected_refs['year']) district_abbrevs = _filter(affected_refs['abbreviation']) district_keys = _filter(affected_refs['key']) district_team_keys_future = DistrictTeam.query(DistrictTeam.district_key.IN(list(district_keys))).fetch_async(None, keys_only=True) district_event_keys_future = Event.query(Event.district_key.IN(list(district_keys))).fetch_async(keys_only=True) queries_and_keys = [] for year in years: queries_and_keys.append(DistrictsInYearQuery(year)) for abbrev in district_abbrevs: queries_and_keys.append(DistrictHistoryQuery(abbrev)) for key in district_keys: queries_and_keys.append(DistrictQuery(key.id())) for dt_key in district_team_keys_future.get_result(): team_key = dt_key.id().split('_')[1] queries_and_keys.append(TeamDistrictsQuery(team_key)) # Necessary because APIv3 Event models include the District model affected_event_refs = { 'key': set(), 'year': set(), 'district_key': district_keys, } for event_key in district_event_keys_future.get_result(): affected_event_refs['key'].add(event_key) affected_event_refs['year'].add(int(event_key.id()[:4])) queries_and_keys += event_updated(affected_event_refs) return queries_and_keys
def _render(self, year=None, explicit_year=False): state_prov = self.request.get('state_prov', None) districts_future = DistrictsInYearQuery(year).fetch_async() all_events_future = event_query.EventListQuery( year).fetch_async() # Needed for state_prov if state_prov: events_future = Event.query( Event.year == year, Event.state_prov == state_prov).fetch_async() else: events_future = all_events_future events = events_future.get_result() if state_prov == '' or (state_prov and not events): self.redirect(self.request.path, abort=True) EventHelper.sort_events(events) week_events = EventHelper.groupByWeek(events) districts = [] # a tuple of (district abbrev, district name) for district in districts_future.get_result(): districts.append((district.abbreviation, district.display_name)) districts = sorted(districts, key=lambda d: d[1]) valid_state_provs = set() for event in all_events_future.get_result(): if event.state_prov: valid_state_provs.add(event.state_prov) valid_state_provs = sorted(valid_state_provs) self.template_values.update({ "events": events, "explicit_year": explicit_year, "selected_year": year, "valid_years": self.VALID_YEARS, "week_events": week_events, "districts": districts, "state_prov": state_prov, "valid_state_provs": valid_state_provs, }) if year == datetime.datetime.now().year: self._cache_expiration = self.SHORT_CACHE_EXPIRATION return jinja2_engine.render('event_list.html', self.template_values)
def _render(self, year=None, explicit_year=False): state_prov = self.request.get('state_prov', None) districts_future = DistrictsInYearQuery(year).fetch_async() all_events_future = event_query.EventListQuery(year).fetch_async() # Needed for state_prov if state_prov: events_future = Event.query(Event.year==year, Event.state_prov==state_prov).fetch_async() else: events_future = all_events_future events = events_future.get_result() if state_prov == '' or (state_prov and not events): self.redirect(self.request.path, abort=True) EventHelper.sort_events(events) week_events = EventHelper.groupByWeek(events) districts = [] # a tuple of (district abbrev, district name) for district in districts_future.get_result(): districts.append((district.abbreviation, district.display_name)) districts = sorted(districts, key=lambda d: d[1]) valid_state_provs = set() for event in all_events_future.get_result(): if event.state_prov: valid_state_provs.add(event.state_prov) valid_state_provs = sorted(valid_state_provs) self.template_values.update({ "events": events, "explicit_year": explicit_year, "selected_year": year, "valid_years": self.VALID_YEARS, "week_events": week_events, "districts": districts, "state_prov": state_prov, "valid_state_provs": valid_state_provs, }) if year == datetime.datetime.now().year: self._cache_expiration = self.SHORT_CACHE_EXPIRATION return jinja2_engine.render('event_list.html', self.template_values)
def _render(self, year=None): all_districts = DistrictsInYearQuery(self.year).fetch() districts = list() for district in all_districts: dictionary = dict() dictionary["key"] = district.abbreviation dictionary["name"] = district.display_name districts.append(dictionary) return json.dumps(districts, ensure_ascii=True)
def district_updated(affected_refs): years = filter(None, affected_refs['year']) district_abbrevs = filter(None, affected_refs['abbreviation']) queries_and_keys = [] for year in years: queries_and_keys.append(DistrictsInYearQuery(year)) for abbrev in district_abbrevs: queries_and_keys.append(DistrictHistoryQuery(abbrev)) return queries_and_keys
def _render(self, district_abbrev, year=None, explicit_year=False): district = DistrictQuery('{}{}'.format(year, district_abbrev)).fetch() if not district: self.abort(404) events_future = DistrictEventsQuery(district.key_name).fetch_async() # needed for district teams district_teams_future = DistrictTeamsQuery(district.key_name).fetch_async() # needed for valid_years history_future = DistrictHistoryQuery(district.abbreviation).fetch_async() # needed for valid_districts districts_in_year_future = DistrictsInYearQuery(district.year).fetch_async() # needed for active team statuses live_events = [] if year == datetime.datetime.now().year: # Only show active teams for current year live_events = EventHelper.getWeekEvents() live_eventteams_futures = [] for event in live_events: live_eventteams_futures.append(EventTeamsQuery(event.key_name).fetch_async()) events = events_future.get_result() EventHelper.sort_events(events) events_by_key = {} for event in events: events_by_key[event.key.id()] = event week_events = EventHelper.groupByWeek(events) valid_districts = set() districts_in_year = districts_in_year_future.get_result() for dist in districts_in_year: valid_districts.add((dist.display_name, dist.abbreviation)) valid_districts = sorted(valid_districts, key=lambda (name, _): name) teams = TeamHelper.sortTeams(district_teams_future.get_result()) team_keys = set([t.key.id() for t in teams]) num_teams = len(teams) middle_value = num_teams / 2 if num_teams % 2 != 0: middle_value += 1 teams_a, teams_b = teams[:middle_value], teams[middle_value:] # Currently Competing Team Status event_team_keys = [] for event, teams_future in zip(live_events, live_eventteams_futures): for team in teams_future.get_result(): if team.key.id() in team_keys: event_team_keys.append(ndb.Key(EventTeam, '{}_{}'.format(event.key.id(), team.key.id()))) # Should be in context cache ndb.get_multi(event_team_keys) # Warms context cache live_events_with_teams = [] for event, teams_future in zip(live_events, live_eventteams_futures): teams_and_statuses = [] has_teams = False for team in teams_future.get_result(): if team.key.id() in team_keys: has_teams = True event_team = EventTeam.get_by_id('{}_{}'.format(event.key.id(), team.key.id())) # Should be in context cache status_str = { 'alliance': EventTeamStatusHelper.generate_team_at_event_alliance_status_string(team.key.id(), event_team.status), 'playoff': EventTeamStatusHelper.generate_team_at_event_playoff_status_string(team.key.id(), event_team.status), } teams_and_statuses.append(( team, event_team.status, status_str )) if has_teams: teams_and_statuses.sort(key=lambda x: x[0].team_number) live_events_with_teams.append((event, teams_and_statuses)) live_events_with_teams.sort(key=lambda x: x[0].name) live_events_with_teams.sort(key=lambda x: EventHelper.distantFutureIfNoStartDate(x[0])) live_events_with_teams.sort(key=lambda x: EventHelper.distantFutureIfNoEndDate(x[0])) # Get valid years district_history = history_future.get_result() valid_years = map(lambda d: d.year, district_history) valid_years = sorted(valid_years) self.template_values.update({ 'explicit_year': explicit_year, 'year': year, 'valid_years': valid_years, 'valid_districts': valid_districts, 'district_name': district.display_name, 'district_abbrev': district_abbrev, 'week_events': week_events, 'events_by_key': events_by_key, 'rankings': district.rankings, 'advancement': district.advancement, 'teams_a': teams_a, 'teams_b': teams_b, 'live_events_with_teams': live_events_with_teams, }) return jinja2_engine.render('district_details.html', self.template_values)
def parse(self, response): events = [] year_districts_future = DistrictsInYearQuery(self.season).fetch_async() for event in response['hits']['hits']: first_eid = event['_id'] event = event['_source'] event_type = EventHelper.parseEventType(event['event_subtype']) if event_type in self.TYPES_TO_SKIP: continue code = event['event_code'].lower() key = "{}{}".format(self.season, code) name = event['event_name'] short_name = EventHelper.getShortName(name) city = event.get('event_city', None) state_prov = event.get('event_stateprov', None) country = event.get('event_country', None) postalcode = event.get('event_postalcode', None) start = datetime.datetime.strptime(event['date_start'], self.DATE_FORMAT_STR) end = datetime.datetime.strptime( event['date_end'], self.DATE_FORMAT_STR) + datetime.timedelta( hours=23, minutes=59, seconds=59) venue_address = event['event_venue'] if 'event_address1' in event and event['event_address1']: venue_address += '\n' + event['event_address1'] if 'event_address2' in event and event['event_address2']: venue_address += '\n' + event['event_address2'] venue_address += '\n{}, {} {}\n{}'.format( event['event_city'], event['event_stateprov'], event['event_postalcode'], event['event_country']) raw_website = event.get('event_web_url', None) website = urlparse.urlparse( raw_website, 'http').geturl() if raw_website else None # Decide what district (if any) this event is in if event_type in EventType.DISTRICT_EVENT_TYPES: district_enum = EventHelper.getDistrictEnumFromEventName(name) district_key = EventHelper.getDistrictKeyFromEventName( name, year_districts_future) else: district_enum = DistrictType.NO_DISTRICT district_key = None events.append( Event(id=key, name=name, short_name=short_name, event_short=code, event_type_enum=event_type, official=True, start_date=start, end_date=end, venue=event['event_venue'], city=city, state_prov=state_prov, country=country, postalcode=postalcode, venue_address=venue_address, year=self.season, event_district_enum=district_enum, district_key=district_key, first_eid=first_eid, website=website)) return events
def _render(self, district_abbrev, year=None, explicit_year=False): district = DistrictQuery('{}{}'.format(year, district_abbrev)).fetch() if not district: self.abort(404) events_future = DistrictEventsQuery(district.key_name).fetch_async() # needed for district teams district_teams_future = DistrictTeamsQuery(district.key_name).fetch_async() # needed for valid_years history_future = DistrictHistoryQuery(district.abbreviation).fetch_async() # needed for valid_districts districts_in_year_future = DistrictsInYearQuery(district.year).fetch_async() # Temp disabled on 2017-02-18 -fangeugene # # Needed for active team statuses # live_events = [] # if year == datetime.datetime.now().year: # Only show active teams for current year # live_events = EventHelper.getWeekEvents() # live_eventteams_futures = [] # for event in live_events: # live_eventteams_futures.append(EventTeamsQuery(event.key_name).fetch_async()) events = events_future.get_result() EventHelper.sort_events(events) events_by_key = {} for event in events: events_by_key[event.key.id()] = event week_events = EventHelper.groupByWeek(events) valid_districts = set() districts_in_year = districts_in_year_future.get_result() for dist in districts_in_year: valid_districts.add((dist.display_name, dist.abbreviation)) valid_districts = sorted(valid_districts, key=lambda (name, _): name) teams = TeamHelper.sortTeams(district_teams_future.get_result()) num_teams = len(teams) middle_value = num_teams / 2 if num_teams % 2 != 0: middle_value += 1 teams_a, teams_b = teams[:middle_value], teams[middle_value:] # Temp disabled on 2017-02-18 -fangeugene # # Currently Competing Team Status # live_events_with_teams = EventTeamStatusHelper.buildEventTeamStatus(live_events, live_eventteams_futures, teams) # live_events_with_teams.sort(key=lambda x: x[0].name) # Get valid years district_history = history_future.get_result() valid_years = map(lambda d: d.year, district_history) valid_years = sorted(valid_years) self.template_values.update({ 'explicit_year': explicit_year, 'year': year, 'valid_years': valid_years, 'valid_districts': valid_districts, 'district_name': district.display_name, 'district_abbrev': district_abbrev, 'week_events': week_events, 'events_by_key': events_by_key, 'rankings': district.rankings, 'teams_a': teams_a, 'teams_b': teams_b, # 'live_events_with_teams': live_events_with_teams, # Temp disabled on 2017-02-18 -fangeugene }) return jinja2_engine.render('district_details.html', self.template_values)
def _render(self, district_abbrev, year=None, explicit_year=False): district = DistrictQuery('{}{}'.format(year, district_abbrev)).fetch() if not district: self.abort(404) events_future = DistrictEventsQuery(district.key_name).fetch_async() # needed for district teams district_teams_future = DistrictTeamsQuery( district.key_name).fetch_async() # needed for valid_years history_future = DistrictHistoryQuery( district.abbreviation).fetch_async() # needed for valid_districts districts_in_year_future = DistrictsInYearQuery( district.year).fetch_async() # needed for active team statuses live_events = [] if year == datetime.datetime.now( ).year: # Only show active teams for current year live_events = EventHelper.getWeekEvents() live_eventteams_futures = [] for event in live_events: live_eventteams_futures.append( EventTeamsQuery(event.key_name).fetch_async()) events = events_future.get_result() EventHelper.sort_events(events) events_by_key = {} for event in events: events_by_key[event.key.id()] = event week_events = EventHelper.groupByWeek(events) valid_districts = set() districts_in_year = districts_in_year_future.get_result() for dist in districts_in_year: valid_districts.add((dist.display_name, dist.abbreviation)) valid_districts = sorted(valid_districts, key=lambda (name, _): name) teams = TeamHelper.sortTeams(district_teams_future.get_result()) team_keys = set([t.key.id() for t in teams]) num_teams = len(teams) middle_value = num_teams / 2 if num_teams % 2 != 0: middle_value += 1 teams_a, teams_b = teams[:middle_value], teams[middle_value:] # Currently Competing Team Status event_team_keys = [] for event, teams_future in zip(live_events, live_eventteams_futures): for team in teams_future.get_result(): if team.key.id() in team_keys: event_team_keys.append( ndb.Key(EventTeam, '{}_{}'.format( event.key.id(), team.key.id()))) # Should be in context cache ndb.get_multi(event_team_keys) # Warms context cache live_events_with_teams = [] for event, teams_future in zip(live_events, live_eventteams_futures): teams_and_statuses = [] has_teams = False for team in teams_future.get_result(): if team.key.id() in team_keys: has_teams = True event_team = EventTeam.get_by_id('{}_{}'.format( event.key.id(), team.key.id())) # Should be in context cache status_str = { 'alliance': EventTeamStatusHelper. generate_team_at_event_alliance_status_string( team.key.id(), event_team.status), 'playoff': EventTeamStatusHelper. generate_team_at_event_playoff_status_string( team.key.id(), event_team.status), } teams_and_statuses.append( (team, event_team.status, status_str)) if has_teams: teams_and_statuses.sort(key=lambda x: x[0].team_number) live_events_with_teams.append((event, teams_and_statuses)) live_events_with_teams.sort(key=lambda x: x[0].name) live_events_with_teams.sort( key=lambda x: EventHelper.distantFutureIfNoStartDate(x[0])) live_events_with_teams.sort( key=lambda x: EventHelper.distantFutureIfNoEndDate(x[0])) # Get valid years district_history = history_future.get_result() valid_years = map(lambda d: d.year, district_history) valid_years = sorted(valid_years) self.template_values.update({ 'explicit_year': explicit_year, 'year': year, 'valid_years': valid_years, 'valid_districts': valid_districts, 'district_name': district.display_name, 'district_abbrev': district_abbrev, 'week_events': week_events, 'events_by_key': events_by_key, 'rankings': district.rankings, 'advancement': district.advancement, 'teams_a': teams_a, 'teams_b': teams_b, 'live_events_with_teams': live_events_with_teams, }) return jinja2_engine.render('district_details.html', self.template_values)