def get(self, year): df = DatafeedFMSAPI('v2.0') fmsapi_events, event_list_districts = df.getEventList(year) events = EventManipulator.createOrUpdate(fmsapi_events) fmsapi_districts = df.getDistrictList(year) merged_districts = DistrictManipulator.mergeModels( fmsapi_districts, event_list_districts) districts = DistrictManipulator.createOrUpdate(merged_districts) # Fetch event details for each event for event in events: taskqueue.add(queue_name='datafeed', target='backend-tasks', url='/backend-tasks/get/event_details/' + event.key_name, method='GET') template_values = { "events": events, "districts": districts, } if 'X-Appengine-Taskname' not in self.request.headers: # Only write out if not in taskqueue path = os.path.join( os.path.dirname(__file__), '../templates/datafeeds/fms_event_list_get.html') self.response.out.write(template.render(path, template_values))
def get(self, year): year = int(year) year_dcmps = DistrictChampsInYearQuery(year).fetch() districts_to_write = [] for dcmp in year_dcmps: district_abbrev = DistrictType.type_abbrevs[ dcmp.event_district_enum] district_key = District.renderKeyName(year, district_abbrev) logging.info("Creating {}".format(district_key)) district = District( id=district_key, year=year, abbreviation=district_abbrev, display_name=DistrictType.type_names[dcmp.event_district_enum], elasticsearch_name=next( (k for k, v in DistrictType.elasticsearch_names.iteritems() if v == dcmp.event_district_enum), None)) districts_to_write.append(district) logging.info("Writing {} new districts".format( len(districts_to_write))) DistrictManipulator.createOrUpdate(districts_to_write, run_post_update_hook=False) for dcmp in year_dcmps: district_abbrev = DistrictType.type_abbrevs[ dcmp.event_district_enum] district_key = District.renderKeyName(year, district_abbrev) district_events_future = DistrictEventsQuery( district_key).fetch_async() district_events = district_events_future.get_result() logging.info("Found {} events to update".format( len(district_events))) events_to_write = [] for event in district_events: event.district_key = ndb.Key(District, district_key) events_to_write.append(event) EventManipulator.createOrUpdate(events_to_write) for dcmp in year_dcmps: district_abbrev = DistrictType.type_abbrevs[ dcmp.event_district_enum] district_key = District.renderKeyName(year, district_abbrev) districtteams_future = DistrictTeam.query( DistrictTeam.year == year, DistrictTeam.district == DistrictType.abbrevs.get( district_abbrev, None)).fetch_async() districtteams = districtteams_future.get_result() logging.info("Found {} DistrictTeams to update".format( len(districtteams))) districtteams_to_write = [] for districtteam in districtteams: districtteam.district_key = ndb.Key(District, district_key) districtteams_to_write.append(districtteam) DistrictTeamManipulator.createOrUpdate(districtteams_to_write)
def get(self, year): df = DatafeedFMSAPI('v2.0') fmsapi_events, event_list_districts = df.getEventList(year) events = EventManipulator.createOrUpdate(fmsapi_events) fmsapi_districts = df.getDistrictList(year) merged_districts = DistrictManipulator.mergeModels(fmsapi_districts, event_list_districts) districts = DistrictManipulator.createOrUpdate(merged_districts) # Fetch event details for each event for event in events: taskqueue.add( queue_name='datafeed', target='backend-tasks', url='/backend-tasks/get/event_details/'+event.key_name, method='GET' ) template_values = { "events": events, "districts": districts, } if 'X-Appengine-Taskname' not in self.request.headers: # Only write out if not in taskqueue path = os.path.join(os.path.dirname(__file__), '../templates/datafeeds/fms_event_list_get.html') self.response.out.write(template.render(path, template_values))
def get(self, year): df = DatafeedFMSAPI('v2.0') fmsapi_events, event_list_districts = df.getEventList(year) # All regular-season events can be inserted without any work involved. # We need to de-duplicate offseason events from the FRC Events API with a different code than the TBA event code fmsapi_events_offseason = [e for e in fmsapi_events if e.is_offseason] event_keys_to_put = set([e.key_name for e in fmsapi_events]) - set( [e.key_name for e in fmsapi_events_offseason]) events_to_put = [ e for e in fmsapi_events if e.key_name in event_keys_to_put ] matched_offseason_events, new_offseason_events = \ OffseasonEventHelper.categorize_offseasons(int(year), fmsapi_events_offseason) # For all matched offseason events, make sure the FIRST code matches the TBA FIRST code for tba_event, first_event in matched_offseason_events: tba_event.first_code = first_event.event_short events_to_put.append( tba_event) # Update TBA events - discard the FIRST event # For all new offseason events we can't automatically match, create suggestions SuggestionCreator.createDummyOffseasonSuggestions(new_offseason_events) events = EventManipulator.createOrUpdate(events_to_put) or [] fmsapi_districts = df.getDistrictList(year) merged_districts = DistrictManipulator.mergeModels( fmsapi_districts, event_list_districts) if merged_districts: districts = DistrictManipulator.createOrUpdate(merged_districts) else: districts = [] # Fetch event details for each event for event in events: taskqueue.add(queue_name='datafeed', target='backend-tasks', url='/backend-tasks/get/event_details/' + event.key_name, method='GET') template_values = { "events": events, "districts": districts, } if 'X-Appengine-Taskname' not in self.request.headers: # Only write out if not in taskqueue path = os.path.join( os.path.dirname(__file__), '../templates/datafeeds/fms_event_list_get.html') self.response.out.write(template.render(path, template_values))
def post(self, district_key): self._require_admin() district = District(id=District.renderKeyName( self.request.get("year"), self.request.get("abbreviation")), year=int(self.request.get("year")), abbreviation=self.request.get("abbreviation"), display_name=self.request.get("display_name")) DistrictManipulator.createOrUpdate(district) self.redirect('/admin/districts/' + self.request.get("year"))
def store_district(data): district = District(id=data['key']) district.year = data['year'] district.abbreviation = data['abbreviation'] district.display_name = data['display_name'] return DistrictManipulator.createOrUpdate(district)
def get(self, year): year = int(year) year_dcmps = DistrictChampsInYearQuery(year).fetch() districts_to_write = [] for dcmp in year_dcmps: district_abbrev = DistrictType.type_abbrevs[dcmp.event_district_enum] district_key = District.renderKeyName(year, district_abbrev) logging.info("Creating {}".format(district_key)) district = District( id=district_key, year=year, abbreviation=district_abbrev, display_name=DistrictType.type_names[dcmp.event_district_enum], elasticsearch_name=next((k for k, v in DistrictType.elasticsearch_names.iteritems() if v == dcmp.event_district_enum), None) ) districts_to_write.append(district) logging.info("Writing {} new districts".format(len(districts_to_write))) DistrictManipulator.createOrUpdate(districts_to_write, run_post_update_hook=False) for dcmp in year_dcmps: district_abbrev = DistrictType.type_abbrevs[dcmp.event_district_enum] district_key = District.renderKeyName(year, district_abbrev) district_events_future = DistrictEventsQuery(district_key).fetch_async() district_events = district_events_future.get_result() logging.info("Found {} events to update".format(len(district_events))) events_to_write = [] for event in district_events: event.district_key = ndb.Key(District, district_key) events_to_write.append(event) EventManipulator.createOrUpdate(events_to_write) for dcmp in year_dcmps: district_abbrev = DistrictType.type_abbrevs[dcmp.event_district_enum] district_key = District.renderKeyName(year, district_abbrev) districtteams_future = DistrictTeam.query(DistrictTeam.year == year, DistrictTeam.district == DistrictType.abbrevs.get(district_abbrev, None)).fetch_async() districtteams = districtteams_future.get_result() logging.info("Found {} DistrictTeams to update".format(len(districtteams))) districtteams_to_write = [] for districtteam in districtteams: districtteam.district_key = ndb.Key(District, district_key) districtteams_to_write.append(districtteam) DistrictTeamManipulator.createOrUpdate(districtteams_to_write)
def setUp(self): self.testbed = testbed.Testbed() self.testbed.activate() self.testbed.init_datastore_v3_stub() self.testbed.init_memcache_stub() self.testbed.init_taskqueue_stub(root_path=".") ndb.get_context().clear_cache() # Prevent data from leaking between tests # Create districts districts = [] for code in ['mar', 'isr', 'nc', 'ne', 'pnw', 'pch', 'chs', 'in', 'ont', 'fim', 'tx']: year = 2017 districts.append(District( id=District.renderKeyName(year, code), year=year, abbreviation=code, )) DistrictManipulator.createOrUpdate(districts)
def get(self, district_key): district = District.get_by_id(district_key) if not district: self.response.out.write( "District {} does not exist!".format(district_key)) return events_future = DistrictEventsQuery(district_key).fetch_async() teams_future = DistrictTeamsQuery(district_key).fetch_async() events = events_future.get_result() for event in events: event.prep_details() EventHelper.sort_events(events) team_totals = DistrictHelper.calculate_rankings( events, teams_future, district.year) rankings = [] current_rank = 1 for key, points in team_totals: point_detail = {} point_detail["rank"] = current_rank point_detail["team_key"] = key point_detail["event_points"] = [] for event, event_points in points["event_points"]: event_points['event_key'] = event.key.id() event_points['district_cmp'] = ( event.event_type_enum == EventType.DISTRICT_CMP or event.event_type_enum == EventType.DISTRICT_CMP_DIVISION) point_detail["event_points"].append(event_points) point_detail["rookie_bonus"] = points.get("rookie_bonus", 0) point_detail["point_total"] = points["point_total"] rankings.append(point_detail) current_rank += 1 if rankings: district.rankings = rankings DistrictManipulator.createOrUpdate(district) if 'X-Appengine-Taskname' not in self.request.headers: # Only write out if not in taskqueue self.response.out.write( "Finished calculating rankings for: {}".format(district_key))
def get(self, year): df = DatafeedFMSAPI('v2.0') fmsapi_districts = df.getDistrictList(year) districts = DistrictManipulator.createOrUpdate(fmsapi_districts) template_values = { "districts": districts, } if 'X-Appengine-Taskname' not in self.request.headers: # Only write out if not in taskqueue path = os.path.join(os.path.dirname(__file__), '../templates/datafeeds/fms_district_list_get.html') self.response.out.write(template.render(path, template_values))
def get(self, district_key): district = District.get_by_id(district_key) if not district: self.response.out.write("District {} does not exist!".format(district_key)) return events_future = DistrictEventsQuery(district_key).fetch_async() teams_future = DistrictTeamsQuery(district_key).fetch_async() events = events_future.get_result() for event in events: event.prep_details() EventHelper.sort_events(events) team_totals = DistrictHelper.calculate_rankings(events, teams_future, district.year) rankings = [] current_rank = 1 for key, points in team_totals: point_detail = {} point_detail["rank"] = current_rank point_detail["team_key"] = key point_detail["event_points"] = [] for event, event_points in points["event_points"]: event_points['event_key'] = event.key.id() event_points['district_cmp'] = ( event.event_type_enum == EventType.DISTRICT_CMP or event.event_type_enum == EventType.DISTRICT_CMP_DIVISION) point_detail["event_points"].append(event_points) point_detail["rookie_bonus"] = points.get("rookie_bonus", 0) point_detail["point_total"] = points["point_total"] rankings.append(point_detail) current_rank += 1 if rankings: district.rankings = rankings DistrictManipulator.createOrUpdate(district) if 'X-Appengine-Taskname' not in self.request.headers: # Only write out if not in taskqueue self.response.out.write("Finished calculating rankings for: {}".format(district_key))
def get(self, year): df = DatafeedFMSAPI('v2.0') fmsapi_districts = df.getDistrictList(year) districts = DistrictManipulator.createOrUpdate(fmsapi_districts) template_values = { "districts": districts, } if 'X-Appengine-Taskname' not in self.request.headers: # Only write out if not in taskqueue path = os.path.join( os.path.dirname(__file__), '../templates/datafeeds/fms_district_list_get.html') self.response.out.write(template.render(path, template_values))
def setUp(self): self.testbed = testbed.Testbed() self.testbed.activate() self.testbed.init_datastore_v3_stub() self.testbed.init_memcache_stub() self.testbed.init_taskqueue_stub(root_path=".") ndb.get_context().clear_cache( ) # Prevent data from leaking between tests # Create districts districts = [] for code in [ 'mar', 'isr', 'nc', 'ne', 'pnw', 'pch', 'chs', 'in', 'ont', 'fim', 'tx' ]: year = 2017 districts.append( District( id=District.renderKeyName(year, code), year=year, abbreviation=code, )) DistrictManipulator.createOrUpdate(districts)
def get(self, district_key): df = DatafeedFMSAPI('v2.0') district_with_rankings = df.getDistrictRankings(district_key) districts = [] if district_with_rankings: districts = DistrictManipulator.createOrUpdate(district_with_rankings) template_values = { "districts": [districts], } if 'X-Appengine-Taskname' not in self.request.headers: # Only write out if not in taskqueue path = os.path.join(os.path.dirname(__file__), '../templates/datafeeds/fms_district_list_get.html') self.response.out.write(template.render(path, template_values))
def get(self, district_key): df = DatafeedFMSAPI('v2.0') district_with_rankings = df.getDistrictRankings(district_key) districts = [] if district_with_rankings: districts = DistrictManipulator.createOrUpdate( district_with_rankings) template_values = { "districts": [districts], } if 'X-Appengine-Taskname' not in self.request.headers: # Only write out if not in taskqueue path = os.path.join( os.path.dirname(__file__), '../templates/datafeeds/fms_district_list_get.html') self.response.out.write(template.render(path, template_values))
def get(self, event_key): df = DatafeedFMSAPI('v2.0') df2 = DatafeedFIRSTElasticSearch() event = Event.get_by_id(event_key) # Update event fmsapi_events, fmsapi_districts = df.getEventDetails(event_key) elasticsearch_events = df2.getEventDetails(event) updated_event = EventManipulator.mergeModels(fmsapi_events, elasticsearch_events) if updated_event: event = EventManipulator.createOrUpdate(updated_event) DistrictManipulator.createOrUpdate(fmsapi_districts) models = df.getEventTeams(event_key) teams = [] district_teams = [] robots = [] for group in models: # models is a list of tuples (team, districtTeam, robot) if isinstance(group[0], Team): teams.append(group[0]) if isinstance(group[1], DistrictTeam): district_teams.append(group[1]) if isinstance(group[2], Robot): robots.append(group[2]) # Merge teams teams = TeamManipulator.mergeModels(teams, df2.getEventTeams(event)) # Write new models if teams and event.year == tba_config.MAX_YEAR: # Only update from latest year teams = TeamManipulator.createOrUpdate(teams) district_teams = DistrictTeamManipulator.createOrUpdate(district_teams) robots = RobotManipulator.createOrUpdate(robots) if not teams: # No teams found registered for this event teams = [] if type(teams) is not list: teams = [teams] # Build EventTeams cmp_hack_sitevar = Sitevar.get_or_insert('cmp_registration_hacks') events_without_eventteams = cmp_hack_sitevar.contents.get('skip_eventteams', []) \ if cmp_hack_sitevar else [] skip_eventteams = event_key in events_without_eventteams event_teams = [ EventTeam(id=event.key_name + "_" + team.key_name, event=event.key, team=team.key, year=event.year) for team in teams ] if not skip_eventteams else [] # Delete eventteams of teams that are no longer registered if event_teams and not skip_eventteams: existing_event_team_keys = set( EventTeam.query(EventTeam.event == event.key).fetch( 1000, keys_only=True)) event_team_keys = set([et.key for et in event_teams]) et_keys_to_delete = existing_event_team_keys.difference( event_team_keys) EventTeamManipulator.delete_keys(et_keys_to_delete) event_teams = EventTeamManipulator.createOrUpdate(event_teams) if type(event_teams) is not list: event_teams = [event_teams] if event.year in {2018, 2019, 2020}: avatars, keys_to_delete = df.getEventTeamAvatars(event.key_name) if avatars: MediaManipulator.createOrUpdate(avatars) MediaManipulator.delete_keys(keys_to_delete) template_values = { 'event': event, 'event_teams': event_teams, } if 'X-Appengine-Taskname' not in self.request.headers: # Only write out if not in taskqueue path = os.path.join( os.path.dirname(__file__), '../templates/datafeeds/usfirst_event_details_get.html') self.response.out.write(template.render(path, template_values))
def get(self, event_key): df = DatafeedFMSAPI('v2.0') df2 = DatafeedFIRSTElasticSearch() event = Event.get_by_id(event_key) # Update event fmsapi_events, fmsapi_districts = df.getEventDetails(event_key) elasticsearch_events = df2.getEventDetails(event) updated_event = EventManipulator.mergeModels( fmsapi_events, elasticsearch_events) if updated_event: event = EventManipulator.createOrUpdate(updated_event) DistrictManipulator.createOrUpdate(fmsapi_districts) models = df.getEventTeams(event_key) teams = [] district_teams = [] robots = [] for group in models: # models is a list of tuples (team, districtTeam, robot) if isinstance(group[0], Team): teams.append(group[0]) if isinstance(group[1], DistrictTeam): district_teams.append(group[1]) if isinstance(group[2], Robot): robots.append(group[2]) # Merge teams teams = TeamManipulator.mergeModels(teams, df2.getEventTeams(event)) # Write new models if teams: teams = TeamManipulator.createOrUpdate(teams) district_teams = DistrictTeamManipulator.createOrUpdate(district_teams) robots = RobotManipulator.createOrUpdate(robots) if not teams: # No teams found registered for this event teams = [] if type(teams) is not list: teams = [teams] # Build EventTeams cmp_hack_sitevar = Sitevar.get_or_insert('cmp_registration_hacks') events_without_eventteams = cmp_hack_sitevar.contents.get('skip_eventteams', []) \ if cmp_hack_sitevar else [] skip_eventteams = event_key in events_without_eventteams event_teams = [EventTeam( id=event.key_name + "_" + team.key_name, event=event.key, team=team.key, year=event.year) for team in teams] if not skip_eventteams else [] # Delete eventteams of teams that are no longer registered if event_teams != [] or skip_eventteams: existing_event_team_keys = set(EventTeam.query(EventTeam.event == event.key).fetch(1000, keys_only=True)) event_team_keys = set([et.key for et in event_teams]) et_keys_to_delete = existing_event_team_keys.difference(event_team_keys) EventTeamManipulator.delete_keys(et_keys_to_delete) event_teams = EventTeamManipulator.createOrUpdate(event_teams) if type(event_teams) is not list: event_teams = [event_teams] template_values = { 'event': event, 'event_teams': event_teams, } if 'X-Appengine-Taskname' not in self.request.headers: # Only write out if not in taskqueue path = os.path.join(os.path.dirname(__file__), '../templates/datafeeds/usfirst_event_details_get.html') self.response.out.write(template.render(path, template_values))