def get(self, year): year = int(year) year_dcmps = DistrictChampsInYearQuery(year).fetch() districts_to_write = [] for dcmp in year_dcmps: district_abbrev = DistrictType.type_abbrevs[ dcmp.event_district_enum] district_key = District.renderKeyName(year, district_abbrev) logging.info("Creating {}".format(district_key)) district = District( id=district_key, year=year, abbreviation=district_abbrev, display_name=DistrictType.type_names[dcmp.event_district_enum], elasticsearch_name=next( (k for k, v in DistrictType.elasticsearch_names.iteritems() if v == dcmp.event_district_enum), None)) districts_to_write.append(district) logging.info("Writing {} new districts".format( len(districts_to_write))) DistrictManipulator.createOrUpdate(districts_to_write, run_post_update_hook=False) for dcmp in year_dcmps: district_abbrev = DistrictType.type_abbrevs[ dcmp.event_district_enum] district_key = District.renderKeyName(year, district_abbrev) district_events_future = DistrictEventsQuery( district_key).fetch_async() district_events = district_events_future.get_result() logging.info("Found {} events to update".format( len(district_events))) events_to_write = [] for event in district_events: event.district_key = ndb.Key(District, district_key) events_to_write.append(event) EventManipulator.createOrUpdate(events_to_write) for dcmp in year_dcmps: district_abbrev = DistrictType.type_abbrevs[ dcmp.event_district_enum] district_key = District.renderKeyName(year, district_abbrev) districtteams_future = DistrictTeam.query( DistrictTeam.year == year, DistrictTeam.district == DistrictType.abbrevs.get( district_abbrev, None)).fetch_async() districtteams = districtteams_future.get_result() logging.info("Found {} DistrictTeams to update".format( len(districtteams))) districtteams_to_write = [] for districtteam in districtteams: districtteam.district_key = ndb.Key(District, district_key) districtteams_to_write.append(districtteam) DistrictTeamManipulator.createOrUpdate(districtteams_to_write)
def postUpdateHook(cls, districts, updated_attr_list, is_new_list): """ To run after a district has been updated. For new districts, tries to guess the names based on other year's data """ for (district, is_new, updated_attrs) in zip(districts, is_new_list, updated_attr_list): if is_new and (not district.display_name or not district.elasticsearch_name): last_year_key = District.renderKeyName(district.year - 1, district.abbreviation) last_year_district = District.get_by_id(last_year_key) update = False if last_year_district: if not district.display_name: district.display_name = last_year_district.display_name update = True if not district.elasticsearch_name: district.elasticsearch_name = last_year_district.elasticsearch_name update = True if update: cls.createOrUpdate(district, run_post_update_hook=False) if 'display_name' in updated_attrs or 'elasticsearch_name' in updated_attrs: # Set all other instances of this district to have the values all_past_years = DistrictHistoryQuery(district.abbreviation).fetch() to_put = [] for other_district in all_past_years: if other_district.year != district.year: other_district.display_name = district.display_name other_district.elasticsearch_name = district.elasticsearch_name to_put.append(other_district) cls.createOrUpdate(to_put, run_post_update_hook=False)
def parse(self, response): events = [] districts = {} for event in response['Events']: code = event['code'].lower() event_type = EventType.PRESEASON if code == 'week0' else self.EVENT_TYPES.get(event['type'].lower(), None) if event_type is None: logging.warn("Event type '{}' not recognized!".format(event['type'])) continue name = event['name'] short_name = EventHelper.getShortName(name) district_enum = EventHelper.parseDistrictName(event['districtCode'].lower()) if event['districtCode'] else DistrictType.NO_DISTRICT district_key = District.renderKeyName(self.season, event['districtCode'].lower()) if event['districtCode'] else None venue = event['venue'] city = event['city'] state_prov = event['stateprov'] country = event['country'] start = datetime.datetime.strptime(event['dateStart'], self.DATE_FORMAT_STR) end = datetime.datetime.strptime(event['dateEnd'], self.DATE_FORMAT_STR) website = event.get('website') # TODO read timezone from API # Special cases for champs if code in self.EVENT_CODE_EXCEPTIONS: code, short_name = self.EVENT_CODE_EXCEPTIONS[code] if code in self.EINSTEIN_CODES: name = '{} Field'.format(short_name) start = end.replace(hour=0, minute=0, second=0, microsecond=0) # Set to beginning of last day else: # Divisions name = '{} Division'.format(short_name) events.append(Event( id="{}{}".format(self.season, code), name=name, short_name=short_name, event_short=code, event_type_enum=event_type, official=True, start_date=start, end_date=end, venue=venue, city=city, state_prov=state_prov, country=country, venue_address=None, # Even though FRC API provides address, ElasticSearch is more detailed year=self.season, event_district_enum=district_enum, district_key=ndb.Key(District, district_key) if district_key else None, website=website, )) # Build District Model if district_key and district_key not in districts: districts[district_key] = District( id=district_key, year=self.season, abbreviation=event['districtCode'].lower(), ) return events, list(districts.values())
def postUpdateHook(cls, districts, updated_attr_list, is_new_list): """ To run after a district has been updated. For new districts, tries to guess the names based on other year's data """ for (district, is_new, updated_attrs) in zip(districts, is_new_list, updated_attr_list): if is_new and (not district.display_name or not district.elasticsearch_name): last_year_key = District.renderKeyName(district.year - 1, district.abbreviation) last_year_district = District.get_by_id(last_year_key) update = False if last_year_district: if not district.display_name: district.display_name = last_year_district.display_name update = True if not district.elasticsearch_name: district.elasticsearch_name = last_year_district.elasticsearch_name update = True if update: cls.createOrUpdate(district, run_post_update_hook=False) if 'display_name' in updated_attrs or 'elasticsearch_name' in updated_attrs: # Set all other instances of this district to have the values all_past_years = DistrictHistoryQuery( district.abbreviation).fetch() to_put = [] for other_district in all_past_years: if other_district.year != district.year: other_district.display_name = district.display_name other_district.elasticsearch_name = district.elasticsearch_name to_put.append(other_district) cls.createOrUpdate(to_put, run_post_update_hook=False)
def get(self, year): year = int(year) year_dcmps = DistrictChampsInYearQuery(year).fetch() districts_to_write = [] for dcmp in year_dcmps: district_abbrev = DistrictType.type_abbrevs[dcmp.event_district_enum] district_key = District.renderKeyName(year, district_abbrev) logging.info("Creating {}".format(district_key)) district = District( id=district_key, year=year, abbreviation=district_abbrev, display_name=DistrictType.type_names[dcmp.event_district_enum], elasticsearch_name=next((k for k, v in DistrictType.elasticsearch_names.iteritems() if v == dcmp.event_district_enum), None) ) districts_to_write.append(district) logging.info("Writing {} new districts".format(len(districts_to_write))) DistrictManipulator.createOrUpdate(districts_to_write, run_post_update_hook=False) for dcmp in year_dcmps: district_abbrev = DistrictType.type_abbrevs[dcmp.event_district_enum] district_key = District.renderKeyName(year, district_abbrev) district_events_future = DistrictEventsQuery(district_key).fetch_async() district_events = district_events_future.get_result() logging.info("Found {} events to update".format(len(district_events))) events_to_write = [] for event in district_events: event.district_key = ndb.Key(District, district_key) events_to_write.append(event) EventManipulator.createOrUpdate(events_to_write) for dcmp in year_dcmps: district_abbrev = DistrictType.type_abbrevs[dcmp.event_district_enum] district_key = District.renderKeyName(year, district_abbrev) districtteams_future = DistrictTeam.query(DistrictTeam.year == year, DistrictTeam.district == DistrictType.abbrevs.get(district_abbrev, None)).fetch_async() districtteams = districtteams_future.get_result() logging.info("Found {} DistrictTeams to update".format(len(districtteams))) districtteams_to_write = [] for districtteam in districtteams: districtteam.district_key = ndb.Key(District, district_key) districtteams_to_write.append(districtteam) DistrictTeamManipulator.createOrUpdate(districtteams_to_write)
def parse(self, response): """ Parse team info from FMSAPI Returns a tuple of: list of models (Team, DistrictTeam, Robot), and a Boolean indicating if there are more pages to be fetched """ # Get team json # don't need to null check, if error, HTTP code != 200, so we wont' get here current_page = response['pageCurrent'] total_pages = response['pageTotal'] teams = response['teams'] ret_models = [] for teamData in teams: # Fix issue where FIRST's API returns dummy website for all teams if teamData['website'] is not None and 'www.firstinspires.org' in teamData['website']: website = None else: website = WebsiteHelper.format_url(teamData.get('website', None)) team = Team( id="frc{}".format(teamData['teamNumber']), team_number=teamData['teamNumber'], name=teamData['nameFull'], nickname=teamData['nameShort'], school_name=teamData.get('schoolName'), home_cmp=teamData.get('homeCMP').lower() if teamData.get('homeCMP') else None, city=teamData['city'], state_prov=teamData['stateProv'], country=teamData['country'], website=website, rookie_year=teamData['rookieYear'] ) districtTeam = None if teamData['districtCode']: districtAbbrev = DistrictType.abbrevs[teamData['districtCode'].lower()] districtTeam = DistrictTeam( id=DistrictTeam.renderKeyName(self.year, districtAbbrev, team.key_name), team=ndb.Key(Team, team.key_name), year=self.year, district=districtAbbrev, district_key=ndb.Key(District, District.renderKeyName(self.year, teamData['districtCode'].lower())), ) robot = None if teamData['robotName']: robot = Robot( id=Robot.renderKeyName(team.key_name, self.year), team=ndb.Key(Team, team.key_name), year=self.year, robot_name=teamData['robotName'].strip() ) ret_models.append((team, districtTeam, robot)) return (ret_models, (current_page < total_pages))
def parse(self, response): """ Parse team info from FMSAPI Returns a tuple of: list of models (Team, DistrictTeam, Robot), and a Boolean indicating if there are more pages to be fetched """ # Get team json # don't need to null check, if error, HTTP code != 200, so we wont' get here current_page = response['pageCurrent'] total_pages = response['pageTotal'] teams = response['teams'] ret_models = [] for teamData in teams: # Fix issue where FIRST's API returns dummy website for all teams if teamData[ 'website'] is not None and 'www.firstinspires.org' in teamData[ 'website']: website = None else: website = WebsiteHelper.format_url( teamData.get('website', None)) team = Team(id="frc{}".format(teamData['teamNumber']), team_number=teamData['teamNumber'], name=teamData['nameFull'], nickname=teamData['nameShort'], school_name=teamData.get('schoolName'), home_cmp=teamData.get('homeCMP').lower() if teamData.get('homeCMP') else None, city=teamData['city'], state_prov=teamData['stateProv'], country=teamData['country'], website=website, rookie_year=teamData['rookieYear']) districtTeam = None if teamData['districtCode']: districtKey = District.renderKeyName( self.year, teamData['districtCode'].lower()) districtTeam = DistrictTeam( id=DistrictTeam.renderKeyName(districtKey, team.key_name), team=ndb.Key(Team, team.key_name), year=self.year, district_key=ndb.Key(District, districtKey), ) robot = None if teamData['robotName']: robot = Robot(id=Robot.renderKeyName(team.key_name, self.year), team=ndb.Key(Team, team.key_name), year=self.year, robot_name=teamData['robotName'].strip()) ret_models.append((team, districtTeam, robot)) return (ret_models, (current_page < total_pages))
def post(self, district_key): self._require_admin() district = District(id=District.renderKeyName( self.request.get("year"), self.request.get("abbreviation")), year=int(self.request.get("year")), abbreviation=self.request.get("abbreviation"), display_name=self.request.get("display_name")) DistrictManipulator.createOrUpdate(district) self.redirect('/admin/districts/' + self.request.get("year"))
def parse(self, response): districts = [] for district in response['districts']: district_code = district['code'].lower() district_key = District.renderKeyName(self.season, district_code) districts.append(District( id=district_key, abbreviation=district_code, year=self.season, display_name=district['name'], )) return districts
def parse(self, response): districts = [] for district in response['districts']: district_code = district['code'].lower() district_key = District.renderKeyName(self.season, district_code) districts.append( District( id=district_key, abbreviation=district_code, year=self.season, display_name=district['name'], )) return districts
def setUp(self): self.testbed = testbed.Testbed() self.testbed.activate() self.testbed.init_datastore_v3_stub() self.testbed.init_memcache_stub() self.testbed.init_taskqueue_stub(root_path=".") ndb.get_context().clear_cache() # Prevent data from leaking between tests # Create districts districts = [] for code in ['mar', 'isr', 'nc', 'ne', 'pnw', 'pch', 'chs', 'in', 'ont', 'fim', 'tx']: year = 2017 districts.append(District( id=District.renderKeyName(year, code), year=year, abbreviation=code, )) DistrictManipulator.createOrUpdate(districts)
def _render(self, district_abbrev, year=None): self._set_district(district_abbrev, self.year) if self.year < 2009: return json.dumps([], ensure_ascii=True) events_future = DistrictEventsQuery( District.renderKeyName(self.year, district_abbrev)).fetch_async() district_teams_future = DistrictTeamsQuery("{}{}".format( year, district_abbrev)).fetch_async() events = events_future.get_result() if not events: return json.dumps([], ensure_ascii=True) EventHelper.sort_events(events) team_totals = DistrictHelper.calculate_rankings( events, district_teams_future.get_result(), self.year) rankings = [] current_rank = 1 for key, points in team_totals: point_detail = {} point_detail["rank"] = current_rank point_detail["team_key"] = key point_detail["event_points"] = {} for event in points["event_points"]: event_key = event[0].key_name point_detail["event_points"][event_key] = event[1] event_details = Event.get_by_id(event_key) point_detail["event_points"][event[0].key_name][ 'district_cmp'] = True if event_details.event_type_enum == EventType.DISTRICT_CMP else False if "rookie_bonus" in points: point_detail["rookie_bonus"] = points["rookie_bonus"] else: point_detail["rookie_bonus"] = 0 point_detail["point_total"] = points["point_total"] rankings.append(point_detail) current_rank += 1 return json.dumps(rankings)
def _render(self, district_abbrev, year=None): self._set_district(district_abbrev, self.year) if self.year < 2009: return json.dumps([], ensure_ascii=True) events_future = DistrictEventsQuery(District.renderKeyName(self.year, district_abbrev)).fetch_async() district_teams_future = DistrictTeamsQuery("{}{}".format(year, district_abbrev)).fetch_async() events = events_future.get_result() if not events: return json.dumps([], ensure_ascii=True) EventHelper.sort_events(events) team_totals = DistrictHelper.calculate_rankings(events, district_teams_future.get_result(), self.year) rankings = [] current_rank = 1 for key, points in team_totals: point_detail = {} point_detail["rank"] = current_rank point_detail["team_key"] = key point_detail["event_points"] = {} for event in points["event_points"]: event_key = event[0].key_name point_detail["event_points"][event_key] = event[1] event_details = Event.get_by_id(event_key) point_detail["event_points"][event[0].key_name]['district_cmp'] = True if event_details.event_type_enum == EventType.DISTRICT_CMP else False if "rookie_bonus" in points: point_detail["rookie_bonus"] = points["rookie_bonus"] else: point_detail["rookie_bonus"] = 0 point_detail["point_total"] = points["point_total"] rankings.append(point_detail) current_rank += 1 return json.dumps(rankings)
def setUp(self): self.testbed = testbed.Testbed() self.testbed.activate() self.testbed.init_datastore_v3_stub() self.testbed.init_memcache_stub() self.testbed.init_taskqueue_stub(root_path=".") ndb.get_context().clear_cache( ) # Prevent data from leaking between tests # Create districts districts = [] for code in [ 'mar', 'isr', 'nc', 'ne', 'pnw', 'pch', 'chs', 'in', 'ont', 'fim', 'tx' ]: year = 2017 districts.append( District( id=District.renderKeyName(year, code), year=year, abbreviation=code, )) DistrictManipulator.createOrUpdate(districts)
def parse(self, response): events = [] districts = {} for event in response['Events']: code = event['code'].lower() event_type = EventType.PRESEASON if code == 'week0' else self.EVENT_TYPES.get( event['type'].lower(), None) if event_type is None: logging.warn("Event type '{}' not recognized!".format( event['type'])) continue name = event['name'] short_name = EventHelper.getShortName(name) district_enum = EventHelper.parseDistrictName( event['districtCode'].lower( )) if event['districtCode'] else DistrictType.NO_DISTRICT district_key = District.renderKeyName( self.season, event['districtCode'].lower() ) if event['districtCode'] else None venue = event['venue'] city = event['city'] state_prov = event['stateprov'] country = event['country'] start = datetime.datetime.strptime(event['dateStart'], self.DATE_FORMAT_STR) end = datetime.datetime.strptime(event['dateEnd'], self.DATE_FORMAT_STR) website = event.get('website') # TODO read timezone from API # Special cases for champs if code in self.EVENT_CODE_EXCEPTIONS: code, short_name = self.EVENT_CODE_EXCEPTIONS[code] if code in self.EINSTEIN_CODES: name = '{} Field'.format(short_name) start = end.replace( hour=0, minute=0, second=0, microsecond=0) # Set to beginning of last day else: # Divisions name = '{} Division'.format(short_name) events.append( Event( id="{}{}".format(self.season, code), name=name, short_name=short_name, event_short=code, event_type_enum=event_type, official=True, start_date=start, end_date=end, venue=venue, city=city, state_prov=state_prov, country=country, venue_address= None, # Even though FRC API provides address, ElasticSearch is more detailed year=self.season, event_district_enum=district_enum, district_key=ndb.Key(District, district_key) if district_key else None, website=website, )) # Build District Model if district_key and district_key not in districts: districts[district_key] = District( id=district_key, year=self.season, abbreviation=event['districtCode'].lower(), ) return events, list(districts.values())
def parse(self, response): """ Parse team info from FMSAPI Returns a tuple of: list of models (Team, DistrictTeam, Robot), and a Boolean indicating if there are more pages to be fetched """ # Get team json # don't need to null check, if error, HTTP code != 200, so we wont' get here current_page = response['pageCurrent'] total_pages = response['pageTotal'] teams = response['teams'] ret_models = [] for teamData in teams: # Fix issue where FIRST's API returns dummy website for all teams if teamData[ 'website'] is not None and 'www.firstinspires.org' in teamData[ 'website']: website = None else: raw_website = teamData.get('website', None) website = urlparse.urlparse( raw_website, 'http').geturl() if raw_website else None # Fix oddity with urlparse having three slashes after the scheme (#1635) website = website.replace('///', '//') if website else None team = Team(id="frc{}".format(teamData['teamNumber']), team_number=teamData['teamNumber'], name=teamData['nameFull'], nickname=teamData['nameShort'], city=teamData['city'], state_prov=teamData['stateProv'], country=teamData['country'], website=website, rookie_year=teamData['rookieYear']) districtTeam = None if teamData['districtCode']: districtAbbrev = DistrictType.abbrevs[ teamData['districtCode'].lower()] districtTeam = DistrictTeam( id=DistrictTeam.renderKeyName(self.year, districtAbbrev, team.key_name), team=ndb.Key(Team, team.key_name), year=self.year, district=districtAbbrev, district_key=ndb.Key( District, District.renderKeyName( self.year, teamData['districtCode'].lower())), ) robot = None if teamData['robotName']: robot = Robot(id=Robot.renderKeyName(team.key_name, self.year), team=ndb.Key(Team, team.key_name), year=self.year, robot_name=teamData['robotName'].strip()) ret_models.append((team, districtTeam, robot)) return (ret_models, (current_page < total_pages))
def parse(self, response): events = [] districts = {} cmp_hack_sitevar = Sitevar.get_or_insert('cmp_registration_hacks') divisions_to_skip = cmp_hack_sitevar.contents.get('divisions_to_skip', []) \ if cmp_hack_sitevar else [] event_name_override = cmp_hack_sitevar.contents.get('event_name_override', []) \ if cmp_hack_sitevar else [] events_to_change_dates = cmp_hack_sitevar.contents.get('set_start_to_last_day', []) \ if cmp_hack_sitevar else [] for event in response['Events']: code = event['code'].lower() event_type = EventType.PRESEASON if code == 'week0' else self.EVENT_TYPES.get(event['type'].lower(), None) if event_type is None: logging.warn("Event type '{}' not recognized!".format(event['type'])) continue name = event['name'] short_name = EventHelper.getShortName(name) district_enum = EventHelper.parseDistrictName(event['districtCode'].lower()) if event['districtCode'] else DistrictType.NO_DISTRICT district_key = District.renderKeyName(self.season, event['districtCode'].lower()) if event['districtCode'] else None venue = event['venue'] city = event['city'] state_prov = event['stateprov'] country = event['country'] start = datetime.datetime.strptime(event['dateStart'], self.DATE_FORMAT_STR) end = datetime.datetime.strptime(event['dateEnd'], self.DATE_FORMAT_STR) website = event.get('website') # TODO read timezone from API # Special cases for district championship divisions if event_type == EventType.DISTRICT_CMP_DIVISION: split_name = name.split('-') short_name = '{} - {}'.format( ''.join(item[0].upper() for item in split_name[0].split()), split_name[-1].replace('Division', '').strip()) # Special cases for champs if code in self.EVENT_CODE_EXCEPTIONS: code, short_name = self.EVENT_CODE_EXCEPTIONS[code] # FIRST indicates CMP registration before divisions are assigned by adding all teams # to Einstein. We will hack around that by not storing divisions and renaming # Einstein to simply "Championship" when certain sitevar flags are set if code in self.EINSTEIN_CODES: override = [item for item in event_name_override if item['event'] == "{}{}".format(self.season, code)] if override: name = short_name.format(override[0]['name']) short_name = short_name.format(override[0]['short_name']) else: # Divisions name = '{} Division'.format(short_name) event_key = "{}{}".format(self.season, code) if event_key in divisions_to_skip: continue # Allow an overriding the start date to be the beginning of the last day if event_key in events_to_change_dates: start = end.replace(hour=0, minute=0, second=0, microsecond=0) events.append(Event( id=event_key, name=name, short_name=short_name, event_short=code, event_type_enum=event_type, official=True, start_date=start, end_date=end, venue=venue, city=city, state_prov=state_prov, country=country, venue_address=None, # Even though FRC API provides address, ElasticSearch is more detailed year=self.season, event_district_enum=district_enum, district_key=ndb.Key(District, district_key) if district_key else None, website=website, )) # Build District Model if district_key and district_key not in districts: districts[district_key] = District( id=district_key, year=self.season, abbreviation=event['districtCode'].lower(), ) return events, list(districts.values())
def parse(self, response): events = [] districts = {} cmp_hack_sitevar = Sitevar.get_or_insert('cmp_registration_hacks') divisions_to_skip = cmp_hack_sitevar.contents.get('divisions_to_skip', []) \ if cmp_hack_sitevar else [] event_name_override = cmp_hack_sitevar.contents.get('event_name_override', []) \ if cmp_hack_sitevar else [] events_to_change_dates = cmp_hack_sitevar.contents.get('set_start_to_last_day', []) \ if cmp_hack_sitevar else [] for event in response['Events']: code = event['code'].lower() api_event_type = event['type'].lower() event_type = EventType.PRESEASON if code == 'week0' else self.EVENT_TYPES.get(api_event_type, None) if event_type is None and not self.event_short: logging.warn("Event type '{}' not recognized!".format(api_event_type)) continue # Some event types should be marked as unofficial, so sync is disabled official = True if api_event_type in self.NON_OFFICIAL_EVENT_TYPES: official = False name = event['name'] short_name = EventHelper.getShortName(name, district_code=event['districtCode']) district_enum = EventHelper.parseDistrictName(event['districtCode'].lower()) if event['districtCode'] else DistrictType.NO_DISTRICT district_key = District.renderKeyName(self.season, event['districtCode'].lower()) if event['districtCode'] else None address = event.get('address') venue = event['venue'] city = event['city'] state_prov = event['stateprov'] country = event['country'] start = datetime.datetime.strptime(event['dateStart'], self.DATE_FORMAT_STR) end = datetime.datetime.strptime(event['dateEnd'], self.DATE_FORMAT_STR) website = event.get('website') webcasts = [WebcastParser.webcast_dict_from_url(url) for url in event.get('webcasts', [])] # TODO read timezone from API # Special cases for district championship divisions if event_type == EventType.DISTRICT_CMP_DIVISION: split_name = name.split('-') short_name = '{} - {}'.format( ''.join(item[0].upper() for item in split_name[0].split()), split_name[-1].replace('Division', '').strip()) # Special cases for champs if code in self.EVENT_CODE_EXCEPTIONS: code, short_name = self.EVENT_CODE_EXCEPTIONS[code] # FIRST indicates CMP registration before divisions are assigned by adding all teams # to Einstein. We will hack around that by not storing divisions and renaming # Einstein to simply "Championship" when certain sitevar flags are set if code in self.EINSTEIN_CODES: override = [item for item in event_name_override if item['event'] == "{}{}".format(self.season, code)] if override: name = short_name.format(override[0]['name']) short_name = short_name.format(override[0]['short_name']) else: # Divisions name = '{} Division'.format(short_name) elif self.event_short: code = self.event_short event_key = "{}{}".format(self.season, code) if event_key in divisions_to_skip: continue # Allow an overriding the start date to be the beginning of the last day if event_key in events_to_change_dates: start = end.replace(hour=0, minute=0, second=0, microsecond=0) events.append(Event( id=event_key, name=name, short_name=short_name, event_short=code, event_type_enum=event_type, official=official, start_date=start, end_date=end, venue=venue, city=city, state_prov=state_prov, country=country, venue_address=address, year=self.season, event_district_enum=district_enum, district_key=ndb.Key(District, district_key) if district_key else None, website=website, webcast_json=json.dumps(webcasts) if webcasts else None, )) # Build District Model if district_key and district_key not in districts: districts[district_key] = District( id=district_key, year=self.season, abbreviation=event['districtCode'].lower(), ) # Prep for division <-> parent associations district_champs_by_district = {} champ_events = [] for event in events: if event.event_type_enum == EventType.DISTRICT_CMP: district_champs_by_district[event.district_key] = event elif event.event_type_enum == EventType.CMP_FINALS: champ_events.append(event) # Build district cmp division <-> parent associations based on district # Build cmp division <-> parent associations based on date for event in events: parent_event = None if event.event_type_enum == EventType.DISTRICT_CMP_DIVISION: parent_event = district_champs_by_district.get(event.district_key) elif event.event_type_enum == EventType.CMP_DIVISION: for parent_event in champ_events: if abs(parent_event.end_date - event.end_date) < datetime.timedelta(days=1): break else: parent_event = None else: continue if parent_event is None: continue parent_event.divisions = sorted(parent_event.divisions + [event.key]) event.parent_event = parent_event.key return events, list(districts.values())
def parse(self, response): events = [] districts = {} cmp_hack_sitevar = Sitevar.get_or_insert('cmp_registration_hacks') store_cmp_division = cmp_hack_sitevar.contents.get('should_store_divisions', True) \ if cmp_hack_sitevar else True einstein_name = cmp_hack_sitevar.contents.get('einstein_name', self.EINSTEIN_NAME_DEFAULT) \ if cmp_hack_sitevar else self.EINSTEIN_NAME_DEFAULT einstein_short_name = cmp_hack_sitevar.contents.get('einstein_short_name', self.EINSTEIN_SHORT_NAME_DEFAULT) \ if cmp_hack_sitevar else self.EINSTEIN_SHORT_NAME_DEFAULT change_einstein_dates = cmp_hack_sitevar.contents.get('should_change_einstein_dates', False) \ if cmp_hack_sitevar else False for event in response['Events']: code = event['code'].lower() event_type = EventType.PRESEASON if code == 'week0' else self.EVENT_TYPES.get(event['type'].lower(), None) if event_type is None: logging.warn("Event type '{}' not recognized!".format(event['type'])) continue name = event['name'] short_name = EventHelper.getShortName(name) district_enum = EventHelper.parseDistrictName(event['districtCode'].lower()) if event['districtCode'] else DistrictType.NO_DISTRICT district_key = District.renderKeyName(self.season, event['districtCode'].lower()) if event['districtCode'] else None venue = event['venue'] city = event['city'] state_prov = event['stateprov'] country = event['country'] start = datetime.datetime.strptime(event['dateStart'], self.DATE_FORMAT_STR) end = datetime.datetime.strptime(event['dateEnd'], self.DATE_FORMAT_STR) website = event.get('website') # TODO read timezone from API # Special cases for champs if code in self.EVENT_CODE_EXCEPTIONS: code, short_name = self.EVENT_CODE_EXCEPTIONS[code] # FIRST indicates CMP registration before divisions are assigned by adding all teams # to Einstein. We will hack around that by not storing divisions and renaming # Einstein to simply "Championship" when certain sitevar flags are set if code in self.EINSTEIN_CODES: name = short_name.format(einstein_name) short_name = short_name.format(einstein_short_name) if change_einstein_dates: # Set to beginning of last day start = end.replace(hour=0, minute=0, second=0, microsecond=0) else: # Divisions name = '{} Division'.format(short_name) # Allow skipping storing CMP divisions before they're announced if not store_cmp_division: continue events.append(Event( id="{}{}".format(self.season, code), name=name, short_name=short_name, event_short=code, event_type_enum=event_type, official=True, start_date=start, end_date=end, venue=venue, city=city, state_prov=state_prov, country=country, venue_address=None, # Even though FRC API provides address, ElasticSearch is more detailed year=self.season, event_district_enum=district_enum, district_key=ndb.Key(District, district_key) if district_key else None, website=website, )) # Build District Model if district_key and district_key not in districts: districts[district_key] = District( id=district_key, year=self.season, abbreviation=event['districtCode'].lower(), ) return events, list(districts.values())
def parse(self, response): events = [] districts = {} cmp_hack_sitevar = Sitevar.get_or_insert('cmp_registration_hacks') divisions_to_skip = cmp_hack_sitevar.contents.get('divisions_to_skip', []) \ if cmp_hack_sitevar else [] event_name_override = cmp_hack_sitevar.contents.get('event_name_override', []) \ if cmp_hack_sitevar else [] events_to_change_dates = cmp_hack_sitevar.contents.get('set_start_to_last_day', []) \ if cmp_hack_sitevar else [] for event in response['Events']: code = event['code'].lower() event_type = EventType.PRESEASON if code == 'week0' else self.EVENT_TYPES.get(event['type'].lower(), None) if event_type is None and not self.event_short: logging.warn("Event type '{}' not recognized!".format(event['type'])) continue name = event['name'] short_name = EventHelper.getShortName(name, district_code=event['districtCode']) district_enum = EventHelper.parseDistrictName(event['districtCode'].lower()) if event['districtCode'] else DistrictType.NO_DISTRICT district_key = District.renderKeyName(self.season, event['districtCode'].lower()) if event['districtCode'] else None venue = event['venue'] city = event['city'] state_prov = event['stateprov'] country = event['country'] start = datetime.datetime.strptime(event['dateStart'], self.DATE_FORMAT_STR) end = datetime.datetime.strptime(event['dateEnd'], self.DATE_FORMAT_STR) website = event.get('website') # TODO read timezone from API # Special cases for district championship divisions if event_type == EventType.DISTRICT_CMP_DIVISION: split_name = name.split('-') short_name = '{} - {}'.format( ''.join(item[0].upper() for item in split_name[0].split()), split_name[-1].replace('Division', '').strip()) # Special cases for champs if code in self.EVENT_CODE_EXCEPTIONS: code, short_name = self.EVENT_CODE_EXCEPTIONS[code] # FIRST indicates CMP registration before divisions are assigned by adding all teams # to Einstein. We will hack around that by not storing divisions and renaming # Einstein to simply "Championship" when certain sitevar flags are set if code in self.EINSTEIN_CODES: override = [item for item in event_name_override if item['event'] == "{}{}".format(self.season, code)] if override: name = short_name.format(override[0]['name']) short_name = short_name.format(override[0]['short_name']) else: # Divisions name = '{} Division'.format(short_name) elif self.event_short: code = self.event_short event_key = "{}{}".format(self.season, code) if event_key in divisions_to_skip: continue # Allow an overriding the start date to be the beginning of the last day if event_key in events_to_change_dates: start = end.replace(hour=0, minute=0, second=0, microsecond=0) events.append(Event( id=event_key, name=name, short_name=short_name, event_short=code, event_type_enum=event_type, official=True, start_date=start, end_date=end, venue=venue, city=city, state_prov=state_prov, country=country, venue_address=None, # Even though FRC API provides address, ElasticSearch is more detailed year=self.season, event_district_enum=district_enum, district_key=ndb.Key(District, district_key) if district_key else None, website=website, )) # Build District Model if district_key and district_key not in districts: districts[district_key] = District( id=district_key, year=self.season, abbreviation=event['districtCode'].lower(), ) return events, list(districts.values())