def get(self, year):
year = int(year)
year_dcmps = DistrictChampsInYearQuery(year).fetch()
districts_to_write = []
for dcmp in year_dcmps:
district_abbrev = DistrictType.type_abbrevs[
dcmp.event_district_enum]
district_key = District.renderKeyName(year, district_abbrev)
logging.info("Creating {}".format(district_key))
district = District(
id=district_key,
year=year,
abbreviation=district_abbrev,
display_name=DistrictType.type_names[dcmp.event_district_enum],
elasticsearch_name=next(
(k
for k, v in DistrictType.elasticsearch_names.iteritems()
if v == dcmp.event_district_enum), None))
districts_to_write.append(district)
logging.info("Writing {} new districts".format(
len(districts_to_write)))
DistrictManipulator.createOrUpdate(districts_to_write,
run_post_update_hook=False)
for dcmp in year_dcmps:
district_abbrev = DistrictType.type_abbrevs[
dcmp.event_district_enum]
district_key = District.renderKeyName(year, district_abbrev)
district_events_future = DistrictEventsQuery(
district_key).fetch_async()
district_events = district_events_future.get_result()
logging.info("Found {} events to update".format(
len(district_events)))
events_to_write = []
for event in district_events:
event.district_key = ndb.Key(District, district_key)
events_to_write.append(event)
EventManipulator.createOrUpdate(events_to_write)
for dcmp in year_dcmps:
district_abbrev = DistrictType.type_abbrevs[
dcmp.event_district_enum]
district_key = District.renderKeyName(year, district_abbrev)
districtteams_future = DistrictTeam.query(
DistrictTeam.year == year,
DistrictTeam.district == DistrictType.abbrevs.get(
district_abbrev, None)).fetch_async()
districtteams = districtteams_future.get_result()
logging.info("Found {} DistrictTeams to update".format(
len(districtteams)))
districtteams_to_write = []
for districtteam in districtteams:
districtteam.district_key = ndb.Key(District, district_key)
districtteams_to_write.append(districtteam)
DistrictTeamManipulator.createOrUpdate(districtteams_to_write)
def postUpdateHook(cls, districts, updated_attr_list, is_new_list):
"""
To run after a district has been updated.
For new districts, tries to guess the names based on other year's data
"""
for (district, is_new, updated_attrs) in zip(districts, is_new_list, updated_attr_list):
if is_new and (not district.display_name or not district.elasticsearch_name):
last_year_key = District.renderKeyName(district.year - 1, district.abbreviation)
last_year_district = District.get_by_id(last_year_key)
update = False
if last_year_district:
if not district.display_name:
district.display_name = last_year_district.display_name
update = True
if not district.elasticsearch_name:
district.elasticsearch_name = last_year_district.elasticsearch_name
update = True
if update:
cls.createOrUpdate(district, run_post_update_hook=False)
if 'display_name' in updated_attrs or 'elasticsearch_name' in updated_attrs:
# Set all other instances of this district to have the values
all_past_years = DistrictHistoryQuery(district.abbreviation).fetch()
to_put = []
for other_district in all_past_years:
if other_district.year != district.year:
other_district.display_name = district.display_name
other_district.elasticsearch_name = district.elasticsearch_name
to_put.append(other_district)
cls.createOrUpdate(to_put, run_post_update_hook=False)
def parse(self, response):
events = []
districts = {}
for event in response['Events']:
code = event['code'].lower()
event_type = EventType.PRESEASON if code == 'week0' else self.EVENT_TYPES.get(event['type'].lower(), None)
if event_type is None:
logging.warn("Event type '{}' not recognized!".format(event['type']))
continue
name = event['name']
short_name = EventHelper.getShortName(name)
district_enum = EventHelper.parseDistrictName(event['districtCode'].lower()) if event['districtCode'] else DistrictType.NO_DISTRICT
district_key = District.renderKeyName(self.season, event['districtCode'].lower()) if event['districtCode'] else None
venue = event['venue']
city = event['city']
state_prov = event['stateprov']
country = event['country']
start = datetime.datetime.strptime(event['dateStart'], self.DATE_FORMAT_STR)
end = datetime.datetime.strptime(event['dateEnd'], self.DATE_FORMAT_STR)
website = event.get('website')
# TODO read timezone from API
# Special cases for champs
if code in self.EVENT_CODE_EXCEPTIONS:
code, short_name = self.EVENT_CODE_EXCEPTIONS[code]
if code in self.EINSTEIN_CODES:
name = '{} Field'.format(short_name)
start = end.replace(hour=0, minute=0, second=0, microsecond=0) # Set to beginning of last day
else: # Divisions
name = '{} Division'.format(short_name)
events.append(Event(
id="{}{}".format(self.season, code),
name=name,
short_name=short_name,
event_short=code,
event_type_enum=event_type,
official=True,
start_date=start,
end_date=end,
venue=venue,
city=city,
state_prov=state_prov,
country=country,
venue_address=None, # Even though FRC API provides address, ElasticSearch is more detailed
year=self.season,
event_district_enum=district_enum,
district_key=ndb.Key(District, district_key) if district_key else None,
website=website,
))
# Build District Model
if district_key and district_key not in districts:
districts[district_key] = District(
id=district_key,
year=self.season,
abbreviation=event['districtCode'].lower(),
)
return events, list(districts.values())
def postUpdateHook(cls, districts, updated_attr_list, is_new_list):
"""
To run after a district has been updated.
For new districts, tries to guess the names based on other year's data
"""
for (district, is_new, updated_attrs) in zip(districts, is_new_list,
updated_attr_list):
if is_new and (not district.display_name
or not district.elasticsearch_name):
last_year_key = District.renderKeyName(district.year - 1,
district.abbreviation)
last_year_district = District.get_by_id(last_year_key)
update = False
if last_year_district:
if not district.display_name:
district.display_name = last_year_district.display_name
update = True
if not district.elasticsearch_name:
district.elasticsearch_name = last_year_district.elasticsearch_name
update = True
if update:
cls.createOrUpdate(district,
run_post_update_hook=False)
if 'display_name' in updated_attrs or 'elasticsearch_name' in updated_attrs:
# Set all other instances of this district to have the values
all_past_years = DistrictHistoryQuery(
district.abbreviation).fetch()
to_put = []
for other_district in all_past_years:
if other_district.year != district.year:
other_district.display_name = district.display_name
other_district.elasticsearch_name = district.elasticsearch_name
to_put.append(other_district)
cls.createOrUpdate(to_put, run_post_update_hook=False)
def get(self, year):
year = int(year)
year_dcmps = DistrictChampsInYearQuery(year).fetch()
districts_to_write = []
for dcmp in year_dcmps:
district_abbrev = DistrictType.type_abbrevs[dcmp.event_district_enum]
district_key = District.renderKeyName(year, district_abbrev)
logging.info("Creating {}".format(district_key))
district = District(
id=district_key,
year=year,
abbreviation=district_abbrev,
display_name=DistrictType.type_names[dcmp.event_district_enum],
elasticsearch_name=next((k for k, v in DistrictType.elasticsearch_names.iteritems() if v == dcmp.event_district_enum), None)
)
districts_to_write.append(district)
logging.info("Writing {} new districts".format(len(districts_to_write)))
DistrictManipulator.createOrUpdate(districts_to_write, run_post_update_hook=False)
for dcmp in year_dcmps:
district_abbrev = DistrictType.type_abbrevs[dcmp.event_district_enum]
district_key = District.renderKeyName(year, district_abbrev)
district_events_future = DistrictEventsQuery(district_key).fetch_async()
district_events = district_events_future.get_result()
logging.info("Found {} events to update".format(len(district_events)))
events_to_write = []
for event in district_events:
event.district_key = ndb.Key(District, district_key)
events_to_write.append(event)
EventManipulator.createOrUpdate(events_to_write)
for dcmp in year_dcmps:
district_abbrev = DistrictType.type_abbrevs[dcmp.event_district_enum]
district_key = District.renderKeyName(year, district_abbrev)
districtteams_future = DistrictTeam.query(DistrictTeam.year == year, DistrictTeam.district == DistrictType.abbrevs.get(district_abbrev, None)).fetch_async()
districtteams = districtteams_future.get_result()
logging.info("Found {} DistrictTeams to update".format(len(districtteams)))
districtteams_to_write = []
for districtteam in districtteams:
districtteam.district_key = ndb.Key(District, district_key)
districtteams_to_write.append(districtteam)
DistrictTeamManipulator.createOrUpdate(districtteams_to_write)
def parse(self, response):
"""
Parse team info from FMSAPI
Returns a tuple of: list of models (Team, DistrictTeam, Robot),
and a Boolean indicating if there are more pages to be fetched
"""
# Get team json
# don't need to null check, if error, HTTP code != 200, so we wont' get here
current_page = response['pageCurrent']
total_pages = response['pageTotal']
teams = response['teams']
ret_models = []
for teamData in teams:
# Fix issue where FIRST's API returns dummy website for all teams
if teamData['website'] is not None and 'www.firstinspires.org' in teamData['website']:
website = None
else:
website = WebsiteHelper.format_url(teamData.get('website', None))
team = Team(
id="frc{}".format(teamData['teamNumber']),
team_number=teamData['teamNumber'],
name=teamData['nameFull'],
nickname=teamData['nameShort'],
school_name=teamData.get('schoolName'),
home_cmp=teamData.get('homeCMP').lower() if teamData.get('homeCMP') else None,
city=teamData['city'],
state_prov=teamData['stateProv'],
country=teamData['country'],
website=website,
rookie_year=teamData['rookieYear']
)
districtTeam = None
if teamData['districtCode']:
districtAbbrev = DistrictType.abbrevs[teamData['districtCode'].lower()]
districtTeam = DistrictTeam(
id=DistrictTeam.renderKeyName(self.year, districtAbbrev, team.key_name),
team=ndb.Key(Team, team.key_name),
year=self.year,
district=districtAbbrev,
district_key=ndb.Key(District, District.renderKeyName(self.year, teamData['districtCode'].lower())),
)
robot = None
if teamData['robotName']:
robot = Robot(
id=Robot.renderKeyName(team.key_name, self.year),
team=ndb.Key(Team, team.key_name),
year=self.year,
robot_name=teamData['robotName'].strip()
)
ret_models.append((team, districtTeam, robot))
return (ret_models, (current_page < total_pages))
def parse(self, response):
"""
Parse team info from FMSAPI
Returns a tuple of: list of models (Team, DistrictTeam, Robot),
and a Boolean indicating if there are more pages to be fetched
"""
# Get team json
# don't need to null check, if error, HTTP code != 200, so we wont' get here
current_page = response['pageCurrent']
total_pages = response['pageTotal']
teams = response['teams']
ret_models = []
for teamData in teams:
# Fix issue where FIRST's API returns dummy website for all teams
if teamData[
'website'] is not None and 'www.firstinspires.org' in teamData[
'website']:
website = None
else:
website = WebsiteHelper.format_url(
teamData.get('website', None))
team = Team(id="frc{}".format(teamData['teamNumber']),
team_number=teamData['teamNumber'],
name=teamData['nameFull'],
nickname=teamData['nameShort'],
school_name=teamData.get('schoolName'),
home_cmp=teamData.get('homeCMP').lower()
if teamData.get('homeCMP') else None,
city=teamData['city'],
state_prov=teamData['stateProv'],
country=teamData['country'],
website=website,
rookie_year=teamData['rookieYear'])
districtTeam = None
if teamData['districtCode']:
districtKey = District.renderKeyName(
self.year, teamData['districtCode'].lower())
districtTeam = DistrictTeam(
id=DistrictTeam.renderKeyName(districtKey, team.key_name),
team=ndb.Key(Team, team.key_name),
year=self.year,
district_key=ndb.Key(District, districtKey),
)
robot = None
if teamData['robotName']:
robot = Robot(id=Robot.renderKeyName(team.key_name, self.year),
team=ndb.Key(Team, team.key_name),
year=self.year,
robot_name=teamData['robotName'].strip())
ret_models.append((team, districtTeam, robot))
return (ret_models, (current_page < total_pages))
def post(self, district_key):
self._require_admin()
district = District(id=District.renderKeyName(
self.request.get("year"), self.request.get("abbreviation")),
year=int(self.request.get("year")),
abbreviation=self.request.get("abbreviation"),
display_name=self.request.get("display_name"))
DistrictManipulator.createOrUpdate(district)
self.redirect('/admin/districts/' + self.request.get("year"))
def parse(self, response):
districts = []
for district in response['districts']:
district_code = district['code'].lower()
district_key = District.renderKeyName(self.season, district_code)
districts.append(District(
id=district_key,
abbreviation=district_code,
year=self.season,
display_name=district['name'],
))
return districts
def parse(self, response):
districts = []
for district in response['districts']:
district_code = district['code'].lower()
district_key = District.renderKeyName(self.season, district_code)
districts.append(
District(
id=district_key,
abbreviation=district_code,
year=self.season,
display_name=district['name'],
))
return districts
def setUp(self):
self.testbed = testbed.Testbed()
self.testbed.activate()
self.testbed.init_datastore_v3_stub()
self.testbed.init_memcache_stub()
self.testbed.init_taskqueue_stub(root_path=".")
ndb.get_context().clear_cache() # Prevent data from leaking between tests
# Create districts
districts = []
for code in ['mar', 'isr', 'nc', 'ne', 'pnw', 'pch', 'chs', 'in', 'ont', 'fim', 'tx']:
year = 2017
districts.append(District(
id=District.renderKeyName(year, code),
year=year,
abbreviation=code,
))
DistrictManipulator.createOrUpdate(districts)
def _render(self, district_abbrev, year=None):
self._set_district(district_abbrev, self.year)
if self.year < 2009:
return json.dumps([], ensure_ascii=True)
events_future = DistrictEventsQuery(
District.renderKeyName(self.year, district_abbrev)).fetch_async()
district_teams_future = DistrictTeamsQuery("{}{}".format(
year, district_abbrev)).fetch_async()
events = events_future.get_result()
if not events:
return json.dumps([], ensure_ascii=True)
EventHelper.sort_events(events)
team_totals = DistrictHelper.calculate_rankings(
events, district_teams_future.get_result(), self.year)
rankings = []
current_rank = 1
for key, points in team_totals:
point_detail = {}
point_detail["rank"] = current_rank
point_detail["team_key"] = key
point_detail["event_points"] = {}
for event in points["event_points"]:
event_key = event[0].key_name
point_detail["event_points"][event_key] = event[1]
event_details = Event.get_by_id(event_key)
point_detail["event_points"][event[0].key_name][
'district_cmp'] = True if event_details.event_type_enum == EventType.DISTRICT_CMP else False
if "rookie_bonus" in points:
point_detail["rookie_bonus"] = points["rookie_bonus"]
else:
point_detail["rookie_bonus"] = 0
point_detail["point_total"] = points["point_total"]
rankings.append(point_detail)
current_rank += 1
return json.dumps(rankings)
def _render(self, district_abbrev, year=None):
self._set_district(district_abbrev, self.year)
if self.year < 2009:
return json.dumps([], ensure_ascii=True)
events_future = DistrictEventsQuery(District.renderKeyName(self.year, district_abbrev)).fetch_async()
district_teams_future = DistrictTeamsQuery("{}{}".format(year, district_abbrev)).fetch_async()
events = events_future.get_result()
if not events:
return json.dumps([], ensure_ascii=True)
EventHelper.sort_events(events)
team_totals = DistrictHelper.calculate_rankings(events, district_teams_future.get_result(), self.year)
rankings = []
current_rank = 1
for key, points in team_totals:
point_detail = {}
point_detail["rank"] = current_rank
point_detail["team_key"] = key
point_detail["event_points"] = {}
for event in points["event_points"]:
event_key = event[0].key_name
point_detail["event_points"][event_key] = event[1]
event_details = Event.get_by_id(event_key)
point_detail["event_points"][event[0].key_name]['district_cmp'] = True if event_details.event_type_enum == EventType.DISTRICT_CMP else False
if "rookie_bonus" in points:
point_detail["rookie_bonus"] = points["rookie_bonus"]
else:
point_detail["rookie_bonus"] = 0
point_detail["point_total"] = points["point_total"]
rankings.append(point_detail)
current_rank += 1
return json.dumps(rankings)
def setUp(self):
self.testbed = testbed.Testbed()
self.testbed.activate()
self.testbed.init_datastore_v3_stub()
self.testbed.init_memcache_stub()
self.testbed.init_taskqueue_stub(root_path=".")
ndb.get_context().clear_cache(
) # Prevent data from leaking between tests
# Create districts
districts = []
for code in [
'mar', 'isr', 'nc', 'ne', 'pnw', 'pch', 'chs', 'in', 'ont',
'fim', 'tx'
]:
year = 2017
districts.append(
District(
id=District.renderKeyName(year, code),
year=year,
abbreviation=code,
))
DistrictManipulator.createOrUpdate(districts)
def parse(self, response):
events = []
districts = {}
for event in response['Events']:
code = event['code'].lower()
event_type = EventType.PRESEASON if code == 'week0' else self.EVENT_TYPES.get(
event['type'].lower(), None)
if event_type is None:
logging.warn("Event type '{}' not recognized!".format(
event['type']))
continue
name = event['name']
short_name = EventHelper.getShortName(name)
district_enum = EventHelper.parseDistrictName(
event['districtCode'].lower(
)) if event['districtCode'] else DistrictType.NO_DISTRICT
district_key = District.renderKeyName(
self.season, event['districtCode'].lower()
) if event['districtCode'] else None
venue = event['venue']
city = event['city']
state_prov = event['stateprov']
country = event['country']
start = datetime.datetime.strptime(event['dateStart'],
self.DATE_FORMAT_STR)
end = datetime.datetime.strptime(event['dateEnd'],
self.DATE_FORMAT_STR)
website = event.get('website')
# TODO read timezone from API
# Special cases for champs
if code in self.EVENT_CODE_EXCEPTIONS:
code, short_name = self.EVENT_CODE_EXCEPTIONS[code]
if code in self.EINSTEIN_CODES:
name = '{} Field'.format(short_name)
start = end.replace(
hour=0, minute=0, second=0,
microsecond=0) # Set to beginning of last day
else: # Divisions
name = '{} Division'.format(short_name)
events.append(
Event(
id="{}{}".format(self.season, code),
name=name,
short_name=short_name,
event_short=code,
event_type_enum=event_type,
official=True,
start_date=start,
end_date=end,
venue=venue,
city=city,
state_prov=state_prov,
country=country,
venue_address=
None, # Even though FRC API provides address, ElasticSearch is more detailed
year=self.season,
event_district_enum=district_enum,
district_key=ndb.Key(District, district_key)
if district_key else None,
website=website,
))
# Build District Model
if district_key and district_key not in districts:
districts[district_key] = District(
id=district_key,
year=self.season,
abbreviation=event['districtCode'].lower(),
)
return events, list(districts.values())
def parse(self, response):
"""
Parse team info from FMSAPI
Returns a tuple of: list of models (Team, DistrictTeam, Robot),
and a Boolean indicating if there are more pages to be fetched
"""
# Get team json
# don't need to null check, if error, HTTP code != 200, so we wont' get here
current_page = response['pageCurrent']
total_pages = response['pageTotal']
teams = response['teams']
ret_models = []
for teamData in teams:
# Fix issue where FIRST's API returns dummy website for all teams
if teamData[
'website'] is not None and 'www.firstinspires.org' in teamData[
'website']:
website = None
else:
raw_website = teamData.get('website', None)
website = urlparse.urlparse(
raw_website, 'http').geturl() if raw_website else None
# Fix oddity with urlparse having three slashes after the scheme (#1635)
website = website.replace('///', '//') if website else None
team = Team(id="frc{}".format(teamData['teamNumber']),
team_number=teamData['teamNumber'],
name=teamData['nameFull'],
nickname=teamData['nameShort'],
city=teamData['city'],
state_prov=teamData['stateProv'],
country=teamData['country'],
website=website,
rookie_year=teamData['rookieYear'])
districtTeam = None
if teamData['districtCode']:
districtAbbrev = DistrictType.abbrevs[
teamData['districtCode'].lower()]
districtTeam = DistrictTeam(
id=DistrictTeam.renderKeyName(self.year, districtAbbrev,
team.key_name),
team=ndb.Key(Team, team.key_name),
year=self.year,
district=districtAbbrev,
district_key=ndb.Key(
District,
District.renderKeyName(
self.year, teamData['districtCode'].lower())),
)
robot = None
if teamData['robotName']:
robot = Robot(id=Robot.renderKeyName(team.key_name, self.year),
team=ndb.Key(Team, team.key_name),
year=self.year,
robot_name=teamData['robotName'].strip())
ret_models.append((team, districtTeam, robot))
return (ret_models, (current_page < total_pages))
def parse(self, response):
events = []
districts = {}
cmp_hack_sitevar = Sitevar.get_or_insert('cmp_registration_hacks')
divisions_to_skip = cmp_hack_sitevar.contents.get('divisions_to_skip', []) \
if cmp_hack_sitevar else []
event_name_override = cmp_hack_sitevar.contents.get('event_name_override', []) \
if cmp_hack_sitevar else []
events_to_change_dates = cmp_hack_sitevar.contents.get('set_start_to_last_day', []) \
if cmp_hack_sitevar else []
for event in response['Events']:
code = event['code'].lower()
event_type = EventType.PRESEASON if code == 'week0' else self.EVENT_TYPES.get(event['type'].lower(), None)
if event_type is None:
logging.warn("Event type '{}' not recognized!".format(event['type']))
continue
name = event['name']
short_name = EventHelper.getShortName(name)
district_enum = EventHelper.parseDistrictName(event['districtCode'].lower()) if event['districtCode'] else DistrictType.NO_DISTRICT
district_key = District.renderKeyName(self.season, event['districtCode'].lower()) if event['districtCode'] else None
venue = event['venue']
city = event['city']
state_prov = event['stateprov']
country = event['country']
start = datetime.datetime.strptime(event['dateStart'], self.DATE_FORMAT_STR)
end = datetime.datetime.strptime(event['dateEnd'], self.DATE_FORMAT_STR)
website = event.get('website')
# TODO read timezone from API
# Special cases for district championship divisions
if event_type == EventType.DISTRICT_CMP_DIVISION:
split_name = name.split('-')
short_name = '{} - {}'.format(
''.join(item[0].upper() for item in split_name[0].split()),
split_name[-1].replace('Division', '').strip())
# Special cases for champs
if code in self.EVENT_CODE_EXCEPTIONS:
code, short_name = self.EVENT_CODE_EXCEPTIONS[code]
# FIRST indicates CMP registration before divisions are assigned by adding all teams
# to Einstein. We will hack around that by not storing divisions and renaming
# Einstein to simply "Championship" when certain sitevar flags are set
if code in self.EINSTEIN_CODES:
override = [item for item in event_name_override if item['event'] == "{}{}".format(self.season, code)]
if override:
name = short_name.format(override[0]['name'])
short_name = short_name.format(override[0]['short_name'])
else: # Divisions
name = '{} Division'.format(short_name)
event_key = "{}{}".format(self.season, code)
if event_key in divisions_to_skip:
continue
# Allow an overriding the start date to be the beginning of the last day
if event_key in events_to_change_dates:
start = end.replace(hour=0, minute=0, second=0, microsecond=0)
events.append(Event(
id=event_key,
name=name,
short_name=short_name,
event_short=code,
event_type_enum=event_type,
official=True,
start_date=start,
end_date=end,
venue=venue,
city=city,
state_prov=state_prov,
country=country,
venue_address=None, # Even though FRC API provides address, ElasticSearch is more detailed
year=self.season,
event_district_enum=district_enum,
district_key=ndb.Key(District, district_key) if district_key else None,
website=website,
))
# Build District Model
if district_key and district_key not in districts:
districts[district_key] = District(
id=district_key,
year=self.season,
abbreviation=event['districtCode'].lower(),
)
return events, list(districts.values())
def parse(self, response):
events = []
districts = {}
cmp_hack_sitevar = Sitevar.get_or_insert('cmp_registration_hacks')
divisions_to_skip = cmp_hack_sitevar.contents.get('divisions_to_skip', []) \
if cmp_hack_sitevar else []
event_name_override = cmp_hack_sitevar.contents.get('event_name_override', []) \
if cmp_hack_sitevar else []
events_to_change_dates = cmp_hack_sitevar.contents.get('set_start_to_last_day', []) \
if cmp_hack_sitevar else []
for event in response['Events']:
code = event['code'].lower()
api_event_type = event['type'].lower()
event_type = EventType.PRESEASON if code == 'week0' else self.EVENT_TYPES.get(api_event_type, None)
if event_type is None and not self.event_short:
logging.warn("Event type '{}' not recognized!".format(api_event_type))
continue
# Some event types should be marked as unofficial, so sync is disabled
official = True
if api_event_type in self.NON_OFFICIAL_EVENT_TYPES:
official = False
name = event['name']
short_name = EventHelper.getShortName(name, district_code=event['districtCode'])
district_enum = EventHelper.parseDistrictName(event['districtCode'].lower()) if event['districtCode'] else DistrictType.NO_DISTRICT
district_key = District.renderKeyName(self.season, event['districtCode'].lower()) if event['districtCode'] else None
address = event.get('address')
venue = event['venue']
city = event['city']
state_prov = event['stateprov']
country = event['country']
start = datetime.datetime.strptime(event['dateStart'], self.DATE_FORMAT_STR)
end = datetime.datetime.strptime(event['dateEnd'], self.DATE_FORMAT_STR)
website = event.get('website')
webcasts = [WebcastParser.webcast_dict_from_url(url) for url in event.get('webcasts', [])]
# TODO read timezone from API
# Special cases for district championship divisions
if event_type == EventType.DISTRICT_CMP_DIVISION:
split_name = name.split('-')
short_name = '{} - {}'.format(
''.join(item[0].upper() for item in split_name[0].split()),
split_name[-1].replace('Division', '').strip())
# Special cases for champs
if code in self.EVENT_CODE_EXCEPTIONS:
code, short_name = self.EVENT_CODE_EXCEPTIONS[code]
# FIRST indicates CMP registration before divisions are assigned by adding all teams
# to Einstein. We will hack around that by not storing divisions and renaming
# Einstein to simply "Championship" when certain sitevar flags are set
if code in self.EINSTEIN_CODES:
override = [item for item in event_name_override if item['event'] == "{}{}".format(self.season, code)]
if override:
name = short_name.format(override[0]['name'])
short_name = short_name.format(override[0]['short_name'])
else: # Divisions
name = '{} Division'.format(short_name)
elif self.event_short:
code = self.event_short
event_key = "{}{}".format(self.season, code)
if event_key in divisions_to_skip:
continue
# Allow an overriding the start date to be the beginning of the last day
if event_key in events_to_change_dates:
start = end.replace(hour=0, minute=0, second=0, microsecond=0)
events.append(Event(
id=event_key,
name=name,
short_name=short_name,
event_short=code,
event_type_enum=event_type,
official=official,
start_date=start,
end_date=end,
venue=venue,
city=city,
state_prov=state_prov,
country=country,
venue_address=address,
year=self.season,
event_district_enum=district_enum,
district_key=ndb.Key(District, district_key) if district_key else None,
website=website,
webcast_json=json.dumps(webcasts) if webcasts else None,
))
# Build District Model
if district_key and district_key not in districts:
districts[district_key] = District(
id=district_key,
year=self.season,
abbreviation=event['districtCode'].lower(),
)
# Prep for division <-> parent associations
district_champs_by_district = {}
champ_events = []
for event in events:
if event.event_type_enum == EventType.DISTRICT_CMP:
district_champs_by_district[event.district_key] = event
elif event.event_type_enum == EventType.CMP_FINALS:
champ_events.append(event)
# Build district cmp division <-> parent associations based on district
# Build cmp division <-> parent associations based on date
for event in events:
parent_event = None
if event.event_type_enum == EventType.DISTRICT_CMP_DIVISION:
parent_event = district_champs_by_district.get(event.district_key)
elif event.event_type_enum == EventType.CMP_DIVISION:
for parent_event in champ_events:
if abs(parent_event.end_date - event.end_date) < datetime.timedelta(days=1):
break
else:
parent_event = None
else:
continue
if parent_event is None:
continue
parent_event.divisions = sorted(parent_event.divisions + [event.key])
event.parent_event = parent_event.key
return events, list(districts.values())
def parse(self, response):
events = []
districts = {}
cmp_hack_sitevar = Sitevar.get_or_insert('cmp_registration_hacks')
store_cmp_division = cmp_hack_sitevar.contents.get('should_store_divisions', True) \
if cmp_hack_sitevar else True
einstein_name = cmp_hack_sitevar.contents.get('einstein_name', self.EINSTEIN_NAME_DEFAULT) \
if cmp_hack_sitevar else self.EINSTEIN_NAME_DEFAULT
einstein_short_name = cmp_hack_sitevar.contents.get('einstein_short_name', self.EINSTEIN_SHORT_NAME_DEFAULT) \
if cmp_hack_sitevar else self.EINSTEIN_SHORT_NAME_DEFAULT
change_einstein_dates = cmp_hack_sitevar.contents.get('should_change_einstein_dates', False) \
if cmp_hack_sitevar else False
for event in response['Events']:
code = event['code'].lower()
event_type = EventType.PRESEASON if code == 'week0' else self.EVENT_TYPES.get(event['type'].lower(), None)
if event_type is None:
logging.warn("Event type '{}' not recognized!".format(event['type']))
continue
name = event['name']
short_name = EventHelper.getShortName(name)
district_enum = EventHelper.parseDistrictName(event['districtCode'].lower()) if event['districtCode'] else DistrictType.NO_DISTRICT
district_key = District.renderKeyName(self.season, event['districtCode'].lower()) if event['districtCode'] else None
venue = event['venue']
city = event['city']
state_prov = event['stateprov']
country = event['country']
start = datetime.datetime.strptime(event['dateStart'], self.DATE_FORMAT_STR)
end = datetime.datetime.strptime(event['dateEnd'], self.DATE_FORMAT_STR)
website = event.get('website')
# TODO read timezone from API
# Special cases for champs
if code in self.EVENT_CODE_EXCEPTIONS:
code, short_name = self.EVENT_CODE_EXCEPTIONS[code]
# FIRST indicates CMP registration before divisions are assigned by adding all teams
# to Einstein. We will hack around that by not storing divisions and renaming
# Einstein to simply "Championship" when certain sitevar flags are set
if code in self.EINSTEIN_CODES:
name = short_name.format(einstein_name)
short_name = short_name.format(einstein_short_name)
if change_einstein_dates:
# Set to beginning of last day
start = end.replace(hour=0, minute=0, second=0, microsecond=0)
else: # Divisions
name = '{} Division'.format(short_name)
# Allow skipping storing CMP divisions before they're announced
if not store_cmp_division:
continue
events.append(Event(
id="{}{}".format(self.season, code),
name=name,
short_name=short_name,
event_short=code,
event_type_enum=event_type,
official=True,
start_date=start,
end_date=end,
venue=venue,
city=city,
state_prov=state_prov,
country=country,
venue_address=None, # Even though FRC API provides address, ElasticSearch is more detailed
year=self.season,
event_district_enum=district_enum,
district_key=ndb.Key(District, district_key) if district_key else None,
website=website,
))
# Build District Model
if district_key and district_key not in districts:
districts[district_key] = District(
id=district_key,
year=self.season,
abbreviation=event['districtCode'].lower(),
)
return events, list(districts.values())
def parse(self, response):
events = []
districts = {}
cmp_hack_sitevar = Sitevar.get_or_insert('cmp_registration_hacks')
divisions_to_skip = cmp_hack_sitevar.contents.get('divisions_to_skip', []) \
if cmp_hack_sitevar else []
event_name_override = cmp_hack_sitevar.contents.get('event_name_override', []) \
if cmp_hack_sitevar else []
events_to_change_dates = cmp_hack_sitevar.contents.get('set_start_to_last_day', []) \
if cmp_hack_sitevar else []
for event in response['Events']:
code = event['code'].lower()
event_type = EventType.PRESEASON if code == 'week0' else self.EVENT_TYPES.get(event['type'].lower(), None)
if event_type is None and not self.event_short:
logging.warn("Event type '{}' not recognized!".format(event['type']))
continue
name = event['name']
short_name = EventHelper.getShortName(name, district_code=event['districtCode'])
district_enum = EventHelper.parseDistrictName(event['districtCode'].lower()) if event['districtCode'] else DistrictType.NO_DISTRICT
district_key = District.renderKeyName(self.season, event['districtCode'].lower()) if event['districtCode'] else None
venue = event['venue']
city = event['city']
state_prov = event['stateprov']
country = event['country']
start = datetime.datetime.strptime(event['dateStart'], self.DATE_FORMAT_STR)
end = datetime.datetime.strptime(event['dateEnd'], self.DATE_FORMAT_STR)
website = event.get('website')
# TODO read timezone from API
# Special cases for district championship divisions
if event_type == EventType.DISTRICT_CMP_DIVISION:
split_name = name.split('-')
short_name = '{} - {}'.format(
''.join(item[0].upper() for item in split_name[0].split()),
split_name[-1].replace('Division', '').strip())
# Special cases for champs
if code in self.EVENT_CODE_EXCEPTIONS:
code, short_name = self.EVENT_CODE_EXCEPTIONS[code]
# FIRST indicates CMP registration before divisions are assigned by adding all teams
# to Einstein. We will hack around that by not storing divisions and renaming
# Einstein to simply "Championship" when certain sitevar flags are set
if code in self.EINSTEIN_CODES:
override = [item for item in event_name_override if item['event'] == "{}{}".format(self.season, code)]
if override:
name = short_name.format(override[0]['name'])
short_name = short_name.format(override[0]['short_name'])
else: # Divisions
name = '{} Division'.format(short_name)
elif self.event_short:
code = self.event_short
event_key = "{}{}".format(self.season, code)
if event_key in divisions_to_skip:
continue
# Allow an overriding the start date to be the beginning of the last day
if event_key in events_to_change_dates:
start = end.replace(hour=0, minute=0, second=0, microsecond=0)
events.append(Event(
id=event_key,
name=name,
short_name=short_name,
event_short=code,
event_type_enum=event_type,
official=True,
start_date=start,
end_date=end,
venue=venue,
city=city,
state_prov=state_prov,
country=country,
venue_address=None, # Even though FRC API provides address, ElasticSearch is more detailed
year=self.season,
event_district_enum=district_enum,
district_key=ndb.Key(District, district_key) if district_key else None,
website=website,
))
# Build District Model
if district_key and district_key not in districts:
districts[district_key] = District(
id=district_key,
year=self.season,
abbreviation=event['districtCode'].lower(),
)
return events, list(districts.values())
