def testTwitchUrl(self):
res = WebcastParser.webcast_dict_from_url("http://twitch.tv/frcgamesense")
self.assertIsNotNone(res)
self.assertEqual(res['type'], 'twitch')
self.assertEqual(res['channel'], 'frcgamesense')
unknown = WebcastParser.webcast_dict_from_url("http://twitch.tv/")
self.assertIsNone(unknown)
def testTwitchUrl(self):
res = WebcastParser.webcast_dict_from_url("http://twitch.tv/frcgamesense")
self.assertIsNotNone(res)
self.assertEqual(res['type'], 'twitch')
self.assertEqual(res['channel'], 'frcgamesense')
unknown = WebcastParser.webcast_dict_from_url("http://twitch.tv/")
self.assertIsNone(unknown)
def testUstream(self):
res = WebcastParser.webcast_dict_from_url('http://www.ustream.tv/decoraheagles')
self.assertIsNotNone(res)
self.assertEqual(res['type'], 'ustream')
self.assertEqual(res['channel'], '3064708')
bad = WebcastParser.webcast_dict_from_url('http://ustream.tv/')
self.assertIsNone(bad)
def testUstream(self):
res = WebcastParser.webcast_dict_from_url(
'http://www.ustream.tv/decoraheagles')
self.assertIsNotNone(res)
self.assertEqual(res['type'], 'ustream')
self.assertEqual(res['channel'], '3064708')
bad = WebcastParser.webcast_dict_from_url('http://ustream.tv/')
self.assertIsNone(bad)
def testYouTubeUrl(self):
yt_long = WebcastParser.webcast_dict_from_url("http://www.youtube.com/watch?v=I-IrVbsl_K8")
self.assertIsNotNone(yt_long)
self.assertEqual(yt_long['type'], 'youtube')
self.assertEqual(yt_long['channel'], 'I-IrVbsl_K8')
yt_short = WebcastParser.webcast_dict_from_url("http://youtu.be/I-IrVbsl_K8")
self.assertIsNotNone(yt_short)
self.assertEqual(yt_short['type'], 'youtube')
self.assertEqual(yt_short['channel'], 'I-IrVbsl_K8')
bad_long = WebcastParser.webcast_dict_from_url('"http://www.youtube.com/')
self.assertIsNone(bad_long)
bad_short = WebcastParser.webcast_dict_from_url("http://youtu.be/")
self.assertIsNone(bad_short)
def testYouTubeUrl(self):
yt_long = WebcastParser.webcast_dict_from_url("http://www.youtube.com/watch?v=I-IrVbsl_K8")
self.assertIsNotNone(yt_long)
self.assertEqual(yt_long['type'], 'youtube')
self.assertEqual(yt_long['channel'], 'I-IrVbsl_K8')
yt_short = WebcastParser.webcast_dict_from_url("http://youtu.be/I-IrVbsl_K8")
self.assertIsNotNone(yt_short)
self.assertEqual(yt_short['type'], 'youtube')
self.assertEqual(yt_short['channel'], 'I-IrVbsl_K8')
bad_long = WebcastParser.webcast_dict_from_url('"http://www.youtube.com/')
self.assertIsNone(bad_long)
bad_short = WebcastParser.webcast_dict_from_url("http://youtu.be/")
self.assertIsNone(bad_short)
def createEventWebcastSuggestion(cls, author_account_key, webcast_url, event_key):
"""Create a Event Webcast Suggestion. Returns status string"""
webcast_url = webcast_url.strip()
if not webcast_url.startswith('http://') and not webcast_url.startswith('https://'):
webcast_url = 'http://' + webcast_url
try:
webcast_dict = WebcastParser.webcast_dict_from_url(webcast_url)
except Exception, e:
logging.exception(e)
webcast_dict = None
def createEventWebcastSuggestion(cls, author_account_key, webcast_url,
event_key):
"""Create a Event Webcast Suggestion. Returns status string"""
webcast_url = webcast_url.strip()
if not webcast_url.startswith(
'http://') and not webcast_url.startswith('https://'):
webcast_url = 'http://' + webcast_url
try:
webcast_dict = WebcastParser.webcast_dict_from_url(webcast_url)
except Exception, e:
logging.exception(e)
webcast_dict = None
def createEventWebcastSuggestion(cls, author_account_key, webcast_url, webcast_date, event_key):
"""Create a Event Webcast Suggestion. Returns status string"""
webcast_url = WebsiteHelper.format_url(webcast_url)
webcast_date = webcast_date.strip()
if webcast_date:
try:
datetime.strptime(webcast_date, "%Y-%m-%d")
except ValueError:
return 'invalid_date'
else:
webcast_date = None
try:
webcast_dict = WebcastParser.webcast_dict_from_url(webcast_url)
except Exception, e:
logging.exception(e)
webcast_dict = None
def createEventWebcastSuggestion(cls, author_account_key, webcast_url, webcast_date, event_key):
"""Create a Event Webcast Suggestion. Returns status string"""
webcast_url = WebsiteHelper.format_url(webcast_url)
webcast_date = webcast_date.strip()
if webcast_date:
try:
datetime.strptime(webcast_date, "%Y-%m-%d")
except ValueError:
return 'invalid_date'
else:
webcast_date = None
try:
webcast_dict = WebcastParser.webcast_dict_from_url(webcast_url)
except Exception, e:
logging.exception(e)
webcast_dict = None
def createEventWebcastSuggestion(cls, author_account_key, webcast_url, webcast_date, event_key):
"""Create a Event Webcast Suggestion. Returns status string"""
webcast_url = webcast_url.strip()
if not webcast_url.startswith('http://') and not webcast_url.startswith('https://'):
webcast_url = 'http://' + webcast_url
webcast_date = webcast_date.strip()
if webcast_date:
try:
datetime.strptime(webcast_date, "%Y-%m-%d")
except ValueError:
return 'invalid_date'
else:
webcast_date = None
try:
webcast_dict = WebcastParser.webcast_dict_from_url(webcast_url)
except Exception, e:
logging.exception(e)
webcast_dict = None
def createEventWebcastSuggestion(cls, author_account_key, webcast_url,
webcast_date, event_key):
"""Create a Event Webcast Suggestion. Returns status string"""
webcast_url = webcast_url.strip()
if not webcast_url.startswith(
'http://') and not webcast_url.startswith('https://'):
webcast_url = 'http://' + webcast_url
webcast_date = webcast_date.strip()
if webcast_date:
try:
datetime.strptime(webcast_date, "%Y-%m-%d")
except ValueError:
return 'invalid_date'
else:
webcast_date = None
try:
webcast_dict = WebcastParser.webcast_dict_from_url(webcast_url)
except Exception, e:
logging.exception(e)
webcast_dict = None
def testUnknownUrl(self):
bad = WebcastParser.webcast_dict_from_url("http://mywebsite.somewebcast")
self.assertIsNone(bad)
def parse(self, response):
events = []
districts = {}
cmp_hack_sitevar = Sitevar.get_or_insert('cmp_registration_hacks')
divisions_to_skip = cmp_hack_sitevar.contents.get('divisions_to_skip', []) \
if cmp_hack_sitevar else []
event_name_override = cmp_hack_sitevar.contents.get('event_name_override', []) \
if cmp_hack_sitevar else []
events_to_change_dates = cmp_hack_sitevar.contents.get('set_start_to_last_day', []) \
if cmp_hack_sitevar else []
for event in response['Events']:
code = event['code'].lower()
api_event_type = event['type'].lower()
event_type = EventType.PRESEASON if code == 'week0' else self.EVENT_TYPES.get(api_event_type, None)
if event_type is None and not self.event_short:
logging.warn("Event type '{}' not recognized!".format(api_event_type))
continue
# Some event types should be marked as unofficial, so sync is disabled
official = True
if api_event_type in self.NON_OFFICIAL_EVENT_TYPES:
official = False
name = event['name']
short_name = EventHelper.getShortName(name, district_code=event['districtCode'])
district_enum = EventHelper.parseDistrictName(event['districtCode'].lower()) if event['districtCode'] else DistrictType.NO_DISTRICT
district_key = District.renderKeyName(self.season, event['districtCode'].lower()) if event['districtCode'] else None
venue = event['venue']
city = event['city']
state_prov = event['stateprov']
country = event['country']
start = datetime.datetime.strptime(event['dateStart'], self.DATE_FORMAT_STR)
end = datetime.datetime.strptime(event['dateEnd'], self.DATE_FORMAT_STR)
website = event.get('website')
webcasts = [WebcastParser.webcast_dict_from_url(url) for url in event.get('webcasts', [])]
# TODO read timezone from API
# Special cases for district championship divisions
if event_type == EventType.DISTRICT_CMP_DIVISION:
split_name = name.split('-')
short_name = '{} - {}'.format(
''.join(item[0].upper() for item in split_name[0].split()),
split_name[-1].replace('Division', '').strip())
# Special cases for champs
if code in self.EVENT_CODE_EXCEPTIONS:
code, short_name = self.EVENT_CODE_EXCEPTIONS[code]
# FIRST indicates CMP registration before divisions are assigned by adding all teams
# to Einstein. We will hack around that by not storing divisions and renaming
# Einstein to simply "Championship" when certain sitevar flags are set
if code in self.EINSTEIN_CODES:
override = [item for item in event_name_override if item['event'] == "{}{}".format(self.season, code)]
if override:
name = short_name.format(override[0]['name'])
short_name = short_name.format(override[0]['short_name'])
else: # Divisions
name = '{} Division'.format(short_name)
elif self.event_short:
code = self.event_short
event_key = "{}{}".format(self.season, code)
if event_key in divisions_to_skip:
continue
# Allow an overriding the start date to be the beginning of the last day
if event_key in events_to_change_dates:
start = end.replace(hour=0, minute=0, second=0, microsecond=0)
events.append(Event(
id=event_key,
name=name,
short_name=short_name,
event_short=code,
event_type_enum=event_type,
official=official,
start_date=start,
end_date=end,
venue=venue,
city=city,
state_prov=state_prov,
country=country,
venue_address=None, # Even though FRC API provides address, ElasticSearch is more detailed
year=self.season,
event_district_enum=district_enum,
district_key=ndb.Key(District, district_key) if district_key else None,
website=website,
webcast_json=json.dumps(webcasts) if webcasts else None,
))
# Build District Model
if district_key and district_key not in districts:
districts[district_key] = District(
id=district_key,
year=self.season,
abbreviation=event['districtCode'].lower(),
)
# Prep for division <-> parent associations
district_champs_by_district = {}
champ_events = []
for event in events:
if event.event_type_enum == EventType.DISTRICT_CMP:
district_champs_by_district[event.district_key] = event
elif event.event_type_enum == EventType.CMP_FINALS:
champ_events.append(event)
# Build district cmp division <-> parent associations based on district
# Build cmp division <-> parent associations based on date
for event in events:
parent_event = None
if event.event_type_enum == EventType.DISTRICT_CMP_DIVISION:
parent_event = district_champs_by_district.get(event.district_key)
elif event.event_type_enum == EventType.CMP_DIVISION:
for parent_event in champ_events:
if abs(parent_event.end_date - event.end_date) < datetime.timedelta(days=1):
break
else:
parent_event = None
else:
continue
if parent_event is None:
continue
parent_event.divisions = sorted(parent_event.divisions + [event.key])
event.parent_event = parent_event.key
return events, list(districts.values())
def parse(self, response):
events = []
districts = {}
cmp_hack_sitevar = Sitevar.get_or_insert('cmp_registration_hacks')
divisions_to_skip = cmp_hack_sitevar.contents.get('divisions_to_skip', []) \
if cmp_hack_sitevar else []
event_name_override = cmp_hack_sitevar.contents.get('event_name_override', []) \
if cmp_hack_sitevar else []
events_to_change_dates = cmp_hack_sitevar.contents.get('set_start_to_last_day', []) \
if cmp_hack_sitevar else []
for event in response['Events']:
code = event['code'].lower()
event_type = EventType.PRESEASON if code == 'week0' else self.EVENT_TYPES.get(event['type'].lower(), None)
if event_type is None and not self.event_short:
logging.warn("Event type '{}' not recognized!".format(event['type']))
continue
name = event['name']
short_name = EventHelper.getShortName(name, district_code=event['districtCode'])
district_enum = EventHelper.parseDistrictName(event['districtCode'].lower()) if event['districtCode'] else DistrictType.NO_DISTRICT
district_key = District.renderKeyName(self.season, event['districtCode'].lower()) if event['districtCode'] else None
venue = event['venue']
city = event['city']
state_prov = event['stateprov']
country = event['country']
start = datetime.datetime.strptime(event['dateStart'], self.DATE_FORMAT_STR)
end = datetime.datetime.strptime(event['dateEnd'], self.DATE_FORMAT_STR)
website = event.get('website')
webcasts = [WebcastParser.webcast_dict_from_url(url) for url in event.get('webcasts', [])]
# TODO read timezone from API
# Special cases for district championship divisions
if event_type == EventType.DISTRICT_CMP_DIVISION:
split_name = name.split('-')
short_name = '{} - {}'.format(
''.join(item[0].upper() for item in split_name[0].split()),
split_name[-1].replace('Division', '').strip())
# Special cases for champs
if code in self.EVENT_CODE_EXCEPTIONS:
code, short_name = self.EVENT_CODE_EXCEPTIONS[code]
# FIRST indicates CMP registration before divisions are assigned by adding all teams
# to Einstein. We will hack around that by not storing divisions and renaming
# Einstein to simply "Championship" when certain sitevar flags are set
if code in self.EINSTEIN_CODES:
override = [item for item in event_name_override if item['event'] == "{}{}".format(self.season, code)]
if override:
name = short_name.format(override[0]['name'])
short_name = short_name.format(override[0]['short_name'])
else: # Divisions
name = '{} Division'.format(short_name)
elif self.event_short:
code = self.event_short
event_key = "{}{}".format(self.season, code)
if event_key in divisions_to_skip:
continue
# Allow an overriding the start date to be the beginning of the last day
if event_key in events_to_change_dates:
start = end.replace(hour=0, minute=0, second=0, microsecond=0)
events.append(Event(
id=event_key,
name=name,
short_name=short_name,
event_short=code,
event_type_enum=event_type,
official=True,
start_date=start,
end_date=end,
venue=venue,
city=city,
state_prov=state_prov,
country=country,
venue_address=None, # Even though FRC API provides address, ElasticSearch is more detailed
year=self.season,
event_district_enum=district_enum,
district_key=ndb.Key(District, district_key) if district_key else None,
website=website,
webcast_json=json.dumps(webcasts) if webcasts else None,
))
# Build District Model
if district_key and district_key not in districts:
districts[district_key] = District(
id=district_key,
year=self.season,
abbreviation=event['districtCode'].lower(),
)
# Prep for division <-> parent associations
district_champs_by_district = {}
champ_events = []
for event in events:
if event.event_type_enum == EventType.DISTRICT_CMP:
district_champs_by_district[event.district_key] = event
elif event.event_type_enum == EventType.CMP_FINALS:
champ_events.append(event)
# Build district cmp division <-> parent associations based on district
# Build cmp division <-> parent associations based on date
for event in events:
parent_event = None
if event.event_type_enum == EventType.DISTRICT_CMP_DIVISION:
parent_event = district_champs_by_district.get(event.district_key)
elif event.event_type_enum == EventType.CMP_DIVISION:
for parent_event in champ_events:
if abs(parent_event.end_date - event.end_date) < datetime.timedelta(days=1):
break
else:
parent_event = None
else:
continue
if parent_event is None:
continue
parent_event.divisions = sorted(parent_event.divisions + [event.key])
event.parent_event = parent_event.key
return events, list(districts.values())
def _process_request(self, request, event_key):
try:
event_info = json.loads(request.body)
except Exception:
self._errors = json.dumps({"Error": "Invalid json. Check input."})
self.abort(400)
if not isinstance(event_info, dict) or not event_info:
self._errors = json.dumps({"Error": "Invalid json. Check input."})
self.abort(400)
do_team_remap = False
for field, value in event_info.iteritems():
if field not in self.ALLOWED_EVENT_PARAMS:
continue
if field == "webcasts":
# Do special processing here because webcasts are janky
if not isinstance(value, list):
self._errors = json.dumps(
{"Error": "Invalid json. Check input"}
)
self.abort(400)
return
webcast_list = []
for webcast in value:
if not isinstance(webcast, dict):
self._errors = json.dumps(
{"Error": "Invalid json. Check input"}
)
self.abort(400)
return
if 'url' in webcast:
webcast_list.append(
WebcastParser.webcast_dict_from_url(webcast['url'])
)
elif 'type' in webcast and 'channel' in webcast:
webcast_list.append(webcast)
webcast_list = [w for w in webcast_list if w is not None]
EventWebcastAdder.add_webcast(
self.event,
webcast_list,
False, # Don't createOrUpdate yet
)
elif field == "remap_teams":
# Validate remap_teams
if not isinstance(value, dict):
raise ParserInputException("Invalid reamap_teams. Check input")
for temp_team, remapped_team in value.items():
temp_match = re.match(r'frc\d+', str(temp_team))
remapped_match = re.match(r'frc\d+[B-Z]?', str(remapped_team))
if not temp_match or (temp_match and (temp_match.group(0) != str(temp_team))):
raise ParserInputException("Bad team: '{}'. Must follow format 'frcXXX'.".format(temp_team))
if not remapped_match or (remapped_match and (remapped_match.group(0) != str(remapped_team))):
raise ParserInputException("Bad team: '{}'. Must follow format 'frcXXX' or 'frcXXX[B-Z]'.".format(remapped_team))
do_team_remap = True
setattr(self.event, field, value)
else:
try:
if field == "first_event_code":
self.event.official = value is not None
field = "first_code" # Internal property is different
setattr(self.event, field, value)
except Exception, e:
self._errors({
"Error": "Unable to set event field",
"Message": str(e)
})
self.abort(400)
def testUnknownUrl(self):
bad = WebcastParser.webcast_dict_from_url("http://mywebsite.somewebcast")
self.assertIsNone(bad)
def _process_request(self, request, event_key):
try:
event_info = json.loads(request.body)
except Exception:
self._errors = json.dumps({"Error": "Invalid json. Check input."})
self.abort(400)
if not isinstance(event_info, dict) or not event_info:
self._errors = json.dumps({"Error": "Invalid json. Check input."})
self.abort(400)
event = Event.get_by_id(event_key)
if not event:
self._errors = json.dumps(
{"Error": "Event {} not found".format(event_key)})
self.abort(404)
do_team_remap = False
for field, value in event_info.iteritems():
if field not in self.ALLOWED_EVENT_PARAMS:
continue
if field == "webcasts":
# Do special processing here because webcasts are janky
if not isinstance(value, list):
self._errors = json.dumps(
{"Error": "Invalid json. Check input"})
self.abort(400)
return
webcast_list = []
for webcast in value:
if not isinstance(webcast, dict):
self._errors = json.dumps(
{"Error": "Invalid json. Check input"})
self.abort(400)
return
if 'url' in webcast:
webcast_list.append(
WebcastParser.webcast_dict_from_url(
webcast['url']))
elif 'type' in webcast and 'channel' in webcast:
webcast_list.append(webcast)
webcast_list = [w for w in webcast_list if w is not None]
EventWebcastAdder.add_webcast(
event,
webcast_list,
False, # Don't createOrUpdate yet
)
elif field == "remap_teams":
# Validate remap_teams
if not isinstance(value, dict):
raise ParserInputException(
"Invalid reamap_teams. Check input")
for temp_team, remapped_team in value.items():
temp_match = re.match(r'frc\d+', str(temp_team))
remapped_match = re.match(r'frc\d+[B-Z]?',
str(remapped_team))
if not temp_match or (
temp_match and
(temp_match.group(0) != str(temp_team))):
raise ParserInputException(
"Bad team: '{}'. Must follow format 'frcXXX'.".
format(temp_team))
if not remapped_match or (
remapped_match and
(remapped_match.group(0) != str(remapped_team))):
raise ParserInputException(
"Bad team: '{}'. Must follow format 'frcXXX' or 'frcXXX[B-Z]'."
.format(remapped_team))
do_team_remap = True
setattr(event, field, value)
else:
try:
if field == "first_event_code":
event.official = value is not None
field = "first_code" # Internal property is different
setattr(event, field, value)
except Exception, e:
self._errors({
"Error": "Unable to set event field",
"Message": str(e)
})
self.abort(400)
