def testTwitchUrl(self): res = WebcastParser.webcast_dict_from_url("http://twitch.tv/frcgamesense") self.assertIsNotNone(res) self.assertEqual(res['type'], 'twitch') self.assertEqual(res['channel'], 'frcgamesense') unknown = WebcastParser.webcast_dict_from_url("http://twitch.tv/") self.assertIsNone(unknown)
def testUstream(self): res = WebcastParser.webcast_dict_from_url('http://www.ustream.tv/decoraheagles') self.assertIsNotNone(res) self.assertEqual(res['type'], 'ustream') self.assertEqual(res['channel'], '3064708') bad = WebcastParser.webcast_dict_from_url('http://ustream.tv/') self.assertIsNone(bad)
def testUstream(self): res = WebcastParser.webcast_dict_from_url( 'http://www.ustream.tv/decoraheagles') self.assertIsNotNone(res) self.assertEqual(res['type'], 'ustream') self.assertEqual(res['channel'], '3064708') bad = WebcastParser.webcast_dict_from_url('http://ustream.tv/') self.assertIsNone(bad)
def testYouTubeUrl(self): yt_long = WebcastParser.webcast_dict_from_url("http://www.youtube.com/watch?v=I-IrVbsl_K8") self.assertIsNotNone(yt_long) self.assertEqual(yt_long['type'], 'youtube') self.assertEqual(yt_long['channel'], 'I-IrVbsl_K8') yt_short = WebcastParser.webcast_dict_from_url("http://youtu.be/I-IrVbsl_K8") self.assertIsNotNone(yt_short) self.assertEqual(yt_short['type'], 'youtube') self.assertEqual(yt_short['channel'], 'I-IrVbsl_K8') bad_long = WebcastParser.webcast_dict_from_url('"http://www.youtube.com/') self.assertIsNone(bad_long) bad_short = WebcastParser.webcast_dict_from_url("http://youtu.be/") self.assertIsNone(bad_short)
def createEventWebcastSuggestion(cls, author_account_key, webcast_url, event_key): """Create a Event Webcast Suggestion. Returns status string""" webcast_url = webcast_url.strip() if not webcast_url.startswith('http://') and not webcast_url.startswith('https://'): webcast_url = 'http://' + webcast_url try: webcast_dict = WebcastParser.webcast_dict_from_url(webcast_url) except Exception, e: logging.exception(e) webcast_dict = None
def createEventWebcastSuggestion(cls, author_account_key, webcast_url, event_key): """Create a Event Webcast Suggestion. Returns status string""" webcast_url = webcast_url.strip() if not webcast_url.startswith( 'http://') and not webcast_url.startswith('https://'): webcast_url = 'http://' + webcast_url try: webcast_dict = WebcastParser.webcast_dict_from_url(webcast_url) except Exception, e: logging.exception(e) webcast_dict = None
def createEventWebcastSuggestion(cls, author_account_key, webcast_url, webcast_date, event_key): """Create a Event Webcast Suggestion. Returns status string""" webcast_url = WebsiteHelper.format_url(webcast_url) webcast_date = webcast_date.strip() if webcast_date: try: datetime.strptime(webcast_date, "%Y-%m-%d") except ValueError: return 'invalid_date' else: webcast_date = None try: webcast_dict = WebcastParser.webcast_dict_from_url(webcast_url) except Exception, e: logging.exception(e) webcast_dict = None
def createEventWebcastSuggestion(cls, author_account_key, webcast_url, webcast_date, event_key): """Create a Event Webcast Suggestion. Returns status string""" webcast_url = webcast_url.strip() if not webcast_url.startswith('http://') and not webcast_url.startswith('https://'): webcast_url = 'http://' + webcast_url webcast_date = webcast_date.strip() if webcast_date: try: datetime.strptime(webcast_date, "%Y-%m-%d") except ValueError: return 'invalid_date' else: webcast_date = None try: webcast_dict = WebcastParser.webcast_dict_from_url(webcast_url) except Exception, e: logging.exception(e) webcast_dict = None
def createEventWebcastSuggestion(cls, author_account_key, webcast_url, webcast_date, event_key): """Create a Event Webcast Suggestion. Returns status string""" webcast_url = webcast_url.strip() if not webcast_url.startswith( 'http://') and not webcast_url.startswith('https://'): webcast_url = 'http://' + webcast_url webcast_date = webcast_date.strip() if webcast_date: try: datetime.strptime(webcast_date, "%Y-%m-%d") except ValueError: return 'invalid_date' else: webcast_date = None try: webcast_dict = WebcastParser.webcast_dict_from_url(webcast_url) except Exception, e: logging.exception(e) webcast_dict = None
def testUnknownUrl(self): bad = WebcastParser.webcast_dict_from_url("http://mywebsite.somewebcast") self.assertIsNone(bad)
def parse(self, response): events = [] districts = {} cmp_hack_sitevar = Sitevar.get_or_insert('cmp_registration_hacks') divisions_to_skip = cmp_hack_sitevar.contents.get('divisions_to_skip', []) \ if cmp_hack_sitevar else [] event_name_override = cmp_hack_sitevar.contents.get('event_name_override', []) \ if cmp_hack_sitevar else [] events_to_change_dates = cmp_hack_sitevar.contents.get('set_start_to_last_day', []) \ if cmp_hack_sitevar else [] for event in response['Events']: code = event['code'].lower() api_event_type = event['type'].lower() event_type = EventType.PRESEASON if code == 'week0' else self.EVENT_TYPES.get(api_event_type, None) if event_type is None and not self.event_short: logging.warn("Event type '{}' not recognized!".format(api_event_type)) continue # Some event types should be marked as unofficial, so sync is disabled official = True if api_event_type in self.NON_OFFICIAL_EVENT_TYPES: official = False name = event['name'] short_name = EventHelper.getShortName(name, district_code=event['districtCode']) district_enum = EventHelper.parseDistrictName(event['districtCode'].lower()) if event['districtCode'] else DistrictType.NO_DISTRICT district_key = District.renderKeyName(self.season, event['districtCode'].lower()) if event['districtCode'] else None venue = event['venue'] city = event['city'] state_prov = event['stateprov'] country = event['country'] start = datetime.datetime.strptime(event['dateStart'], self.DATE_FORMAT_STR) end = datetime.datetime.strptime(event['dateEnd'], self.DATE_FORMAT_STR) website = event.get('website') webcasts = [WebcastParser.webcast_dict_from_url(url) for url in event.get('webcasts', [])] # TODO read timezone from API # Special cases for district championship divisions if event_type == EventType.DISTRICT_CMP_DIVISION: split_name = name.split('-') short_name = '{} - {}'.format( ''.join(item[0].upper() for item in split_name[0].split()), split_name[-1].replace('Division', '').strip()) # Special cases for champs if code in self.EVENT_CODE_EXCEPTIONS: code, short_name = self.EVENT_CODE_EXCEPTIONS[code] # FIRST indicates CMP registration before divisions are assigned by adding all teams # to Einstein. We will hack around that by not storing divisions and renaming # Einstein to simply "Championship" when certain sitevar flags are set if code in self.EINSTEIN_CODES: override = [item for item in event_name_override if item['event'] == "{}{}".format(self.season, code)] if override: name = short_name.format(override[0]['name']) short_name = short_name.format(override[0]['short_name']) else: # Divisions name = '{} Division'.format(short_name) elif self.event_short: code = self.event_short event_key = "{}{}".format(self.season, code) if event_key in divisions_to_skip: continue # Allow an overriding the start date to be the beginning of the last day if event_key in events_to_change_dates: start = end.replace(hour=0, minute=0, second=0, microsecond=0) events.append(Event( id=event_key, name=name, short_name=short_name, event_short=code, event_type_enum=event_type, official=official, start_date=start, end_date=end, venue=venue, city=city, state_prov=state_prov, country=country, venue_address=None, # Even though FRC API provides address, ElasticSearch is more detailed year=self.season, event_district_enum=district_enum, district_key=ndb.Key(District, district_key) if district_key else None, website=website, webcast_json=json.dumps(webcasts) if webcasts else None, )) # Build District Model if district_key and district_key not in districts: districts[district_key] = District( id=district_key, year=self.season, abbreviation=event['districtCode'].lower(), ) # Prep for division <-> parent associations district_champs_by_district = {} champ_events = [] for event in events: if event.event_type_enum == EventType.DISTRICT_CMP: district_champs_by_district[event.district_key] = event elif event.event_type_enum == EventType.CMP_FINALS: champ_events.append(event) # Build district cmp division <-> parent associations based on district # Build cmp division <-> parent associations based on date for event in events: parent_event = None if event.event_type_enum == EventType.DISTRICT_CMP_DIVISION: parent_event = district_champs_by_district.get(event.district_key) elif event.event_type_enum == EventType.CMP_DIVISION: for parent_event in champ_events: if abs(parent_event.end_date - event.end_date) < datetime.timedelta(days=1): break else: parent_event = None else: continue if parent_event is None: continue parent_event.divisions = sorted(parent_event.divisions + [event.key]) event.parent_event = parent_event.key return events, list(districts.values())
def parse(self, response): events = [] districts = {} cmp_hack_sitevar = Sitevar.get_or_insert('cmp_registration_hacks') divisions_to_skip = cmp_hack_sitevar.contents.get('divisions_to_skip', []) \ if cmp_hack_sitevar else [] event_name_override = cmp_hack_sitevar.contents.get('event_name_override', []) \ if cmp_hack_sitevar else [] events_to_change_dates = cmp_hack_sitevar.contents.get('set_start_to_last_day', []) \ if cmp_hack_sitevar else [] for event in response['Events']: code = event['code'].lower() event_type = EventType.PRESEASON if code == 'week0' else self.EVENT_TYPES.get(event['type'].lower(), None) if event_type is None and not self.event_short: logging.warn("Event type '{}' not recognized!".format(event['type'])) continue name = event['name'] short_name = EventHelper.getShortName(name, district_code=event['districtCode']) district_enum = EventHelper.parseDistrictName(event['districtCode'].lower()) if event['districtCode'] else DistrictType.NO_DISTRICT district_key = District.renderKeyName(self.season, event['districtCode'].lower()) if event['districtCode'] else None venue = event['venue'] city = event['city'] state_prov = event['stateprov'] country = event['country'] start = datetime.datetime.strptime(event['dateStart'], self.DATE_FORMAT_STR) end = datetime.datetime.strptime(event['dateEnd'], self.DATE_FORMAT_STR) website = event.get('website') webcasts = [WebcastParser.webcast_dict_from_url(url) for url in event.get('webcasts', [])] # TODO read timezone from API # Special cases for district championship divisions if event_type == EventType.DISTRICT_CMP_DIVISION: split_name = name.split('-') short_name = '{} - {}'.format( ''.join(item[0].upper() for item in split_name[0].split()), split_name[-1].replace('Division', '').strip()) # Special cases for champs if code in self.EVENT_CODE_EXCEPTIONS: code, short_name = self.EVENT_CODE_EXCEPTIONS[code] # FIRST indicates CMP registration before divisions are assigned by adding all teams # to Einstein. We will hack around that by not storing divisions and renaming # Einstein to simply "Championship" when certain sitevar flags are set if code in self.EINSTEIN_CODES: override = [item for item in event_name_override if item['event'] == "{}{}".format(self.season, code)] if override: name = short_name.format(override[0]['name']) short_name = short_name.format(override[0]['short_name']) else: # Divisions name = '{} Division'.format(short_name) elif self.event_short: code = self.event_short event_key = "{}{}".format(self.season, code) if event_key in divisions_to_skip: continue # Allow an overriding the start date to be the beginning of the last day if event_key in events_to_change_dates: start = end.replace(hour=0, minute=0, second=0, microsecond=0) events.append(Event( id=event_key, name=name, short_name=short_name, event_short=code, event_type_enum=event_type, official=True, start_date=start, end_date=end, venue=venue, city=city, state_prov=state_prov, country=country, venue_address=None, # Even though FRC API provides address, ElasticSearch is more detailed year=self.season, event_district_enum=district_enum, district_key=ndb.Key(District, district_key) if district_key else None, website=website, webcast_json=json.dumps(webcasts) if webcasts else None, )) # Build District Model if district_key and district_key not in districts: districts[district_key] = District( id=district_key, year=self.season, abbreviation=event['districtCode'].lower(), ) # Prep for division <-> parent associations district_champs_by_district = {} champ_events = [] for event in events: if event.event_type_enum == EventType.DISTRICT_CMP: district_champs_by_district[event.district_key] = event elif event.event_type_enum == EventType.CMP_FINALS: champ_events.append(event) # Build district cmp division <-> parent associations based on district # Build cmp division <-> parent associations based on date for event in events: parent_event = None if event.event_type_enum == EventType.DISTRICT_CMP_DIVISION: parent_event = district_champs_by_district.get(event.district_key) elif event.event_type_enum == EventType.CMP_DIVISION: for parent_event in champ_events: if abs(parent_event.end_date - event.end_date) < datetime.timedelta(days=1): break else: parent_event = None else: continue if parent_event is None: continue parent_event.divisions = sorted(parent_event.divisions + [event.key]) event.parent_event = parent_event.key return events, list(districts.values())
def _process_request(self, request, event_key): try: event_info = json.loads(request.body) except Exception: self._errors = json.dumps({"Error": "Invalid json. Check input."}) self.abort(400) if not isinstance(event_info, dict) or not event_info: self._errors = json.dumps({"Error": "Invalid json. Check input."}) self.abort(400) do_team_remap = False for field, value in event_info.iteritems(): if field not in self.ALLOWED_EVENT_PARAMS: continue if field == "webcasts": # Do special processing here because webcasts are janky if not isinstance(value, list): self._errors = json.dumps( {"Error": "Invalid json. Check input"} ) self.abort(400) return webcast_list = [] for webcast in value: if not isinstance(webcast, dict): self._errors = json.dumps( {"Error": "Invalid json. Check input"} ) self.abort(400) return if 'url' in webcast: webcast_list.append( WebcastParser.webcast_dict_from_url(webcast['url']) ) elif 'type' in webcast and 'channel' in webcast: webcast_list.append(webcast) webcast_list = [w for w in webcast_list if w is not None] EventWebcastAdder.add_webcast( self.event, webcast_list, False, # Don't createOrUpdate yet ) elif field == "remap_teams": # Validate remap_teams if not isinstance(value, dict): raise ParserInputException("Invalid reamap_teams. Check input") for temp_team, remapped_team in value.items(): temp_match = re.match(r'frc\d+', str(temp_team)) remapped_match = re.match(r'frc\d+[B-Z]?', str(remapped_team)) if not temp_match or (temp_match and (temp_match.group(0) != str(temp_team))): raise ParserInputException("Bad team: '{}'. Must follow format 'frcXXX'.".format(temp_team)) if not remapped_match or (remapped_match and (remapped_match.group(0) != str(remapped_team))): raise ParserInputException("Bad team: '{}'. Must follow format 'frcXXX' or 'frcXXX[B-Z]'.".format(remapped_team)) do_team_remap = True setattr(self.event, field, value) else: try: if field == "first_event_code": self.event.official = value is not None field = "first_code" # Internal property is different setattr(self.event, field, value) except Exception, e: self._errors({ "Error": "Unable to set event field", "Message": str(e) }) self.abort(400)
def _process_request(self, request, event_key): try: event_info = json.loads(request.body) except Exception: self._errors = json.dumps({"Error": "Invalid json. Check input."}) self.abort(400) if not isinstance(event_info, dict) or not event_info: self._errors = json.dumps({"Error": "Invalid json. Check input."}) self.abort(400) event = Event.get_by_id(event_key) if not event: self._errors = json.dumps( {"Error": "Event {} not found".format(event_key)}) self.abort(404) do_team_remap = False for field, value in event_info.iteritems(): if field not in self.ALLOWED_EVENT_PARAMS: continue if field == "webcasts": # Do special processing here because webcasts are janky if not isinstance(value, list): self._errors = json.dumps( {"Error": "Invalid json. Check input"}) self.abort(400) return webcast_list = [] for webcast in value: if not isinstance(webcast, dict): self._errors = json.dumps( {"Error": "Invalid json. Check input"}) self.abort(400) return if 'url' in webcast: webcast_list.append( WebcastParser.webcast_dict_from_url( webcast['url'])) elif 'type' in webcast and 'channel' in webcast: webcast_list.append(webcast) webcast_list = [w for w in webcast_list if w is not None] EventWebcastAdder.add_webcast( event, webcast_list, False, # Don't createOrUpdate yet ) elif field == "remap_teams": # Validate remap_teams if not isinstance(value, dict): raise ParserInputException( "Invalid reamap_teams. Check input") for temp_team, remapped_team in value.items(): temp_match = re.match(r'frc\d+', str(temp_team)) remapped_match = re.match(r'frc\d+[B-Z]?', str(remapped_team)) if not temp_match or ( temp_match and (temp_match.group(0) != str(temp_team))): raise ParserInputException( "Bad team: '{}'. Must follow format 'frcXXX'.". format(temp_team)) if not remapped_match or ( remapped_match and (remapped_match.group(0) != str(remapped_team))): raise ParserInputException( "Bad team: '{}'. Must follow format 'frcXXX' or 'frcXXX[B-Z]'." .format(remapped_team)) do_team_remap = True setattr(event, field, value) else: try: if field == "first_event_code": event.official = value is not None field = "first_code" # Internal property is different setattr(event, field, value) except Exception, e: self._errors({ "Error": "Unable to set event field", "Message": str(e) }) self.abort(400)