def testTwitchUrl(self):
        res = WebcastParser.webcast_dict_from_url("http://twitch.tv/frcgamesense")
        self.assertIsNotNone(res)
        self.assertEqual(res['type'], 'twitch')
        self.assertEqual(res['channel'], 'frcgamesense')

        unknown = WebcastParser.webcast_dict_from_url("http://twitch.tv/")
        self.assertIsNone(unknown)
    def testTwitchUrl(self):
        res = WebcastParser.webcast_dict_from_url("http://twitch.tv/frcgamesense")
        self.assertIsNotNone(res)
        self.assertEqual(res['type'], 'twitch')
        self.assertEqual(res['channel'], 'frcgamesense')

        unknown = WebcastParser.webcast_dict_from_url("http://twitch.tv/")
        self.assertIsNone(unknown)
    def testUstream(self):
        res = WebcastParser.webcast_dict_from_url('http://www.ustream.tv/decoraheagles')
        self.assertIsNotNone(res)
        self.assertEqual(res['type'], 'ustream')
        self.assertEqual(res['channel'], '3064708')

        bad = WebcastParser.webcast_dict_from_url('http://ustream.tv/')
        self.assertIsNone(bad)
Exemple #4
0
    def testUstream(self):
        res = WebcastParser.webcast_dict_from_url(
            'http://www.ustream.tv/decoraheagles')
        self.assertIsNotNone(res)
        self.assertEqual(res['type'], 'ustream')
        self.assertEqual(res['channel'], '3064708')

        bad = WebcastParser.webcast_dict_from_url('http://ustream.tv/')
        self.assertIsNone(bad)
    def testYouTubeUrl(self):
        yt_long = WebcastParser.webcast_dict_from_url("http://www.youtube.com/watch?v=I-IrVbsl_K8")
        self.assertIsNotNone(yt_long)
        self.assertEqual(yt_long['type'], 'youtube')
        self.assertEqual(yt_long['channel'], 'I-IrVbsl_K8')

        yt_short = WebcastParser.webcast_dict_from_url("http://youtu.be/I-IrVbsl_K8")
        self.assertIsNotNone(yt_short)
        self.assertEqual(yt_short['type'], 'youtube')
        self.assertEqual(yt_short['channel'], 'I-IrVbsl_K8')

        bad_long = WebcastParser.webcast_dict_from_url('"http://www.youtube.com/')
        self.assertIsNone(bad_long)

        bad_short = WebcastParser.webcast_dict_from_url("http://youtu.be/")
        self.assertIsNone(bad_short)
    def testYouTubeUrl(self):
        yt_long = WebcastParser.webcast_dict_from_url("http://www.youtube.com/watch?v=I-IrVbsl_K8")
        self.assertIsNotNone(yt_long)
        self.assertEqual(yt_long['type'], 'youtube')
        self.assertEqual(yt_long['channel'], 'I-IrVbsl_K8')

        yt_short = WebcastParser.webcast_dict_from_url("http://youtu.be/I-IrVbsl_K8")
        self.assertIsNotNone(yt_short)
        self.assertEqual(yt_short['type'], 'youtube')
        self.assertEqual(yt_short['channel'], 'I-IrVbsl_K8')

        bad_long = WebcastParser.webcast_dict_from_url('"http://www.youtube.com/')
        self.assertIsNone(bad_long)

        bad_short = WebcastParser.webcast_dict_from_url("http://youtu.be/")
        self.assertIsNone(bad_short)
    def createEventWebcastSuggestion(cls, author_account_key, webcast_url, event_key):
        """Create a Event Webcast Suggestion. Returns status string"""

        webcast_url = webcast_url.strip()
        if not webcast_url.startswith('http://') and not webcast_url.startswith('https://'):
            webcast_url = 'http://' + webcast_url

        try:
            webcast_dict = WebcastParser.webcast_dict_from_url(webcast_url)
        except Exception, e:
            logging.exception(e)
            webcast_dict = None
    def createEventWebcastSuggestion(cls, author_account_key, webcast_url,
                                     event_key):
        """Create a Event Webcast Suggestion. Returns status string"""

        webcast_url = webcast_url.strip()
        if not webcast_url.startswith(
                'http://') and not webcast_url.startswith('https://'):
            webcast_url = 'http://' + webcast_url

        try:
            webcast_dict = WebcastParser.webcast_dict_from_url(webcast_url)
        except Exception, e:
            logging.exception(e)
            webcast_dict = None
    def createEventWebcastSuggestion(cls, author_account_key, webcast_url, webcast_date, event_key):
        """Create a Event Webcast Suggestion. Returns status string"""

        webcast_url = WebsiteHelper.format_url(webcast_url)

        webcast_date = webcast_date.strip()
        if webcast_date:
            try:
                datetime.strptime(webcast_date, "%Y-%m-%d")
            except ValueError:
                return 'invalid_date'
        else:
            webcast_date = None

        try:
            webcast_dict = WebcastParser.webcast_dict_from_url(webcast_url)
        except Exception, e:
            logging.exception(e)
            webcast_dict = None
    def createEventWebcastSuggestion(cls, author_account_key, webcast_url, webcast_date, event_key):
        """Create a Event Webcast Suggestion. Returns status string"""

        webcast_url = WebsiteHelper.format_url(webcast_url)

        webcast_date = webcast_date.strip()
        if webcast_date:
            try:
                datetime.strptime(webcast_date, "%Y-%m-%d")
            except ValueError:
                return 'invalid_date'
        else:
            webcast_date = None

        try:
            webcast_dict = WebcastParser.webcast_dict_from_url(webcast_url)
        except Exception, e:
            logging.exception(e)
            webcast_dict = None
    def createEventWebcastSuggestion(cls, author_account_key, webcast_url, webcast_date, event_key):
        """Create a Event Webcast Suggestion. Returns status string"""

        webcast_url = webcast_url.strip()
        if not webcast_url.startswith('http://') and not webcast_url.startswith('https://'):
            webcast_url = 'http://' + webcast_url

        webcast_date = webcast_date.strip()
        if webcast_date:
            try:
                datetime.strptime(webcast_date, "%Y-%m-%d")
            except ValueError:
                return 'invalid_date'
        else:
            webcast_date = None

        try:
            webcast_dict = WebcastParser.webcast_dict_from_url(webcast_url)
        except Exception, e:
            logging.exception(e)
            webcast_dict = None
Exemple #12
0
    def createEventWebcastSuggestion(cls, author_account_key, webcast_url,
                                     webcast_date, event_key):
        """Create a Event Webcast Suggestion. Returns status string"""

        webcast_url = webcast_url.strip()
        if not webcast_url.startswith(
                'http://') and not webcast_url.startswith('https://'):
            webcast_url = 'http://' + webcast_url

        webcast_date = webcast_date.strip()
        if webcast_date:
            try:
                datetime.strptime(webcast_date, "%Y-%m-%d")
            except ValueError:
                return 'invalid_date'
        else:
            webcast_date = None

        try:
            webcast_dict = WebcastParser.webcast_dict_from_url(webcast_url)
        except Exception, e:
            logging.exception(e)
            webcast_dict = None
 def testUnknownUrl(self):
     bad = WebcastParser.webcast_dict_from_url("http://mywebsite.somewebcast")
     self.assertIsNone(bad)
Exemple #14
0
    def parse(self, response):
        events = []
        districts = {}

        cmp_hack_sitevar = Sitevar.get_or_insert('cmp_registration_hacks')
        divisions_to_skip = cmp_hack_sitevar.contents.get('divisions_to_skip', []) \
            if cmp_hack_sitevar else []
        event_name_override = cmp_hack_sitevar.contents.get('event_name_override', []) \
            if cmp_hack_sitevar else []
        events_to_change_dates = cmp_hack_sitevar.contents.get('set_start_to_last_day', []) \
            if cmp_hack_sitevar else []

        for event in response['Events']:
            code = event['code'].lower()

            api_event_type = event['type'].lower()
            event_type = EventType.PRESEASON if code == 'week0' else self.EVENT_TYPES.get(api_event_type, None)
            if event_type is None and not self.event_short:
                logging.warn("Event type '{}' not recognized!".format(api_event_type))
                continue

            # Some event types should be marked as unofficial, so sync is disabled
            official = True
            if api_event_type in self.NON_OFFICIAL_EVENT_TYPES:
                official = False

            name = event['name']
            short_name = EventHelper.getShortName(name, district_code=event['districtCode'])
            district_enum = EventHelper.parseDistrictName(event['districtCode'].lower()) if event['districtCode'] else DistrictType.NO_DISTRICT
            district_key = District.renderKeyName(self.season, event['districtCode'].lower()) if event['districtCode'] else None
            venue = event['venue']
            city = event['city']
            state_prov = event['stateprov']
            country = event['country']
            start = datetime.datetime.strptime(event['dateStart'], self.DATE_FORMAT_STR)
            end = datetime.datetime.strptime(event['dateEnd'], self.DATE_FORMAT_STR)
            website = event.get('website')
            webcasts = [WebcastParser.webcast_dict_from_url(url) for url in event.get('webcasts', [])]

            # TODO read timezone from API

            # Special cases for district championship divisions
            if event_type == EventType.DISTRICT_CMP_DIVISION:
                split_name = name.split('-')
                short_name = '{} - {}'.format(
                    ''.join(item[0].upper() for item in split_name[0].split()),
                    split_name[-1].replace('Division', '').strip())

            # Special cases for champs
            if code in self.EVENT_CODE_EXCEPTIONS:
                code, short_name = self.EVENT_CODE_EXCEPTIONS[code]

                # FIRST indicates CMP registration before divisions are assigned by adding all teams
                # to Einstein. We will hack around that by not storing divisions and renaming
                # Einstein to simply "Championship" when certain sitevar flags are set

                if code in self.EINSTEIN_CODES:
                    override = [item for item in event_name_override if item['event'] == "{}{}".format(self.season, code)]
                    if override:
                        name = short_name.format(override[0]['name'])
                        short_name = short_name.format(override[0]['short_name'])
                else:  # Divisions
                    name = '{} Division'.format(short_name)
            elif self.event_short:
                code = self.event_short

            event_key = "{}{}".format(self.season, code)
            if event_key in divisions_to_skip:
                continue

            # Allow an overriding the start date to be the beginning of the last day
            if event_key in events_to_change_dates:
                start = end.replace(hour=0, minute=0, second=0, microsecond=0)

            events.append(Event(
                id=event_key,
                name=name,
                short_name=short_name,
                event_short=code,
                event_type_enum=event_type,
                official=official,
                start_date=start,
                end_date=end,
                venue=venue,
                city=city,
                state_prov=state_prov,
                country=country,
                venue_address=None,  # Even though FRC API provides address, ElasticSearch is more detailed
                year=self.season,
                event_district_enum=district_enum,
                district_key=ndb.Key(District, district_key) if district_key else None,
                website=website,
                webcast_json=json.dumps(webcasts) if webcasts else None,
            ))

            # Build District Model
            if district_key and district_key not in districts:
                districts[district_key] = District(
                    id=district_key,
                    year=self.season,
                    abbreviation=event['districtCode'].lower(),
                )

        # Prep for division <-> parent associations
        district_champs_by_district = {}
        champ_events = []
        for event in events:
            if event.event_type_enum == EventType.DISTRICT_CMP:
                district_champs_by_district[event.district_key] = event
            elif event.event_type_enum == EventType.CMP_FINALS:
                champ_events.append(event)

        # Build district cmp division <-> parent associations based on district
        # Build cmp division <-> parent associations based on date
        for event in events:
            parent_event = None
            if event.event_type_enum == EventType.DISTRICT_CMP_DIVISION:
                parent_event = district_champs_by_district.get(event.district_key)
            elif event.event_type_enum == EventType.CMP_DIVISION:
                for parent_event in champ_events:
                    if abs(parent_event.end_date - event.end_date) < datetime.timedelta(days=1):
                        break
                else:
                    parent_event = None
            else:
                continue

            if parent_event is None:
                continue

            parent_event.divisions = sorted(parent_event.divisions + [event.key])
            event.parent_event = parent_event.key

        return events, list(districts.values())
    def parse(self, response):
        events = []
        districts = {}

        cmp_hack_sitevar = Sitevar.get_or_insert('cmp_registration_hacks')
        divisions_to_skip = cmp_hack_sitevar.contents.get('divisions_to_skip', []) \
            if cmp_hack_sitevar else []
        event_name_override = cmp_hack_sitevar.contents.get('event_name_override', []) \
            if cmp_hack_sitevar else []
        events_to_change_dates = cmp_hack_sitevar.contents.get('set_start_to_last_day', []) \
            if cmp_hack_sitevar else []

        for event in response['Events']:
            code = event['code'].lower()
            event_type = EventType.PRESEASON if code == 'week0' else self.EVENT_TYPES.get(event['type'].lower(), None)
            if event_type is None and not self.event_short:
                logging.warn("Event type '{}' not recognized!".format(event['type']))
                continue
            name = event['name']
            short_name = EventHelper.getShortName(name, district_code=event['districtCode'])
            district_enum = EventHelper.parseDistrictName(event['districtCode'].lower()) if event['districtCode'] else DistrictType.NO_DISTRICT
            district_key = District.renderKeyName(self.season, event['districtCode'].lower()) if event['districtCode'] else None
            venue = event['venue']
            city = event['city']
            state_prov = event['stateprov']
            country = event['country']
            start = datetime.datetime.strptime(event['dateStart'], self.DATE_FORMAT_STR)
            end = datetime.datetime.strptime(event['dateEnd'], self.DATE_FORMAT_STR)
            website = event.get('website')
            webcasts = [WebcastParser.webcast_dict_from_url(url) for url in event.get('webcasts', [])]

            # TODO read timezone from API

            # Special cases for district championship divisions
            if event_type == EventType.DISTRICT_CMP_DIVISION:
                split_name = name.split('-')
                short_name = '{} - {}'.format(
                    ''.join(item[0].upper() for item in split_name[0].split()),
                    split_name[-1].replace('Division', '').strip())

            # Special cases for champs
            if code in self.EVENT_CODE_EXCEPTIONS:
                code, short_name = self.EVENT_CODE_EXCEPTIONS[code]

                # FIRST indicates CMP registration before divisions are assigned by adding all teams
                # to Einstein. We will hack around that by not storing divisions and renaming
                # Einstein to simply "Championship" when certain sitevar flags are set

                if code in self.EINSTEIN_CODES:
                    override = [item for item in event_name_override if item['event'] == "{}{}".format(self.season, code)]
                    if override:
                        name = short_name.format(override[0]['name'])
                        short_name = short_name.format(override[0]['short_name'])
                else:  # Divisions
                    name = '{} Division'.format(short_name)
            elif self.event_short:
                code = self.event_short

            event_key = "{}{}".format(self.season, code)
            if event_key in divisions_to_skip:
                continue

            # Allow an overriding the start date to be the beginning of the last day
            if event_key in events_to_change_dates:
                start = end.replace(hour=0, minute=0, second=0, microsecond=0)

            events.append(Event(
                id=event_key,
                name=name,
                short_name=short_name,
                event_short=code,
                event_type_enum=event_type,
                official=True,
                start_date=start,
                end_date=end,
                venue=venue,
                city=city,
                state_prov=state_prov,
                country=country,
                venue_address=None,  # Even though FRC API provides address, ElasticSearch is more detailed
                year=self.season,
                event_district_enum=district_enum,
                district_key=ndb.Key(District, district_key) if district_key else None,
                website=website,
                webcast_json=json.dumps(webcasts) if webcasts else None,
            ))

            # Build District Model
            if district_key and district_key not in districts:
                districts[district_key] = District(
                    id=district_key,
                    year=self.season,
                    abbreviation=event['districtCode'].lower(),
                )

        # Prep for division <-> parent associations
        district_champs_by_district = {}
        champ_events = []
        for event in events:
            if event.event_type_enum == EventType.DISTRICT_CMP:
                district_champs_by_district[event.district_key] = event
            elif event.event_type_enum == EventType.CMP_FINALS:
                champ_events.append(event)

        # Build district cmp division <-> parent associations based on district
        # Build cmp division <-> parent associations based on date
        for event in events:
            parent_event = None
            if event.event_type_enum == EventType.DISTRICT_CMP_DIVISION:
                parent_event = district_champs_by_district.get(event.district_key)
            elif event.event_type_enum == EventType.CMP_DIVISION:
                for parent_event in champ_events:
                    if abs(parent_event.end_date - event.end_date) < datetime.timedelta(days=1):
                        break
                else:
                    parent_event = None
            else:
                continue

            if parent_event is None:
                continue

            parent_event.divisions = sorted(parent_event.divisions + [event.key])
            event.parent_event = parent_event.key

        return events, list(districts.values())
    def _process_request(self, request, event_key):
        try:
            event_info = json.loads(request.body)
        except Exception:
            self._errors = json.dumps({"Error": "Invalid json. Check input."})
            self.abort(400)

        if not isinstance(event_info, dict) or not event_info:
            self._errors = json.dumps({"Error": "Invalid json. Check input."})
            self.abort(400)

        do_team_remap = False
        for field, value in event_info.iteritems():
            if field not in self.ALLOWED_EVENT_PARAMS:
                continue

            if field == "webcasts":
                # Do special processing here because webcasts are janky
                if not isinstance(value, list):
                    self._errors = json.dumps(
                        {"Error": "Invalid json. Check input"}
                    )
                    self.abort(400)
                    return
                webcast_list = []
                for webcast in value:
                    if not isinstance(webcast, dict):
                        self._errors = json.dumps(
                            {"Error": "Invalid json. Check input"}
                        )
                        self.abort(400)
                        return
                    if 'url' in webcast:
                        webcast_list.append(
                            WebcastParser.webcast_dict_from_url(webcast['url'])
                        )
                    elif 'type' in webcast and 'channel' in webcast:
                        webcast_list.append(webcast)

                webcast_list = [w for w in webcast_list if w is not None]
                EventWebcastAdder.add_webcast(
                    self.event,
                    webcast_list,
                    False,  # Don't createOrUpdate yet
                )
            elif field == "remap_teams":
                # Validate remap_teams
                if not isinstance(value, dict):
                    raise ParserInputException("Invalid reamap_teams. Check input")
                for temp_team, remapped_team in value.items():
                    temp_match = re.match(r'frc\d+', str(temp_team))
                    remapped_match = re.match(r'frc\d+[B-Z]?', str(remapped_team))
                    if not temp_match or (temp_match and (temp_match.group(0) != str(temp_team))):
                        raise ParserInputException("Bad team: '{}'. Must follow format 'frcXXX'.".format(temp_team))
                    if not remapped_match or (remapped_match and (remapped_match.group(0) != str(remapped_team))):
                        raise ParserInputException("Bad team: '{}'. Must follow format 'frcXXX' or 'frcXXX[B-Z]'.".format(remapped_team))
                do_team_remap = True
                setattr(self.event, field, value)
            else:
                try:
                    if field == "first_event_code":
                        self.event.official = value is not None
                        field = "first_code"  # Internal property is different
                    setattr(self.event, field, value)
                except Exception, e:
                    self._errors({
                        "Error": "Unable to set event field",
                        "Message": str(e)
                    })
                    self.abort(400)
 def testUnknownUrl(self):
     bad = WebcastParser.webcast_dict_from_url("http://mywebsite.somewebcast")
     self.assertIsNone(bad)
Exemple #18
0
    def _process_request(self, request, event_key):
        try:
            event_info = json.loads(request.body)
        except Exception:
            self._errors = json.dumps({"Error": "Invalid json. Check input."})
            self.abort(400)

        if not isinstance(event_info, dict) or not event_info:
            self._errors = json.dumps({"Error": "Invalid json. Check input."})
            self.abort(400)

        event = Event.get_by_id(event_key)
        if not event:
            self._errors = json.dumps(
                {"Error": "Event {} not found".format(event_key)})
            self.abort(404)

        do_team_remap = False
        for field, value in event_info.iteritems():
            if field not in self.ALLOWED_EVENT_PARAMS:
                continue

            if field == "webcasts":
                # Do special processing here because webcasts are janky
                if not isinstance(value, list):
                    self._errors = json.dumps(
                        {"Error": "Invalid json. Check input"})
                    self.abort(400)
                    return
                webcast_list = []
                for webcast in value:
                    if not isinstance(webcast, dict):
                        self._errors = json.dumps(
                            {"Error": "Invalid json. Check input"})
                        self.abort(400)
                        return
                    if 'url' in webcast:
                        webcast_list.append(
                            WebcastParser.webcast_dict_from_url(
                                webcast['url']))
                    elif 'type' in webcast and 'channel' in webcast:
                        webcast_list.append(webcast)

                webcast_list = [w for w in webcast_list if w is not None]
                EventWebcastAdder.add_webcast(
                    event,
                    webcast_list,
                    False,  # Don't createOrUpdate yet
                )
            elif field == "remap_teams":
                # Validate remap_teams
                if not isinstance(value, dict):
                    raise ParserInputException(
                        "Invalid reamap_teams. Check input")
                for temp_team, remapped_team in value.items():
                    temp_match = re.match(r'frc\d+', str(temp_team))
                    remapped_match = re.match(r'frc\d+[B-Z]?',
                                              str(remapped_team))
                    if not temp_match or (
                            temp_match and
                        (temp_match.group(0) != str(temp_team))):
                        raise ParserInputException(
                            "Bad team: '{}'. Must follow format 'frcXXX'.".
                            format(temp_team))
                    if not remapped_match or (
                            remapped_match and
                        (remapped_match.group(0) != str(remapped_team))):
                        raise ParserInputException(
                            "Bad team: '{}'. Must follow format 'frcXXX' or 'frcXXX[B-Z]'."
                            .format(remapped_team))
                do_team_remap = True
                setattr(event, field, value)
            else:
                try:
                    if field == "first_event_code":
                        event.official = value is not None
                        field = "first_code"  # Internal property is different
                    setattr(event, field, value)
                except Exception, e:
                    self._errors({
                        "Error": "Unable to set event field",
                        "Message": str(e)
                    })
                    self.abort(400)