Example #1
0
class Sportschau(Plugin):
    _re_player = re.compile(
        r"https?:(//deviceids-medp.wdr.de/ondemand/\S+\.js)")
    _re_json = re.compile(
        r"\$mediaObject.jsonpHelper.storeAndPlay\(({.+})\);?")

    _schema_player = validate.Schema(
        validate.transform(_re_player.search),
        validate.any(
            None,
            validate.Schema(
                validate.get(1),
                validate.transform(lambda url: update_scheme("https:", url)))))
    _schema_json = validate.Schema(
        validate.transform(_re_json.match), validate.get(1),
        validate.transform(parse_json), validate.get("mediaResource"),
        validate.get("dflt"), validate.get("videoURL"),
        validate.transform(lambda url: update_scheme("https:", url)))

    def _get_streams(self):
        player_js = self.session.http.get(self.url, schema=self._schema_player)
        if not player_js:
            return

        log.debug("Found player js {0}".format(player_js))

        hls_url = self.session.http.get(player_js, schema=self._schema_json)

        yield from HLSStream.parse_variant_playlist(self.session,
                                                    hls_url).items()
Example #2
0
    def _get_streams(self):
        m = self._live_url_re.search(self.page.text)
        playlist_url = m and update_scheme(self.url, m.group(1))
        player_url = self.url
        token = self._get_token()

        if playlist_url:
            log.debug("Found playlist URL in the page")
        else:
            live_channel = None
            for div in itertags(self.page.text, "div"):
                if div.attributes.get("id") == "botonLive":
                    live_channel = div.attributes.get("data-canal")

            if live_channel:
                log.debug("Live channel: {0}".format(live_channel))
                player_url = self._channel_urls[live_channel] + quote(token)
                page = self.session.http.get(player_url,
                                             raise_for_status=False)
                if "block access from your country." in page.text:
                    raise PluginError("Content is geo-locked")
                m = self._playlist_re.search(page.text)
                playlist_url = m and update_scheme(self.url, m.group(1))
            else:
                log.error("Could not find the live channel")

        if playlist_url:
            stream_url = "{0}?{1}".format(playlist_url,
                                          urlencode({"iut": token}))
            return HLSStream.parse_variant_playlist(
                self.session, stream_url, headers={"referer": player_url})
Example #3
0
    def _get_streams(self):
        http.headers.update({'User-Agent': useragents.FIREFOX})
        log.debug('Version 2018-07-01')
        log.info('This is a custom plugin. '
                 'For support visit https://github.com/back-to/plugins')
        res = http.get(self.url)

        data = self._data_re.search(res.text)
        if data:
            log.debug('Found _data_re')
            data = self.js_to_json_regex(data.group(1))
            res = http.post(self.api_url, data=data)
            m = self._hls_re.search(res.text)
            if m:
                log.debug('Found _hls_re')
                hls_url = m.group('url')
                hls_url = update_scheme('http://', hls_url)
                log.debug('URL={0}'.format(hls_url))
                streams = HLSStream.parse_variant_playlist(
                    self.session, hls_url)
                if not streams:
                    return {'live': HLSStream(self.session, hls_url)}
                else:
                    return streams

        iframe = self._iframe_re.search(res.text)
        if iframe:
            log.debug('Found _iframe_re')
            iframe_url = iframe.group('url')
            iframe_url = update_scheme('http://', iframe_url)
            log.debug('URL={0}'.format(iframe_url))
            return self.session.streams(iframe_url)
Example #4
0
 def test_update_scheme(self):
     self.assertEqual(
         "https://example.com/foo",
         update_scheme("https://other.com/bar", "//example.com/foo"))
     self.assertEqual(
         "http://example.com/foo",
         update_scheme("http://other.com/bar", "//example.com/foo"))
     self.assertEqual(
         "http://example.com/foo",
         update_scheme("https://other.com/bar", "http://example.com/foo"))
Example #5
0
 def test_update_scheme(self):
     self.assertEqual(
         "https://example.com/foo",  # becomes https
         update_scheme("https://other.com/bar", "//example.com/foo"))
     self.assertEqual(
         "http://example.com/foo",  # becomes http
         update_scheme("http://other.com/bar", "//example.com/foo"))
     self.assertEqual(
         "http://example.com/foo",  # remains unchanged
         update_scheme("https://other.com/bar", "http://example.com/foo"))
     self.assertEqual(
         "https://example.com/foo",  # becomes https
         update_scheme("https://other.com/bar", "example.com/foo"))
Example #6
0
    def _get_streams(self):
        self.session.http.headers.update({
            'Referer':
            'http://www.abweb.com/BIS-TV-Online/bistvo-tele-universal.aspx'
        })

        login_username = self.get_option('username')
        login_password = self.get_option('password')

        if self.options.get('purge_credentials'):
            self.clear_cookies()
            self._authed = False
            log.info('All credentials were successfully removed.')

        if self._authed:
            log.info('Attempting to authenticate using cached cookies')
        elif not self._authed and not (login_username and login_password):
            log.error(
                'A login for ABweb is required, use --abweb-username USERNAME --abweb-password PASSWORD'
            )
            return
        elif not self._authed and not self._login(login_username,
                                                  login_password):
            return

        log.debug('get iframe_url')
        res = self.session.http.get(self.url)
        for iframe in itertags(res.text, 'iframe'):
            iframe_url = iframe.attributes.get('src')
            if iframe_url.startswith('/'):
                iframe_url = url_concat('https://www.abweb.com', iframe_url)
            else:
                iframe_url = update_scheme('https://', iframe_url)
            log.debug('iframe_url={0}'.format(iframe_url))
            break
        else:
            raise PluginError('No iframe_url found.')

        self.session.http.headers.update({'Referer': iframe_url})
        res = self.session.http.get(iframe_url)
        m = self._hls_re.search(res.text)
        if not m:
            raise PluginError('No hls_url found.')

        hls_url = update_scheme('https://', m.group('url'))
        streams = HLSStream.parse_variant_playlist(self.session, hls_url)
        if streams:
            for stream in streams.items():
                yield stream
        else:
            yield 'live', HLSStream(self.session, hls_url)
Example #7
0
    def _get_streams(self):
        """
        Find the streams for web.tv
        :return:
        """
        headers = {}
        res = self.session.http.get(self.url, headers=headers)
        headers["Referer"] = self.url

        sources = self._sources_re.findall(res.text)
        if len(sources):
            sdata = parse_json(sources[0], schema=self._sources_schema)
            for source in sdata:
                self.logger.debug("Found stream of type: {}", source[u'type'])
                if source[u'type'] == u"application/vnd.apple.mpegurl":
                    url = update_scheme(self.url, source[u"src"])

                    try:
                        # try to parse the stream as a variant playlist
                        variant = HLSStream.parse_variant_playlist(self.session, url, headers=headers)
                        if variant:
                            for q, s in variant.items():
                                yield q, s
                        else:
                            # and if that fails, try it as a plain HLS stream
                            yield 'live', HLSStream(self.session, url, headers=headers)
                    except IOError:
                        self.logger.warning("Could not open the stream, perhaps the channel is offline")
Example #8
0
    def _get_streams(self):
        self.session.http.headers.update({'User-Agent': useragents.FIREFOX})

        iframe_url = None
        page = self.session.http.get(self.url)
        for a in itertags(page.text, 'a'):
            if a.attributes.get('class') == 'play-live':
                iframe_url = update_scheme(self.url, a.attributes['data-url'])
                break

        if not iframe_url:
            raise PluginError('Could not find iframe.')

        parsed = urlparse(iframe_url)
        path_list = parsed.path.split('/')
        if len(path_list) != 6:
            # only support a known iframe url style,
            # the video id might be on a different spot if the url changes
            raise PluginError('unsupported iframe URL: {0}'.format(iframe_url))

        res = self.session.http.get(
            self.API_URL.format(netloc=parsed.netloc, id=path_list[4]))

        data = self.session.http.json(res, schema=self._api_schema)
        log.trace('{0!r}'.format(data))

        url = self.PLAYLIST_URL.format(
            app=data['streamProperties']['application'],
            name=data['playStreamName'],
            netloc=data['cdnHost'],
        )
        return HLSStream.parse_variant_playlist(self.session, url)
Example #9
0
    def _get_streams(self):
        url, params = parse_url_params(self.url)

        urlnoproto = self._url_re.match(url).group(2)
        urlnoproto = update_scheme("http://", urlnoproto)

        return HDSStream.parse_manifest(self.session, urlnoproto, **params)
Example #10
0
    def _get_streams(self):
        self.session.http.headers.update({'User-Agent': useragents.FIREFOX})
        log.debug('Version 2018-07-12')
        log.info('This is a custom plugin. '
                 'For support visit https://github.com/back-to/plugins')

        url, params = parse_url_params(self.url)
        urlnoproto = self._url_re.match(url).group(2)
        urlnoproto = update_scheme('http://', urlnoproto)

        streams = self.session.streams(urlnoproto, stream_types=['hls'])

        if not streams:
            log.debug('No stream found for hls-key-uri,'
                      ' stream is not available.')
            return

        stream = streams['best']
        urlnoproto = stream.url

        log.debug('URL={0}; params={1}', urlnoproto, params)
        streams = KeyUriHLSStream.parse_variant_playlist(
            self.session, urlnoproto, **params)
        if not streams:
            return {
                'live': KeyUriHLSStream(self.session, urlnoproto, **params)
            }
        else:
            return streams
Example #11
0
    def _get_streams(self):
        if self.is_live:
            log.debug("Loading live stream for {0}...".format(self.channel))

            res = self.session.http.get(self.live_api_url, data={"r": random.randint(1, 100000)})
            live_data = self.session.http.json(res)

            # all the streams are equal for each type, so pick a random one
            hls_streams = live_data.get("hls")
            if hls_streams:
                url = random.choice(hls_streams)
                url = url + '&' + urlencode(self.hls_session())  # TODO: use update_qsd
                for s in HLSStream.parse_variant_playlist(self.session, url, name_fmt="{pixels}_{bitrate}").items():
                    yield s

            mpd_streams = live_data.get("mpd")
            if mpd_streams:
                url = random.choice(mpd_streams)
                for s in DASHStream.parse_manifest(self.session, url).items():
                    yield s

        elif self.channel == "1tv":
            log.debug("Attempting to find VOD stream for {0}...".format(self.channel))
            vod_data = self.vod_data()
            if vod_data:
                log.info(u"Found VOD: {0}".format(vod_data[0]['title']))
                for stream in vod_data[0]['mbr']:
                    yield stream['name'], HTTPStream(self.session, update_scheme(self.url, stream['src']))
Example #12
0
    def _get_streams(self):
        """
        Find the streams for web.tv
        :return:
        """
        headers = {}
        res = self.session.http.get(self.url, headers=headers)
        headers["Referer"] = self.url

        sources = self._sources_re.findall(res.text)
        if len(sources):
            sdata = parse_json(sources[0], schema=self._sources_schema)
            for source in sdata:
                self.logger.debug("Found stream of type: {}", source[u'type'])
                if source[u'type'] == u"application/vnd.apple.mpegurl":
                    url = update_scheme(self.url, source[u"src"])

                    try:
                        # try to parse the stream as a variant playlist
                        variant = HLSStream.parse_variant_playlist(
                            self.session, url, headers=headers)
                        if variant:
                            for q, s in variant.items():
                                yield q, s
                        else:
                            # and if that fails, try it as a plain HLS stream
                            yield 'live', HLSStream(self.session,
                                                    url,
                                                    headers=headers)
                    except IOError:
                        self.logger.warning(
                            "Could not open the stream, perhaps the channel is offline"
                        )
Example #13
0
    def _get_streams(self):
        data = self.match.groupdict()
        url = update_scheme("http://", data.get("url"))
        params = parse_params(data.get("params"))
        log.debug(f"URL={url}; params={params}")

        return {"live": AkamaiHDStream(self.session, url, **params)}
Example #14
0
    def _get_streams(self):
        if self.is_live:
            self.logger.debug("Loading live stream for {0}...", self.channel)

            res = http.get(self.live_api_url,
                           data={"r": random.randint(1, 100000)})
            live_data = http.json(res)

            for url in live_data.get(
                    "hls",
                [])[:1]:  # only take the first, they are all the same streams
                url = url + '&' + urlencode(
                    self.hls_session())  # TODO: use update_qsd
                for s in HLSStream.parse_variant_playlist(
                        self.session, url,
                        name_fmt="{pixels}_{bitrate}").items():
                    yield s
        elif self.channel == "1tv":
            self.logger.debug("Attempting to find VOD stream...", self.channel)
            vod_data = self.vod_data()
            if vod_data:
                self.logger.info(u"Found VOD: {0}".format(
                    vod_data[0]['title']))
                for stream in vod_data[0]['mbr']:
                    yield stream['name'], HTTPStream(
                        self.session, update_scheme(self.url, stream['src']))
Example #15
0
    def _get_streams(self):
        url, params = parse_url_params(self.url)

        urlnoproto = self._url_re.match(url).group(2)
        urlnoproto = update_scheme("http://", urlnoproto)

        return HDSStream.parse_manifest(self.session, urlnoproto, **params)
Example #16
0
    def _get_streams(self):
        res = self.session.http.get(self.url)
        match = _player_js.search(res.text)
        if match:
            player_js = match.group(0)
            self.logger.info("Found player js {0}", player_js)
        else:
            self.logger.info("Didn't find player js. Probably this page doesn't contain a video")
            return

        res = self.session.http.get(player_js)

        jsonp_start = res.text.find('(') + 1
        jsonp_end = res.text.rfind(')')

        if jsonp_start <= 0 or jsonp_end <= 0:
            self.logger.info("Couldn't extract json metadata from player.js: {0}", player_js)
            return

        json_s = res.text[jsonp_start:jsonp_end]

        stream_metadata = json.loads(json_s)

        hds_url = stream_metadata['mediaResource']['dflt']['videoURL']
        hds_url = update_scheme(self.url, hds_url)

        return HDSStream.parse_manifest(self.session, hds_url).items()
Example #17
0
    def _get_streams(self):
        self.session.http.headers.update({'User-Agent': useragents.FIREFOX})

        iframe_url = None
        page = self.session.http.get(self.url)
        for a in itertags(page.text, 'a'):
            if a.attributes.get('class') == 'play-live':
                iframe_url = update_scheme(self.url, a.attributes['data-url'])
                break

        if not iframe_url:
            raise PluginError('Could not find iframe.')

        parsed = urlparse(iframe_url)
        path_list = parsed.path.split('/')
        if len(path_list) != 6:
            # only support a known iframe url style,
            # the video id might be on a different spot if the url changes
            raise PluginError('unsupported iframe URL: {0}'.format(iframe_url))

        res = self.session.http.get(
            self.API_URL.format(netloc=parsed.netloc, id=path_list[4]))

        data = self.session.http.json(res, schema=self._api_schema)
        log.trace('{0!r}'.format(data))

        url = self.PLAYLIST_URL.format(
            app=data['streamProperties']['application'],
            name=data['playStreamName'],
            netloc=data['cdnHost'],
        )
        return HLSStream.parse_variant_playlist(self.session, url)
Example #18
0
 def find_iframe(self, url):
     self.session.http.headers.update({"User-Agent": useragents.CHROME})
     res = self.session.http.get(self.url)
     for iframe_url in self.iframe_re.findall(res.text):
         if "googletagmanager" not in iframe_url:
             iframe_url = html_unescape(iframe_url)
             return update_scheme(self.url, iframe_url)
Example #19
0
    def _get_streams(self):
        res = self.session.http.get(self.url)
        match = _player_js.search(res.text)
        if match:
            player_js = match.group(0)
            self.logger.info("Found player js {0}", player_js)
        else:
            self.logger.info(
                "Didn't find player js. Probably this page doesn't contain a video"
            )
            return

        res = self.session.http.get(player_js)

        jsonp_start = res.text.find('(') + 1
        jsonp_end = res.text.rfind(')')

        if jsonp_start <= 0 or jsonp_end <= 0:
            self.logger.info(
                "Couldn't extract json metadata from player.js: {0}",
                player_js)
            return

        json_s = res.text[jsonp_start:jsonp_end]

        stream_metadata = json.loads(json_s)

        hds_url = stream_metadata['mediaResource']['dflt']['videoURL']
        hds_url = update_scheme(self.url, hds_url)

        return HDSStream.parse_manifest(self.session, hds_url).items()
Example #20
0
    def _get_live_streams(self):
        # Get channel id
        match = self._url_re.match(self.url)
        channel = match.group('channel')

        # Retrieve live player URL
        res = http.get(self.PLAYER_URL)
        match = self._live_player_re.search(res.text)
        if match is None:
            return []
        live_player_url = update_scheme(self.url,
                                        match.group('live_player_url'))

        # Extract streams from the live player page
        res = http.get(live_player_url)
        stream_datas = re.findall(
            r'{0}(?:_MINI)?:({{.+?}}]}}]}})'.format(self.CHANNEL_MAP[channel]),
            res.text)
        streams = []
        for s in stream_datas:
            for u in self._live_streams_schema.validate(s):
                if u not in streams:
                    streams.append(u)

        return streams
Example #21
0
    def _get_streams(self):
        url, params = parse_url_params(self.url)
        urlnoproto = self._url_re.match(url).group(1)
        urlnoproto = update_scheme("http://", urlnoproto)

        log.debug("URL={0}; params={1}".format(urlnoproto, params))
        return {"live": AkamaiHDStream(self.session, urlnoproto, **params)}
Example #22
0
    def _get_streams(self):
        url, params = parse_url_params(self.url)
        urlnoproto = self._url_re.match(url).group(1)
        urlnoproto = update_scheme("http://", urlnoproto)

        self.logger.debug("URL={0}; params={1}", urlnoproto, params)
        return {"live": HTTPStream(self.session, urlnoproto, **params)}
Example #23
0
    def _find_video_id(self, url):
        m = _url_re.match(url)
        if m and m.group("video_id"):
            log.debug("Video & PL ID from URL")
            return m.group("video_id"), m.group("pl_id")

        res = self.session.http.get(
            update_scheme("https://", url.replace("youtubedl://", "")))
        datam = _ytdata_re.search(res.text)
        if datam:
            data = parse_json(datam.group(1))
            # find the videoRenderer object, where there is a LIVE NOW badge
            for vid_ep in search_dict(data, "currentVideoEndpoint"):
                video_id = vid_ep.get("watchEndpoint", {}).get("videoId")
                if video_id:
                    log.debug("Video ID from currentVideoEndpoint")
                    return video_id, None
            for x in search_dict(data, "videoRenderer"):
                for bstyle in search_dict(x.get("badges", {}), "style"):
                    if bstyle == "BADGE_STYLE_TYPE_LIVE_NOW":
                        if x.get("videoId"):
                            log.debug("Video ID from videoRenderer (live)")
                            return x["videoId"], None

        if "/embed/live_stream" in url:
            for link in itertags(res.text, "link"):
                if link.attributes.get("rel") == "canonical":
                    canon_link = link.attributes.get("href")
                    if canon_link != url:
                        log.debug("Re-directing to canonical URL: {0}".format(
                            canon_link))
                        return self._find_video_id(canon_link), None

        raise PluginError("Could not find a Video ID in URL")
Example #24
0
    def _get_streams(self):
        res = self.session.http.get(self.url)

        # Look for Youtube embedded video first
        for iframe in itertags(res.text, "iframe"):
            if urlparse(iframe.attributes.get("src")).netloc.endswith(
                    "youtube.com"):
                log.debug("Handing off to YouTube plugin")
                return self.session.streams(iframe.attributes.get("src"))

        # Next check for HLS URL with token
        mobile_url_m = self.mobile_url_re.search(res.text)
        mobile_url = mobile_url_m and update_scheme(self.url,
                                                    mobile_url_m.group("url"))
        if mobile_url:
            log.debug("Found mobile stream: {0}".format(mobile_url_m.group(0)))

            token = mobile_url_m and mobile_url_m.group("token")
            if not token and "kralmuzik" in self.url:
                log.debug("Getting Kral Muzik HLS stream token from API")
                token = self.session.http.get(self.kral_token_url).text
            elif not token:
                # if no token is in the url, try to find it else where in the page
                log.debug("Searching for HLS stream token in URL")
                token_m = self.token_re.search(res.text)
                token = token_m and token_m.group("token")

            return HLSStream.parse_variant_playlist(
                self.session,
                mobile_url + token,
                headers={"Referer": self.url})
Example #25
0
    def _get_streams(self):
        url, params = parse_url_params(self.url)
        urlnoproto = self._url_re.match(url).group(1)
        urlnoproto = update_scheme("http://", urlnoproto)

        self.logger.debug("URL={0}; params={1}", urlnoproto, params)
        return {"live": AkamaiHDStream(self.session, urlnoproto, **params)}
Example #26
0
    def _get_streams(self):
        res = self.session.http.get(self.url)
        match = _media_id_re.search(res.text)
        if match:
            media_id = match.group(1)
        else:
            return

        log.debug("Found media id: {0}".format(media_id))
        res = self.session.http.get(MEDIA_URL.format(media_id))
        media = self.session.http.json(res, schema=_media_schema)
        log.trace("{0!r}".format(media))

        for media in media["_mediaArray"]:
            for stream in media["_mediaStreamArray"]:
                stream_ = stream["_stream"]
                if isinstance(stream_, list):
                    if not stream_:
                        continue
                    stream_ = stream_[0]

                stream_ = update_scheme("https://", stream_)
                if ".m3u8" in stream_:
                    parser = self._get_hls_streams
                    parser_name = "HLS"
                elif (".mp4" in stream_ and ".f4m" not in stream_):
                    parser = self._get_http_streams
                    parser_name = "HTTP"
                else:
                    log.error("Unexpected stream type: '{0}'".format(stream_))

                try:
                    yield from parser(stream)
                except OSError as err:
                    log.error("Failed to extract {0} streams: {1}".format(parser_name, err))
Example #27
0
    def _get_streams(self):
        if self.is_live:
            self.logger.debug("Loading live stream for {0}...", self.channel)

            res = self.session.http.get(self.live_api_url, data={"r": random.randint(1, 100000)})
            live_data = self.session.http.json(res)

            # all the streams are equal for each type, so pick a random one
            hls_streams = live_data.get("hls")
            if hls_streams:
                url = random.choice(hls_streams)
                url = url + '&' + urlencode(self.hls_session())  # TODO: use update_qsd
                for s in HLSStream.parse_variant_playlist(self.session, url, name_fmt="{pixels}_{bitrate}").items():
                    yield s

            mpd_streams = live_data.get("mpd")
            if mpd_streams:
                url = random.choice(mpd_streams)
                for s in DASHStream.parse_manifest(self.session, url).items():
                    yield s

        elif self.channel == "1tv":
            self.logger.debug("Attempting to find VOD stream...", self.channel)
            vod_data = self.vod_data()
            if vod_data:
                self.logger.info(u"Found VOD: {0}".format(vod_data[0]['title']))
                for stream in vod_data[0]['mbr']:
                    yield stream['name'], HTTPStream(self.session, update_scheme(self.url, stream['src']))
Example #28
0
 def test_update_scheme(self):
     self.assertEqual(
         "https://example.com/foo",  # becomes https
         update_scheme("https://other.com/bar", "//example.com/foo")
     )
     self.assertEqual(
         "http://example.com/foo",  # becomes http
         update_scheme("http://other.com/bar", "//example.com/foo")
     )
     self.assertEqual(
         "http://example.com/foo",  # remains unchanged
         update_scheme("https://other.com/bar", "http://example.com/foo")
     )
     self.assertEqual(
         "https://example.com/foo",  # becomes https
         update_scheme("https://other.com/bar", "example.com/foo")
     )
    def _get_streams(self):
        data = http.get(self.url, schema=self.config_schema)

        for info in data["files"].values():
            stream_url = update_scheme(self.url, info["url"])
            # pick the smaller of the two dimensions, for landscape v. portrait videos
            res = min(info["width"], info["height"])
            yield "{0}p".format(res), HTTPStream(self.session, stream_url)
Example #30
0
 def merge_path_list(self, static, user):
     for _path_url in user:
         if not _path_url.startswith(('http', '//')):
             _path_url = update_scheme('http://', _path_url)
         _parsed_path_url = urlparse(_path_url)
         if _parsed_path_url.netloc and _parsed_path_url.path:
             static += [(_parsed_path_url.netloc, _parsed_path_url.path)]
     return static
Example #31
0
    def live_api_url(self):
        channel = self.channel
        if channel == "1tv":
            url = self._1tv_api
        else:
            url = self._ctc_api.format(channel=channel)

        return update_scheme(self.url, url)
Example #32
0
    def _get_streams(self):
        data = self.session.http.get(self.url, schema=self.config_schema)

        for info in data["files"].values():
            stream_url = update_scheme(self.url, info["url"])
            # pick the smaller of the two dimensions, for landscape v. portrait videos
            res = min(info["width"], info["height"])
            yield "{0}p".format(res), HTTPStream(self.session, stream_url)
Example #33
0
    def live_api_url(self):
        channel = self.channel
        if channel == "1tv":
            url = self._1tv_api
        else:
            url = self._ctc_api.format(channel=channel)

        return update_scheme(self.url, url)
Example #34
0
 def find_iframe(self, url):
     res = http.get(url)
     # find iframe url
     iframe = self.iframe_re.search(res.text)
     iframe_url = iframe and iframe.group(1)
     if iframe_url:
         iframe_url = update_scheme(self.url, iframe_url)
         self.logger.debug("Found iframe: {}", iframe_url)
     return iframe_url
Example #35
0
    def _get_streams(self):
        res = http.get(self.url)
        mobile_url_m = self.mobile_url_re.search(res.text)
        desktop_url_m = self.desktop_url_re.search(res.text)

        desktop_url = desktop_url_m and update_scheme(
            self.url, desktop_url_m.group("url"))
        mobile_url = mobile_url_m and update_scheme(self.url,
                                                    mobile_url_m.group("url"))

        token = (desktop_url_m and desktop_url_m.group("token")) or (
            mobile_url_m and mobile_url_m.group("token"))
        if not token:
            # if no token is in the url, try to find it else where in the page
            token_m = self.token_re.search(res.text)
            token = token_m and token_m.group("token")

        return self._get_star_streams(desktop_url, mobile_url, token=token)
Example #36
0
    def _get_http_streams(self, info):
        name = QUALITY_MAP.get(info["_quality"], "vod")
        urls = info["_stream"]
        if not isinstance(info["_stream"], list):
            urls = [urls]

        for url in urls:
            stream = HTTPStream(self.session, update_scheme("https://", url))
            yield name, stream
Example #37
0
    def _get_streams(self):
        match = _url_re.match(self.url)
        channel = match.group("channel")

        http.headers.update({"User-Agent": useragents.IPAD})
        # Some problem with SSL on huya.com now, do not use https

        hls_url = http.get(HUYA_URL % channel, schema=_hls_schema)
        yield "live", HLSStream(self.session, update_scheme("http://", hls_url))
Example #38
0
 def find_iframe(self, url):
     res = http.get(url)
     # find iframe url
     iframe = self.iframe_re.search(res.text)
     iframe_url = iframe and iframe.group(1)
     if iframe_url:
         iframe_url = update_scheme(self.url, iframe_url)
         self.logger.debug("Found iframe: {}", iframe_url)
     return iframe_url
Example #39
0
 def _get_streams(self):
     page = self.session.http.get(self.url)
     for iframe in itertags(page.text, u"iframe"):
         url = iframe.attributes["src"]
         log.debug("Handing off of {0}".format(url))
         try:
             return self.session.streams(update_scheme(self.url, url))
         except NoPluginError:
             log.error("Handing off of {0} failed".format(url))
             return None
Example #40
0
 def _get_streams(self):
     page = self.session.http.get(self.url)
     for iframe in itertags(page.text, u"iframe"):
         url = iframe.attributes["src"]
         self.logger.debug("Handing off of {0}".format(url))
         try:
             return self.session.streams(update_scheme(self.url, url))
         except NoPluginError:
             self.logger.error("Handing off of {0} failed".format(url))
             return None
Example #41
0
 def _get_streams(self):
     page = self.session.http.get(self.url)
     iframe = self.iframe_re.search(page.text)
     if iframe:
         self.logger.debug("Handing off of {0}".format(iframe.group("url")))
         try:
             return self.session.streams(update_scheme(self.url, iframe.group("url")))
         except NoPluginError:
             self.logger.error("Handing off of {0} failed".format(iframe.group("url")))
             return None
Example #42
0
    def _get_streams(self):
        match = _url_re.match(self.url)
        channel = match.group("channel")

        self.session.http.headers.update({"User-Agent": useragents.IPAD})
        # Some problem with SSL on huya.com now, do not use https

        hls_url = self.session.http.get(HUYA_URL % channel, schema=_hls_schema)
        yield "live", HLSStream(self.session,
                                update_scheme("http://", hls_url))
Example #43
0
 def _get_streams(self):
     page = http.get(self.url)
     iframe = self.iframe_re.search(page.text)
     if iframe:
         self.logger.debug("Handing off of {0}".format(iframe.group("url")))
         try:
             return self.session.streams(update_scheme(self.url, iframe.group("url")))
         except NoPluginError:
             self.logger.error("Handing off of {0} failed".format(iframe.group("url")))
             return None
Example #44
0
    def _get_streams(self):
        res = http.get(self.url)
        m = self.embed_url_re.search(res.text)
        platform_url = m and m.group("url")

        if platform_url:
            url = update_scheme(self.url, platform_url)
            # hand off to ThePlatform plugin
            p = ThePlatform(url)
            p.bind(self.session, "plugin.nbcsports")
            return p.streams()
Example #45
0
    def _get_streams(self):
        url, params = parse_url_params(self.url)
        urlnoproto = self._url_re.match(url).group(2)
        urlnoproto = update_scheme("http://", urlnoproto)

        self.logger.debug("URL={0}; params={1}", urlnoproto, params)
        streams = HLSStream.parse_variant_playlist(self.session, urlnoproto, **params)
        if not streams:
            return {"live": HLSStream(self.session, urlnoproto, **params)}
        else:
            return streams
Example #46
0
 def _get_streams(self):
     res = self.session.http.get(self.url)
     for iframe in itertags(res.text, "iframe"):
         self.logger.debug("Found iframe: {0}".format(iframe))
         iframe_res = self.session.http.get(iframe.attributes['src'], headers={"Referer": self.url})
         m = self.src_re.search(iframe_res.text)
         surl = m and m.group("url")
         if surl:
             surl = update_scheme(self.url, surl)
             self.logger.debug("Found stream URL: {0}".format(surl))
             return HLSStream.parse_variant_playlist(self.session, surl)
Example #47
0
    def get_iframe_url(self):
        self.logger.debug('search for an iframe')
        res = http.get(self.url)
        m = self._iframe_re.search(res.text)
        if not m:
            raise PluginError('No iframe found.')

        iframe_url = m.group('url')
        iframe_url = update_scheme('http://', iframe_url)
        self.logger.debug('IFRAME URL={0}'.format(iframe_url))
        return iframe_url
Example #48
0
    def _get_streams(self):
        res = http.get(self.url)
        # Search for the iframe in the page
        iframe_m = self.iframe_re.search(res.text)

        ustream_url = iframe_m and iframe_m.group(1)
        if ustream_url and "ustream.tv" in ustream_url:
            try:
                ustream_url = update_scheme(self.url, ustream_url)
                return self.session.streams(ustream_url)
            except NoPluginError:
                raise PluginError("Could not play embedded stream: {0}".format(ustream_url))
Example #49
0
    def _get_streams(self):
        """
        Find the streams for vk.com
        :return:
        """
        self.session.http.headers.update({'User-Agent': useragents.IPHONE_6})

        # If this is a 'videos' catalog URL
        # with an video ID in the GET request, get that instead
        url = self.follow_vk_redirect(self.url)

        m = self._url_re.match(url)
        if not m:
            log.error('URL is not compatible: {0}'.format(url))
            return

        video_id = m.group('video_id')
        log.debug('video ID: {0}'.format(video_id))

        params = {
            'act': 'show_inline',
            'al': '1',
            'video': video_id,
        }
        res = self.session.http.post(self.API_URL, params=params)

        for _i in itertags(res.text, 'iframe'):
            if _i.attributes.get('src'):
                iframe_url = update_scheme(self.url, _i.attributes['src'])
                log.debug('Found iframe: {0}'.format(iframe_url))
                for s in self.session.streams(iframe_url).items():
                    yield s

        for _i in itertags(res.text, 'source'):
            if _i.attributes.get('type') == 'application/vnd.apple.mpegurl':
                video_url = _i.attributes['src']
                # Remove invalid URL
                if video_url.startswith('https://vk.com/'):
                    continue
                streams = HLSStream.parse_variant_playlist(self.session,
                                                           video_url)
                if not streams:
                    yield 'live', HLSStream(self.session, video_url)
                else:
                    for s in streams.items():
                        yield s
            elif _i.attributes.get('type') == 'video/mp4':
                q = 'vod'
                video_url = _i.attributes['src']
                m = self._vod_quality_re.search(video_url)
                if m:
                    q = '{0}p'.format(m.group(1))
                yield q, HTTPStream(self.session, video_url)
Example #50
0
    def _get_streams(self):
        http.headers = {"User-Agent": useragents.CHROME}
        res = http.get(self.url)
        iframe_url = self.find_iframe(res)

        if iframe_url:
            self.logger.debug("Found iframe: {0}", iframe_url)
            res = http.get(iframe_url, headers={"Referer": self.url})
            stream_url = update_scheme(self.url, self.stream_schema.validate(res.text))
            return HLSStream.parse_variant_playlist(self.session,
                                                    stream_url,
                                                    headers={"User-Agent": useragents.CHROME})
Example #51
0
    def _get_streams(self):
        self.session.http.headers.update({"User-Agent": useragents.CHROME})
        res = self.session.http.get(self.url)
        iframe_url = self.find_iframe(res)

        if iframe_url:
            log.debug("Found iframe: {0}", iframe_url)
            res = self.session.http.get(iframe_url, headers={"Referer": self.url})
            stream_url = update_scheme(self.url, self.stream_schema.validate(res.text))
            log.warning("SSL Verification disabled.")
            return HLSStream.parse_variant_playlist(self.session,
                                                    stream_url,
                                                    verify=False)
Example #52
0
    def _get_streams(self):
        self.session.set_option('hls-live-edge', 10)
        res = self.session.http.get(self.url)
        playlist_m = self._playlist_re.search(res.text)

        if playlist_m:
            return HLSStream.parse_variant_playlist(
                self.session,
                update_scheme(self.url, playlist_m.group(1)),
                headers={'Referer': self.url, 'User-Agent': useragents.ANDROID}
            )
        else:
            log.debug("Could not find stream data")
Example #53
0
    def _get_streams(self):
        res = self.session.http.get(self.url)
        mobile_url_m = self.mobile_url_re.search(res.text)

        mobile_url = mobile_url_m and update_scheme(self.url, mobile_url_m.group("url"))

        token = mobile_url_m and mobile_url_m.group("token")
        if not token:
            # if no token is in the url, try to find it else where in the page
            token_m = self.token_re.search(res.text)
            token = token_m and token_m.group("token")

        return HLSStream.parse_variant_playlist(self.session, mobile_url + token,
                                                headers={"Referer": self.url})
Example #54
0
    def _get_streams(self):
        http.headers.update({'User-Agent': useragents.CHROME,
                             'Referer': 'http://www.abweb.com/BIS-TV-Online/bistvo-tele-universal.aspx'})

        login_username = self.get_option('username')
        login_password = self.get_option('password')

        if self.options.get('purge_credentials'):
            self._session_attributes.set('ASP.NET_SessionId', None, expires=0)
            self._session_attributes.set('.abportail1', None, expires=0)
            self._authed = False
            self.logger.info('All credentials were successfully removed.')

        if not self._authed and not (login_username and login_password):
            self.logger.error('A login for ABweb is required, use --abweb-username USERNAME --abweb-password PASSWORD')
            return

        if self._authed:
            if self._expires < time.time():
                self.logger.debug('get new cached cookies')
                # login after 24h
                self.set_expires_time_cache()
                self._authed = False
            else:
                self.logger.info('Attempting to authenticate using cached cookies')
                http.cookies.set('ASP.NET_SessionId', self._session_attributes.get('ASP.NET_SessionId'))
                http.cookies.set('.abportail1', self._session_attributes.get('.abportail1'))

        if not self._authed and not self._login(login_username, login_password):
            return

        iframe_url = self.get_iframe_url()
        http.headers.update({'Referer': iframe_url})

        hls_url = self.get_hls_url(iframe_url)
        hls_url = update_scheme(self.url, hls_url)

        self.logger.debug('URL={0}'.format(hls_url))
        variant = HLSStream.parse_variant_playlist(self.session, hls_url)
        if variant:
            for q, s in variant.items():
                yield q, s
        else:
            yield 'live', HLSStream(self.session, hls_url)
Example #55
0
 def _get_streams_api(self, video_id):
     res = self.session.http.get(self.api_server,
                                 params=dict(video_id=video_id))
     data = self.session.http.json(res)
     if data["success"]:
         for x in itertools.chain(*data['data']['versions'].values()):
             src = update_scheme(self.url, x['src'])
             if x['type'] == "application/x-mpegurl":
                 for s in HLSStream.parse_variant_playlist(self.session, src).items():
                     yield s
             elif x['type'] == "application/dash+xml":
                 for s in DASHStream.parse_manifest(self.session, src).items():
                     yield s
             elif x['type'] == "video/mp4":
                 yield "{0}p".format(x['res']), HTTPStream(self.session, src)
     else:
         log.error("Failed to get streams: {0} ({1})".format(
             data['message'], data['code']
         ))
Example #56
0
    def _get_streams(self):
        vid = self.url_re.match(self.url).group(1)
        self.logger.debug("Found video ID: {0}", vid)

        page = self.session.http.get(self.play_url.format(vid=vid))
        js_url_m = self.js_re.search(page.text)
        if js_url_m:
            js_url = js_url_m.group(1)
            self.logger.debug("Loading player JS: {0}", js_url)

            res = self.session.http.get(js_url)
            metadata_url = update_scheme(self.url, self.setup_schema.validate(res.text))
            data = self.session.http.json(self.session.http.get(metadata_url))

            for source in data["playlist"][0]["sources"]:
                if source["type"] == "application/vnd.apple.mpegurl":
                    for s in HLSStream.parse_variant_playlist(self.session, source["file"]).items():
                        yield s
                elif source["type"] == "video/mp4":
                    yield "{0}p".format(source["height"]), HTTPStream(self.session, source["file"])
Example #57
0
    def _get_live_streams(self):
        # Get channel id
        match = self._url_re.match(self.url)
        channel = match.group('channel')

        # Retrieve live player URL
        res = http.get(self.PLAYER_URL)
        match = self._live_player_re.search(res.text)
        if match is None:
            return []
        live_player_url = update_scheme(self.url, match.group('live_player_url'))

        # Extract streams from the live player page
        res = http.get(live_player_url)
        stream_datas = re.findall(r'{0}(?:_MINI)?:({{.+?}}]}}]}})'.format(self.CHANNEL_MAP[channel]), res.text)
        streams = []
        for s in stream_datas:
            for u in self._live_streams_schema.validate(s):
                if u not in streams:
                    streams.append(u)

        return streams
Example #58
0
    def resolve_url(self, url, follow_redirect=True):
        """Attempts to find a plugin that can use this URL.

        The default protocol (http) will be prefixed to the URL if
        not specified.

        Raises :exc:`NoPluginError` on failure.

        :param url: a URL to match against loaded plugins
        :param follow_redirect: follow redirects

        """
        url = update_scheme("http://", url)

        available_plugins = []
        for name, plugin in self.plugins.items():
            if plugin.can_handle_url(url):
                available_plugins.append(plugin)

        available_plugins.sort(key=lambda x: x.priority(url), reverse=True)
        if available_plugins:
            return available_plugins[0](url)

        if follow_redirect:
            # Attempt to handle a redirect URL
            try:
                res = self.http.head(url, allow_redirects=True, acceptable_status=[501])

                # Fall back to GET request if server doesn't handle HEAD.
                if res.status_code == 501:
                    res = self.http.get(url, stream=True)

                if res.url != url:
                    return self.resolve_url(res.url, follow_redirect=follow_redirect)
            except PluginError:
                pass

        raise NoPluginError
Example #59
0
    def _get_streams(self):
        match = url_re.match(self.url)

        stream_page_scheme = 'https'
        stream_page_domain = match.group(4)
        stream_page_path = match.group(5)
        country_code = CONST_DEFAULT_COUNTRY_CODE

        # create http session and set headers
        http_session = http
        http_session.headers.update(CONST_HEADERS)

        # get cookies
        r = http_session.get(urlunparse((stream_page_scheme, stream_page_domain, stream_page_path, '', '', '')))

        # redirect to profile page means stream is offline
        if '/profile/' in r.url:
            raise NoStreamsError(self.url)
        if not r.ok:
            self.logger.debug("Status code for {0}: {1}", r.url, r.status_code)
            raise NoStreamsError(self.url)
        if len(http_session.cookies) == 0:
            raise PluginError("Can't get a cookies")

        if urlparse(r.url).netloc != stream_page_domain:
            # then redirected to regional subdomain
            country_code = urlparse(r.url).netloc.split('.')[0].lower()

        # time to set variables
        baseurl = urlunparse((stream_page_scheme, urlparse(r.url).netloc, '', '', '', ''))
        amf_gateway_url = urljoin(baseurl, CONST_AMF_GATEWAY_LOCATION)
        stream_page_url = urljoin(baseurl, stream_page_path)

        headers = {
            'User-Agent': useragents.CHROME,
            'Referer': stream_page_url,
            'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
            'X-Requested-With': 'XMLHttpRequest'
        }

        data = 'method=getRoomData&args%5B%5D={0}&args%5B%5D=false'.format(stream_page_path)
        self.logger.debug('DATA: {0}'.format(str(data)))
        # send request and close http-session
        r = http_session.post(url=amf_gateway_url,
                              headers=headers,
                              params={CONST_AMF_GATEWAY_PARAM: country_code},
                              data=data)
        http_session.close()

        if r.status_code != 200:
            raise PluginError("unexpected status code for {0}: {1}", r.url, r.status_code)

        stream_source_info = amf_msg_schema.validate(json.loads(r.text))
        self.logger.debug("source stream info:\n{0}", stream_source_info)

        if not stream_source_info:
            return

        urlnoproto = stream_source_info['localData']['videoServerUrl']
        urlnoproto = update_scheme('https://', urlnoproto)
        performer = stream_source_info['performerData']['username']

        hls_url = '{0}/hls/stream_{1}/playlist.m3u8'.format(urlnoproto, performer)

        if hls_url:
            self.logger.debug('HLS URL: {0}'.format(hls_url))
            try:
                for s in HLSStream.parse_variant_playlist(self.session, hls_url, headers=headers).items():
                    yield s
            except Exception as e:
                if '404' in str(e):
                    self.logger.error('Stream is currently offline or private')
                else:
                    self.logger.error(str(e))
                return
Example #60
0
    def set_option(self, key, value):
        """Sets general options used by plugins and streams originating
        from this session object.

        :param key: key of the option
        :param value: value to set the option to


        **Available options**:

        ======================== =========================================
        hds-live-edge            ( float) Specify the time live HDS
                                 streams will start from the edge of
                                 stream, default: ``10.0``

        hds-segment-attempts     (int) How many attempts should be done
                                 to download each HDS segment, default: ``3``

        hds-segment-threads      (int) The size of the thread pool used
                                 to download segments, default: ``1``

        hds-segment-timeout      (float) HDS segment connect and read
                                 timeout, default: ``10.0``

        hds-timeout              (float) Timeout for reading data from
                                 HDS streams, default: ``60.0``

        hls-live-edge            (int) How many segments from the end
                                 to start live streams on, default: ``3``

        hls-segment-attempts     (int) How many attempts should be done
                                 to download each HLS segment, default: ``3``

        hls-segment-threads      (int) The size of the thread pool used
                                 to download segments, default: ``1``

        hls-segment-timeout      (float) HLS segment connect and read
                                 timeout, default: ``10.0``

        hls-timeout              (float) Timeout for reading data from
                                 HLS streams, default: ``60.0``

        http-proxy               (str) Specify a HTTP proxy to use for
                                 all HTTP requests

        https-proxy              (str) Specify a HTTPS proxy to use for
                                 all HTTPS requests

        http-cookies             (dict or str) A dict or a semi-colon (;)
                                 delimited str of cookies to add to each
                                 HTTP request, e.g. ``foo=bar;baz=qux``

        http-headers             (dict or str) A dict or semi-colon (;)
                                 delimited str of headers to add to each
                                 HTTP request, e.g. ``foo=bar;baz=qux``

        http-query-params        (dict or str) A dict or a ampersand (&)
                                 delimited string of query parameters to
                                 add to each HTTP request,
                                 e.g. ``foo=bar&baz=qux``

        http-trust-env           (bool) Trust HTTP settings set in the
                                 environment, such as environment
                                 variables (HTTP_PROXY, etc) and
                                 ~/.netrc authentication

        http-ssl-verify          (bool) Verify SSL certificates,
                                 default: ``True``

        http-ssl-cert            (str or tuple) SSL certificate to use,
                                 can be either a .pem file (str) or a
                                 .crt/.key pair (tuple)

        http-timeout             (float) General timeout used by all HTTP
                                 requests except the ones covered by
                                 other options, default: ``20.0``

        http-stream-timeout      (float) Timeout for reading data from
                                 HTTP streams, default: ``60.0``

        subprocess-errorlog      (bool) Log errors from subprocesses to
                                 a file located in the temp directory

        subprocess-errorlog-path (str) Log errors from subprocesses to
                                 a specific file

        ringbuffer-size          (int) The size of the internal ring
                                 buffer used by most stream types,
                                 default: ``16777216`` (16MB)

        rtmp-proxy               (str) Specify a proxy (SOCKS) that RTMP
                                 streams will use

        rtmp-rtmpdump            (str) Specify the location of the
                                 rtmpdump executable used by RTMP streams,
                                 e.g. ``/usr/local/bin/rtmpdump``

        rtmp-timeout             (float) Timeout for reading data from
                                 RTMP streams, default: ``60.0``

        ffmpeg-ffmpeg            (str) Specify the location of the
                                 ffmpeg executable use by Muxing streams
                                 e.g. ``/usr/local/bin/ffmpeg``

        ffmpeg-verbose           (bool) Log stderr from ffmpeg to the
                                 console

        ffmpeg-verbose-path      (str) Specify the location of the
                                 ffmpeg stderr log file

        ffmpeg-video-transcode   (str) The codec to use if transcoding
                                 video when muxing with ffmpeg
                                 e.g. ``h264``

        ffmpeg-audio-transcode   (str) The codec to use if transcoding
                                 audio when muxing with ffmpeg
                                 e.g. ``aac``

        stream-segment-attempts  (int) How many attempts should be done
                                 to download each segment, default: ``3``.
                                 General option used by streams not
                                 covered by other options.

        stream-segment-threads   (int) The size of the thread pool used
                                 to download segments, default: ``1``.
                                 General option used by streams not
                                 covered by other options.

        stream-segment-timeout   (float) Segment connect and read
                                 timeout, default: ``10.0``.
                                 General option used by streams not
                                 covered by other options.

        stream-timeout           (float) Timeout for reading data from
                                 stream, default: ``60.0``.
                                 General option used by streams not
                                 covered by other options.

        locale                   (str) Locale setting, in the RFC 1766 format
                                 eg. en_US or es_ES
                                 default: ``system locale``.
        ======================== =========================================

        """

        # Backwards compatibility
        if key == "rtmpdump":
            key = "rtmp-rtmpdump"
        elif key == "rtmpdump-proxy":
            key = "rtmp-proxy"
        elif key == "errorlog":
            key = "subprocess-errorlog"
        elif key == "errorlog-path":
            key = "subprocess-errorlog-path"

        if key == "http-proxy":
            self.http.proxies["http"] = update_scheme("http://", value)
        elif key == "https-proxy":
            self.http.proxies["https"] = update_scheme("https://", value)
        elif key == "http-cookies":
            if isinstance(value, dict):
                self.http.cookies.update(value)
            else:
                self.http.parse_cookies(value)
        elif key == "http-headers":
            if isinstance(value, dict):
                self.http.headers.update(value)
            else:
                self.http.parse_headers(value)
        elif key == "http-query-params":
            if isinstance(value, dict):
                self.http.params.update(value)
            else:
                self.http.parse_query_params(value)
        elif key == "http-trust-env":
            self.http.trust_env = value
        elif key == "http-ssl-verify":
            self.http.verify = value
        elif key == "http-disable-dh":
            if value:
                requests.packages.urllib3.util.ssl_.DEFAULT_CIPHERS += ':!DH'
                try:
                    requests.packages.urllib3.contrib.pyopenssl.DEFAULT_SSL_CIPHER_LIST = \
                        requests.packages.urllib3.util.ssl_.DEFAULT_CIPHERS.encode("ascii")
                except AttributeError:
                    # no ssl to disable the cipher on
                    pass
        elif key == "http-ssl-cert":
            self.http.cert = value
        elif key == "http-timeout":
            self.http.timeout = value
        else:
            self.options.set(key, value)