def update_qsd(url, qsd=None, remove=None): """ Update or remove keys from a query string in a URL :param url: URL to update :param qsd: dict of keys to update, a None value leaves it unchanged :param remove: list of keys to remove, or "*" to remove all note: updated keys are never removed, even if unchanged :return: updated URL """ qsd = qsd or {} remove = remove or [] # parse current query string parsed = urlparse(url) current_qsd = OrderedDict(parse_qsl(parsed.query)) # * removes all possible keys if remove == "*": remove = list(current_qsd.keys()) # remove keys before updating, but leave updated keys untouched for key in remove: if key not in qsd: del current_qsd[key] # and update the query string for key, value in qsd.items(): if value: current_qsd[key] = value return parsed._replace(query=urlencode(current_qsd)).geturl()
def _get_streams(self): if self.is_live: self.logger.debug("Loading live stream for {0}...", self.channel) res = self.session.http.get(self.live_api_url, data={"r": random.randint(1, 100000)}) live_data = self.session.http.json(res) # all the streams are equal for each type, so pick a random one hls_streams = live_data.get("hls") if hls_streams: url = random.choice(hls_streams) url = url + '&' + urlencode(self.hls_session()) # TODO: use update_qsd for s in HLSStream.parse_variant_playlist(self.session, url, name_fmt="{pixels}_{bitrate}").items(): yield s mpd_streams = live_data.get("mpd") if mpd_streams: url = random.choice(mpd_streams) for s in DASHStream.parse_manifest(self.session, url).items(): yield s elif self.channel == "1tv": self.logger.debug("Attempting to find VOD stream...", self.channel) vod_data = self.vod_data() if vod_data: self.logger.info(u"Found VOD: {0}".format(vod_data[0]['title'])) for stream in vod_data[0]['mbr']: yield stream['name'], HTTPStream(self.session, update_scheme(self.url, stream['src']))
def _get_streams(self): m = self._live_url_re.search(self.page.text) playlist_url = m and update_scheme(self.url, m.group(1)) player_url = self.url token = self._get_token() if playlist_url: log.debug("Found playlist URL in the page") else: live_channel = None for div in itertags(self.page.text, "div"): if div.attributes.get("id") == "botonLive": live_channel = div.attributes.get("data-canal") if live_channel: log.debug("Live channel: {0}".format(live_channel)) player_url = self._channel_urls[live_channel] + quote(token) page = self.session.http.get(player_url, raise_for_status=False) if "block access from your country." in page.text: raise PluginError("Content is geo-locked") m = self._playlist_re.search(page.text) playlist_url = m and update_scheme(self.url, m.group(1)) else: log.error("Could not find the live channel") if playlist_url: stream_url = "{0}?{1}".format(playlist_url, urlencode({"iut": token})) return HLSStream.parse_variant_playlist( self.session, stream_url, headers={"referer": player_url})
def _get_streams(self): if self.is_live: log.debug("Loading live stream for {0}...".format(self.channel)) res = self.session.http.get(self.live_api_url, data={"r": random.randint(1, 100000)}) live_data = self.session.http.json(res) # all the streams are equal for each type, so pick a random one hls_streams = live_data.get("hls") if hls_streams: url = random.choice(hls_streams) url = url + '&' + urlencode(self.hls_session()) # TODO: use update_qsd for s in HLSStream.parse_variant_playlist(self.session, url, name_fmt="{pixels}_{bitrate}").items(): yield s mpd_streams = live_data.get("mpd") if mpd_streams: url = random.choice(mpd_streams) for s in DASHStream.parse_manifest(self.session, url).items(): yield s elif self.channel == "1tv": log.debug("Attempting to find VOD stream for {0}...".format(self.channel)) vod_data = self.vod_data() if vod_data: log.info(u"Found VOD: {0}".format(vod_data[0]['title'])) for stream in vod_data[0]['mbr']: yield stream['name'], HTTPStream(self.session, update_scheme(self.url, stream['src']))
def _get_streams(self): headers = {'User-Agent': CHROME} cookie = urlencode( dict( self.session.http.head(self.url, headers={ 'User-Agent': CHROME }).cookies.items())) headers.update({'Cookie': cookie}) res = self.session.http.get(self.url, headers=headers) tags = list(itertags(res.text, 'script')) text = [i for i in tags if 'OmegaTvLive' in i.text][0].text stream = json.loads(re.search('({.+})', text).group(1))['video']['source']['src'] headers.update({"Referer": self.url}) del headers['Cookie'] try: parse_hls = bool(strtobool(self.get_option('parse_hls'))) except AttributeError: parse_hls = True if parse_hls: return HLSStream.parse_variant_playlist(self.session, stream, headers=headers) else: return dict(live=HTTPStream(self.session, stream, headers=headers))
def _get_streams(self): if self.is_live: self.logger.debug("Loading live stream for {0}...", self.channel) res = http.get(self.live_api_url, data={"r": random.randint(1, 100000)}) live_data = http.json(res) for url in live_data.get( "hls", [])[:1]: # only take the first, they are all the same streams url = url + '&' + urlencode( self.hls_session()) # TODO: use update_qsd for s in HLSStream.parse_variant_playlist( self.session, url, name_fmt="{pixels}_{bitrate}").items(): yield s elif self.channel == "1tv": self.logger.debug("Attempting to find VOD stream...", self.channel) vod_data = self.vod_data() if vod_data: self.logger.info(u"Found VOD: {0}".format( vod_data[0]['title'])) for stream in vod_data[0]['mbr']: yield stream['name'], HTTPStream( self.session, update_scheme(self.url, stream['src']))
def test_extract_nonce(self): mock_nonce = "mock-nonce-nse" mock_response = Response() mock_response.url = "http://example.com/?" + urlencode( dict(nonce=mock_nonce)) self.assertEqual(BBCiPlayer._extract_nonce(mock_response), mock_nonce)
def update_qsd(url, qsd=None, remove=None, keep_blank_values=True, safe="", quote_via=quote_plus): """ Update or remove keys from a query string in a URL :param url: URL to update :param qsd: dict of keys to update, a None value leaves it unchanged :param remove: list of keys to remove, or "*" to remove all note: updated keys are never removed, even if unchanged :param keep_blank_values: whether params with blank values should be kept or not :param safe: string of reserved encoding characters, passed to the quote_via function :param quote_via: function which encodes query string keys and values. Default: urllib.parse.quote_plus :return: updated URL """ qsd = qsd or {} remove = remove or [] # parse current query string parsed = urlparse(url) current_qsd = OrderedDict(parse_qsl(parsed.query, keep_blank_values=True)) # * removes all possible keys if remove == "*": remove = list(current_qsd.keys()) # remove keys before updating, but leave updated keys untouched for key in remove: if key not in qsd: del current_qsd[key] # and update the query string for key, value in qsd.items(): if value is not None: current_qsd[key] = value for key, value in list(current_qsd.items() ): # use list() to create a view of the current_qsd if not value and not keep_blank_values and key not in qsd: del current_qsd[key] if is_py3: query = urlencode(query=current_qsd, safe=safe, quote_via=quote_via) else: def dict2query(d): query = [] for key in d.keys(): query.append("{0}={1}".format(key, d[key])) return "&".join(query) query = quote_via(dict2query(current_qsd), safe="=&" + safe) return parsed._replace(query=query).geturl()
def test_extract_nonce(self): mock_nonce = "mock-nonce-nse" last_response = Response() last_response.request = Request('GET', "http://example.com/?" + urlencode(dict( goto="http://example.com/?" + urlencode(dict( state=json.dumps(dict(nonce=mock_nonce)) )) ))) mock_response = Response() mock_response.history = [ Response(), # Add some extra dummy responses in to make sure we always get the last Response(), last_response ] self.assertEqual(BBCiPlayer._extract_nonce(mock_response), mock_nonce)
def _get_streams(self): self.session.set_option("ffmpeg-start-at-zero", True) self.session.http.headers.update({"Accept-Language": "en-US"}) done = False res = self.session.http.get(self.url) log.trace("{0}".format(res.url)) for title in itertags(res.text, "title"): if title.text.startswith("Log into Facebook"): log.error( "Video is not available, You must log in to continue.") return for s in self._parse_streams(res): done = True yield s if done: return # fallback on to playlist log.debug("Falling back to playlist regex") match = self._playlist_re.search(res.text) playlist = match and match.group(1) if playlist: match = self._plurl_re.search(playlist) if match: url = match.group(1) yield "sd", HTTPStream(self.session, url) return # fallback to tahoe player url log.debug("Falling back to tahoe player") video_id = self._url_re.match(self.url).group("video_id") url = self._TAHOE_URL.format(video_id) data = { "__a": 1, "__pc": self._DEFAULT_PC, "__rev": self._DEFAULT_REV, "fb_dtsg": "", } match = self._pc_re.search(res.text) if match: data["__pc"] = match.group(1) match = self._rev_re.search(res.text) if match: data["__rev"] = match.group(1) match = self._dtsg_re.search(res.text) if match: data["fb_dtsg"] = match.group(1) res = self.session.http.post( url, headers={"Content-Type": "application/x-www-form-urlencoded"}, data=urlencode(data).encode("ascii")) for s in self._parse_streams(res): yield s
def auth_url(self, url): parsed = urlparse(url) path, _ = parsed.path.rsplit("/", 1) token_res = http.get(self.token_url, params=dict(acl=path + "/*")) authparams = http.json(token_res, schema=self.token_schema) existing = dict(parse_qsl(parsed.query)) existing.update(dict(parse_qsl(authparams))) return urlunparse(parsed._replace(query=urlencode(existing)))
def test_extract_nonce(self): mock_nonce = "mock-nonce-nse" last_response = Response() last_response.request = Request( 'GET', "http://example.com/?" + urlencode( dict(goto="http://example.com/?" + urlencode(dict(state=json.dumps(dict( nonce=mock_nonce))))))) mock_response = Response() mock_response.history = [ Response( ), # Add some extra dummy responses in to make sure we always get the last Response(), last_response ] self.assertEqual(BBCiPlayer._extract_nonce(mock_response), mock_nonce)
def auth_url(self, url): parsed = urlparse(url) path, _ = parsed.path.rsplit("/", 1) token_res = self.session.http.get(self.token_url, params=dict(acl=path + "/*")) authparams = self.session.http.json(token_res, schema=self.token_schema) existing = dict(parse_qsl(parsed.query)) existing.update(dict(parse_qsl(authparams))) return urlunparse(parsed._replace(query=urlencode(existing)))
def _live(self, station_id): live_url = 'http://f-radiko.smartstream.ne.jp/{}/_definst_/simul-stream.stream/playlist.m3u8'.format( station_id) token, area_id = self._authorize() lsid = hashlib.md5(str(random.random()).encode('utf-8')).hexdigest() live_params = { 'station_id': station_id, 'l': 15, 'lsid': lsid, 'type': 'b' } url = live_url + '?' + urlencode(live_params) return url, token
def _get_streams(self): self.session.http.headers.update({'User-Agent': useragents.CHROME}) done = False res = self.session.http.get(self.url) for s in self._parse_streams(res): done = True yield s if done: return # fallback on to playlist log.debug("Falling back to playlist regex") match = self._playlist_re.search(res.text) playlist = match and match.group(1) if playlist: match = self._plurl_re.search(playlist) if match: url = match.group(1) yield "sd", HTTPStream(self.session, url) return # fallback to tahoe player url log.debug("Falling back to tahoe player") video_id = self._url_re.match(self.url).group("video_id") url = self._TAHOE_URL.format(video_id) data = { "__a": 1, "__pc": self._DEFAULT_PC, "__rev": self._DEFAULT_REV, "fb_dtsg": "", } match = self._pc_re.search(res.text) if match: data["__pc"] = match.group(1) match = self._rev_re.search(res.text) if match: data["__rev"] = match.group(1) match = self._dtsg_re.search(res.text) if match: data["fb_dtsg"] = match.group(1) res = self.session.http.post( url, headers={"Content-Type": "application/x-www-form-urlencoded"}, data=urlencode(data).encode("ascii")) for s in self._parse_streams(res): yield s
def _timefree(self, station_id, start_at): m3u8_url = 'https://tf-rpaa.smartstream.ne.jp/tf/playlist.m3u8' token, area_id = self._authorize() lsid = hashlib.md5(str(random.random()).encode('utf-8')).hexdigest() end_at = self._get_xml(start_at, station_id) m3u8_params = { 'station_id': station_id, 'start_at': start_at, 'ft': start_at, 'end_at': end_at, 'to': end_at, 'l': 15, 'lsid': lsid, 'type': 'b' } url = m3u8_url + '?' + urlencode(m3u8_params) return url, token
def _get_streams(self): headers = {'User-Agent': CHROME} cookie = urlencode(dict(self.session.http.head(self.url, headers={'User-Agent': CHROME}).cookies.items())) headers.update({'Cookie': cookie}) res = self.session.http.get(self.url, headers=headers) tags = list(itertags(res.text, 'script')) m3u8 = [i for i in tags if i.text.startswith(u'var playerInstance')][0].text stream = re.findall('"(.+?)"', m3u8)[1] headers.update({"Referer": self.url}) del headers['Cookie'] parse_hls = bool(strtobool(self.get_option('parse_hls'))) if parse_hls: return HLSStream.parse_variant_playlist(self.session, stream, headers=headers) else: return dict(live=HTTPStream(self.session, stream, headers=headers))
def update_qsd(url, qsd=None, remove=None, keep_blank_values=True): """ Update or remove keys from a query string in a URL :param url: URL to update :param qsd: dict of keys to update, a None value leaves it unchanged :param remove: list of keys to remove, or "*" to remove all note: updated keys are never removed, even if unchanged :param keep_blank_values: if params with blank values should be kept or not :return: updated URL """ qsd = qsd or {} remove = remove or [] # parse current query string parsed = urlparse(url) current_qsd = OrderedDict(parse_qsl(parsed.query, keep_blank_values=True)) # * removes all possible keys if remove == "*": remove = list(current_qsd.keys()) # remove keys before updating, but leave updated keys untouched for key in remove: if key not in qsd: del current_qsd[key] # and update the query string for key, value in qsd.items(): if value is not None: current_qsd[key] = value for key, value in list(current_qsd.items() ): # use list() to create a view of the current_qsd if not value and not keep_blank_values and key not in qsd: del current_qsd[key] return parsed._replace(query=urlencode(current_qsd)).geturl()
def _get_streams(self): m = self._live_url_re.search(self.page.text) playlist_url = m and update_scheme(self.url, m.group(1)) player_url = self.url live_channel = None p = urlparse(player_url) channelnumber = 0 if p.netloc.endswith("tvc.com.ec"): live_channel = "Canal5" elif p.netloc.endswith("rts.com.ec"): live_channel = "Guayaquil" elif p.netloc.endswith("atv.pe"): if p.path.endswith(("ATVMas", "ATVMas/")): live_channel = "ATVMas" channelnumber = 1 else: live_channel = "ATV" token = self._get_token(channelnumber) log.debug("token {0}".format(token)) if playlist_url: log.debug("Found playlist URL in the page") else: if live_channel: log.debug("Live channel: {0}".format(live_channel)) player_url = self._channel_urls[live_channel] + quote(token) page = self.session.http.get(player_url, raise_for_status=False) if "block access from your country." in page.text: raise PluginError("Content is geo-locked") m = self._playlist_re.search(page.text) playlist_url = m and update_scheme(self.url, m.group(1)) else: log.error("Could not find the live channel") if playlist_url: stream_url = "{0}?{1}".format(playlist_url, urlencode({"iut": token})) return HLSStream.parse_variant_playlist(self.session, stream_url, headers={"referer": player_url})
def _get_streams(self): self.session.set_option("ffmpeg-start-at-zero", True) self.session.http.headers.update({"Accept-Language": "en-US"}) done = False res = self.session.http.get(self.url) log.trace("{0}".format(res.url)) title, canonical, self.title = validate.Schema( validate.parse_html(), validate.union(( validate.xml_xpath_string(".//head/title[1]/text()"), validate.xml_xpath_string( ".//head/meta[@res='canonical'][@href][1]/@href"), validate.xml_xpath_string( ".//head/meta[@property='og:title'][@content][1]/@content" ), ))).validate(res.text) if canonical == "https://www.facebook.com/login/" or "log in" in title.lower( ): log.error( "This URL requires a login or may be accessible from a different IP address." ) return for s in self._parse_streams(res): done = True yield s if done: return # fallback on to playlist log.debug("Falling back to playlist regex") match = self._playlist_re.search(res.text) playlist = match and match.group(1) if playlist: match = self._plurl_re.search(playlist) if match: url = match.group(1) yield "sd", HTTPStream(self.session, url) return # fallback to tahoe player url log.debug("Falling back to tahoe player") video_id = self.match.group("video_id") url = self._TAHOE_URL.format(video_id) data = { "__a": 1, "__pc": self._DEFAULT_PC, "__rev": self._DEFAULT_REV, "fb_dtsg": "", } match = self._pc_re.search(res.text) if match: data["__pc"] = match.group(1) match = self._rev_re.search(res.text) if match: data["__rev"] = match.group(1) match = self._dtsg_re.search(res.text) if match: data["fb_dtsg"] = match.group(1) res = self.session.http.post( url, headers={"Content-Type": "application/x-www-form-urlencoded"}, data=urlencode(data).encode("ascii")) for s in self._parse_streams(res): yield s