def update_qsd(url, qsd=None, remove=None):
"""
Update or remove keys from a query string in a URL
:param url: URL to update
:param qsd: dict of keys to update, a None value leaves it unchanged
:param remove: list of keys to remove, or "*" to remove all
note: updated keys are never removed, even if unchanged
:return: updated URL
"""
qsd = qsd or {}
remove = remove or []
# parse current query string
parsed = urlparse(url)
current_qsd = OrderedDict(parse_qsl(parsed.query))
# * removes all possible keys
if remove == "*":
remove = list(current_qsd.keys())
# remove keys before updating, but leave updated keys untouched
for key in remove:
if key not in qsd:
del current_qsd[key]
# and update the query string
for key, value in qsd.items():
if value:
current_qsd[key] = value
return parsed._replace(query=urlencode(current_qsd)).geturl()
def _get_streams(self):
if self.is_live:
self.logger.debug("Loading live stream for {0}...", self.channel)
res = self.session.http.get(self.live_api_url, data={"r": random.randint(1, 100000)})
live_data = self.session.http.json(res)
# all the streams are equal for each type, so pick a random one
hls_streams = live_data.get("hls")
if hls_streams:
url = random.choice(hls_streams)
url = url + '&' + urlencode(self.hls_session()) # TODO: use update_qsd
for s in HLSStream.parse_variant_playlist(self.session, url, name_fmt="{pixels}_{bitrate}").items():
yield s
mpd_streams = live_data.get("mpd")
if mpd_streams:
url = random.choice(mpd_streams)
for s in DASHStream.parse_manifest(self.session, url).items():
yield s
elif self.channel == "1tv":
self.logger.debug("Attempting to find VOD stream...", self.channel)
vod_data = self.vod_data()
if vod_data:
self.logger.info(u"Found VOD: {0}".format(vod_data[0]['title']))
for stream in vod_data[0]['mbr']:
yield stream['name'], HTTPStream(self.session, update_scheme(self.url, stream['src']))
def _get_streams(self):
m = self._live_url_re.search(self.page.text)
playlist_url = m and update_scheme(self.url, m.group(1))
player_url = self.url
token = self._get_token()
if playlist_url:
log.debug("Found playlist URL in the page")
else:
live_channel = None
for div in itertags(self.page.text, "div"):
if div.attributes.get("id") == "botonLive":
live_channel = div.attributes.get("data-canal")
if live_channel:
log.debug("Live channel: {0}".format(live_channel))
player_url = self._channel_urls[live_channel] + quote(token)
page = self.session.http.get(player_url,
raise_for_status=False)
if "block access from your country." in page.text:
raise PluginError("Content is geo-locked")
m = self._playlist_re.search(page.text)
playlist_url = m and update_scheme(self.url, m.group(1))
else:
log.error("Could not find the live channel")
if playlist_url:
stream_url = "{0}?{1}".format(playlist_url,
urlencode({"iut": token}))
return HLSStream.parse_variant_playlist(
self.session, stream_url, headers={"referer": player_url})
def _get_streams(self):
if self.is_live:
log.debug("Loading live stream for {0}...".format(self.channel))
res = self.session.http.get(self.live_api_url, data={"r": random.randint(1, 100000)})
live_data = self.session.http.json(res)
# all the streams are equal for each type, so pick a random one
hls_streams = live_data.get("hls")
if hls_streams:
url = random.choice(hls_streams)
url = url + '&' + urlencode(self.hls_session()) # TODO: use update_qsd
for s in HLSStream.parse_variant_playlist(self.session, url, name_fmt="{pixels}_{bitrate}").items():
yield s
mpd_streams = live_data.get("mpd")
if mpd_streams:
url = random.choice(mpd_streams)
for s in DASHStream.parse_manifest(self.session, url).items():
yield s
elif self.channel == "1tv":
log.debug("Attempting to find VOD stream for {0}...".format(self.channel))
vod_data = self.vod_data()
if vod_data:
log.info(u"Found VOD: {0}".format(vod_data[0]['title']))
for stream in vod_data[0]['mbr']:
yield stream['name'], HTTPStream(self.session, update_scheme(self.url, stream['src']))
def _get_streams(self):
headers = {'User-Agent': CHROME}
cookie = urlencode(
dict(
self.session.http.head(self.url,
headers={
'User-Agent': CHROME
}).cookies.items()))
headers.update({'Cookie': cookie})
res = self.session.http.get(self.url, headers=headers)
tags = list(itertags(res.text, 'script'))
text = [i for i in tags if 'OmegaTvLive' in i.text][0].text
stream = json.loads(re.search('({.+})',
text).group(1))['video']['source']['src']
headers.update({"Referer": self.url})
del headers['Cookie']
try:
parse_hls = bool(strtobool(self.get_option('parse_hls')))
except AttributeError:
parse_hls = True
if parse_hls:
return HLSStream.parse_variant_playlist(self.session,
stream,
headers=headers)
else:
return dict(live=HTTPStream(self.session, stream, headers=headers))
def _get_streams(self):
if self.is_live:
self.logger.debug("Loading live stream for {0}...", self.channel)
res = http.get(self.live_api_url,
data={"r": random.randint(1, 100000)})
live_data = http.json(res)
for url in live_data.get(
"hls",
[])[:1]: # only take the first, they are all the same streams
url = url + '&' + urlencode(
self.hls_session()) # TODO: use update_qsd
for s in HLSStream.parse_variant_playlist(
self.session, url,
name_fmt="{pixels}_{bitrate}").items():
yield s
elif self.channel == "1tv":
self.logger.debug("Attempting to find VOD stream...", self.channel)
vod_data = self.vod_data()
if vod_data:
self.logger.info(u"Found VOD: {0}".format(
vod_data[0]['title']))
for stream in vod_data[0]['mbr']:
yield stream['name'], HTTPStream(
self.session, update_scheme(self.url, stream['src']))
def test_extract_nonce(self):
mock_nonce = "mock-nonce-nse"
mock_response = Response()
mock_response.url = "http://example.com/?" + urlencode(
dict(nonce=mock_nonce))
self.assertEqual(BBCiPlayer._extract_nonce(mock_response), mock_nonce)
def update_qsd(url,
qsd=None,
remove=None,
keep_blank_values=True,
safe="",
quote_via=quote_plus):
"""
Update or remove keys from a query string in a URL
:param url: URL to update
:param qsd: dict of keys to update, a None value leaves it unchanged
:param remove: list of keys to remove, or "*" to remove all
note: updated keys are never removed, even if unchanged
:param keep_blank_values: whether params with blank values should be kept or not
:param safe: string of reserved encoding characters, passed to the quote_via function
:param quote_via: function which encodes query string keys and values. Default: urllib.parse.quote_plus
:return: updated URL
"""
qsd = qsd or {}
remove = remove or []
# parse current query string
parsed = urlparse(url)
current_qsd = OrderedDict(parse_qsl(parsed.query, keep_blank_values=True))
# * removes all possible keys
if remove == "*":
remove = list(current_qsd.keys())
# remove keys before updating, but leave updated keys untouched
for key in remove:
if key not in qsd:
del current_qsd[key]
# and update the query string
for key, value in qsd.items():
if value is not None:
current_qsd[key] = value
for key, value in list(current_qsd.items()
): # use list() to create a view of the current_qsd
if not value and not keep_blank_values and key not in qsd:
del current_qsd[key]
if is_py3:
query = urlencode(query=current_qsd, safe=safe, quote_via=quote_via)
else:
def dict2query(d):
query = []
for key in d.keys():
query.append("{0}={1}".format(key, d[key]))
return "&".join(query)
query = quote_via(dict2query(current_qsd), safe="=&" + safe)
return parsed._replace(query=query).geturl()
def test_extract_nonce(self):
mock_nonce = "mock-nonce-nse"
last_response = Response()
last_response.request = Request('GET', "http://example.com/?" + urlencode(dict(
goto="http://example.com/?" + urlencode(dict(
state=json.dumps(dict(nonce=mock_nonce))
))
)))
mock_response = Response()
mock_response.history = [
Response(), # Add some extra dummy responses in to make sure we always get the last
Response(),
last_response
]
self.assertEqual(BBCiPlayer._extract_nonce(mock_response), mock_nonce)
def _get_streams(self):
self.session.set_option("ffmpeg-start-at-zero", True)
self.session.http.headers.update({"Accept-Language": "en-US"})
done = False
res = self.session.http.get(self.url)
log.trace("{0}".format(res.url))
for title in itertags(res.text, "title"):
if title.text.startswith("Log into Facebook"):
log.error(
"Video is not available, You must log in to continue.")
return
for s in self._parse_streams(res):
done = True
yield s
if done:
return
# fallback on to playlist
log.debug("Falling back to playlist regex")
match = self._playlist_re.search(res.text)
playlist = match and match.group(1)
if playlist:
match = self._plurl_re.search(playlist)
if match:
url = match.group(1)
yield "sd", HTTPStream(self.session, url)
return
# fallback to tahoe player url
log.debug("Falling back to tahoe player")
video_id = self._url_re.match(self.url).group("video_id")
url = self._TAHOE_URL.format(video_id)
data = {
"__a": 1,
"__pc": self._DEFAULT_PC,
"__rev": self._DEFAULT_REV,
"fb_dtsg": "",
}
match = self._pc_re.search(res.text)
if match:
data["__pc"] = match.group(1)
match = self._rev_re.search(res.text)
if match:
data["__rev"] = match.group(1)
match = self._dtsg_re.search(res.text)
if match:
data["fb_dtsg"] = match.group(1)
res = self.session.http.post(
url,
headers={"Content-Type": "application/x-www-form-urlencoded"},
data=urlencode(data).encode("ascii"))
for s in self._parse_streams(res):
yield s
def auth_url(self, url):
parsed = urlparse(url)
path, _ = parsed.path.rsplit("/", 1)
token_res = http.get(self.token_url, params=dict(acl=path + "/*"))
authparams = http.json(token_res, schema=self.token_schema)
existing = dict(parse_qsl(parsed.query))
existing.update(dict(parse_qsl(authparams)))
return urlunparse(parsed._replace(query=urlencode(existing)))
def test_extract_nonce(self):
mock_nonce = "mock-nonce-nse"
last_response = Response()
last_response.request = Request(
'GET', "http://example.com/?" + urlencode(
dict(goto="http://example.com/?" +
urlencode(dict(state=json.dumps(dict(
nonce=mock_nonce)))))))
mock_response = Response()
mock_response.history = [
Response(
), # Add some extra dummy responses in to make sure we always get the last
Response(),
last_response
]
self.assertEqual(BBCiPlayer._extract_nonce(mock_response), mock_nonce)
def auth_url(self, url):
parsed = urlparse(url)
path, _ = parsed.path.rsplit("/", 1)
token_res = self.session.http.get(self.token_url, params=dict(acl=path + "/*"))
authparams = self.session.http.json(token_res, schema=self.token_schema)
existing = dict(parse_qsl(parsed.query))
existing.update(dict(parse_qsl(authparams)))
return urlunparse(parsed._replace(query=urlencode(existing)))
def _live(self, station_id):
live_url = 'http://f-radiko.smartstream.ne.jp/{}/_definst_/simul-stream.stream/playlist.m3u8'.format(
station_id)
token, area_id = self._authorize()
lsid = hashlib.md5(str(random.random()).encode('utf-8')).hexdigest()
live_params = {
'station_id': station_id,
'l': 15,
'lsid': lsid,
'type': 'b'
}
url = live_url + '?' + urlencode(live_params)
return url, token
def _get_streams(self):
self.session.http.headers.update({'User-Agent': useragents.CHROME})
done = False
res = self.session.http.get(self.url)
for s in self._parse_streams(res):
done = True
yield s
if done:
return
# fallback on to playlist
log.debug("Falling back to playlist regex")
match = self._playlist_re.search(res.text)
playlist = match and match.group(1)
if playlist:
match = self._plurl_re.search(playlist)
if match:
url = match.group(1)
yield "sd", HTTPStream(self.session, url)
return
# fallback to tahoe player url
log.debug("Falling back to tahoe player")
video_id = self._url_re.match(self.url).group("video_id")
url = self._TAHOE_URL.format(video_id)
data = {
"__a": 1,
"__pc": self._DEFAULT_PC,
"__rev": self._DEFAULT_REV,
"fb_dtsg": "",
}
match = self._pc_re.search(res.text)
if match:
data["__pc"] = match.group(1)
match = self._rev_re.search(res.text)
if match:
data["__rev"] = match.group(1)
match = self._dtsg_re.search(res.text)
if match:
data["fb_dtsg"] = match.group(1)
res = self.session.http.post(
url,
headers={"Content-Type": "application/x-www-form-urlencoded"},
data=urlencode(data).encode("ascii"))
for s in self._parse_streams(res):
yield s
def _timefree(self, station_id, start_at):
m3u8_url = 'https://tf-rpaa.smartstream.ne.jp/tf/playlist.m3u8'
token, area_id = self._authorize()
lsid = hashlib.md5(str(random.random()).encode('utf-8')).hexdigest()
end_at = self._get_xml(start_at, station_id)
m3u8_params = {
'station_id': station_id,
'start_at': start_at,
'ft': start_at,
'end_at': end_at,
'to': end_at,
'l': 15,
'lsid': lsid,
'type': 'b'
}
url = m3u8_url + '?' + urlencode(m3u8_params)
return url, token
def _get_streams(self):
headers = {'User-Agent': CHROME}
cookie = urlencode(dict(self.session.http.head(self.url, headers={'User-Agent': CHROME}).cookies.items()))
headers.update({'Cookie': cookie})
res = self.session.http.get(self.url, headers=headers)
tags = list(itertags(res.text, 'script'))
m3u8 = [i for i in tags if i.text.startswith(u'var playerInstance')][0].text
stream = re.findall('"(.+?)"', m3u8)[1]
headers.update({"Referer": self.url})
del headers['Cookie']
parse_hls = bool(strtobool(self.get_option('parse_hls')))
if parse_hls:
return HLSStream.parse_variant_playlist(self.session, stream, headers=headers)
else:
return dict(live=HTTPStream(self.session, stream, headers=headers))
def update_qsd(url, qsd=None, remove=None, keep_blank_values=True):
"""
Update or remove keys from a query string in a URL
:param url: URL to update
:param qsd: dict of keys to update, a None value leaves it unchanged
:param remove: list of keys to remove, or "*" to remove all
note: updated keys are never removed, even if unchanged
:param keep_blank_values: if params with blank values should be kept or not
:return: updated URL
"""
qsd = qsd or {}
remove = remove or []
# parse current query string
parsed = urlparse(url)
current_qsd = OrderedDict(parse_qsl(parsed.query, keep_blank_values=True))
# * removes all possible keys
if remove == "*":
remove = list(current_qsd.keys())
# remove keys before updating, but leave updated keys untouched
for key in remove:
if key not in qsd:
del current_qsd[key]
# and update the query string
for key, value in qsd.items():
if value is not None:
current_qsd[key] = value
for key, value in list(current_qsd.items()
): # use list() to create a view of the current_qsd
if not value and not keep_blank_values and key not in qsd:
del current_qsd[key]
return parsed._replace(query=urlencode(current_qsd)).geturl()
def _get_streams(self):
m = self._live_url_re.search(self.page.text)
playlist_url = m and update_scheme(self.url, m.group(1))
player_url = self.url
live_channel = None
p = urlparse(player_url)
channelnumber = 0
if p.netloc.endswith("tvc.com.ec"):
live_channel = "Canal5"
elif p.netloc.endswith("rts.com.ec"):
live_channel = "Guayaquil"
elif p.netloc.endswith("atv.pe"):
if p.path.endswith(("ATVMas", "ATVMas/")):
live_channel = "ATVMas"
channelnumber = 1
else:
live_channel = "ATV"
token = self._get_token(channelnumber)
log.debug("token {0}".format(token))
if playlist_url:
log.debug("Found playlist URL in the page")
else:
if live_channel:
log.debug("Live channel: {0}".format(live_channel))
player_url = self._channel_urls[live_channel] + quote(token)
page = self.session.http.get(player_url, raise_for_status=False)
if "block access from your country." in page.text:
raise PluginError("Content is geo-locked")
m = self._playlist_re.search(page.text)
playlist_url = m and update_scheme(self.url, m.group(1))
else:
log.error("Could not find the live channel")
if playlist_url:
stream_url = "{0}?{1}".format(playlist_url, urlencode({"iut": token}))
return HLSStream.parse_variant_playlist(self.session, stream_url, headers={"referer": player_url})
def _get_streams(self):
self.session.set_option("ffmpeg-start-at-zero", True)
self.session.http.headers.update({"Accept-Language": "en-US"})
done = False
res = self.session.http.get(self.url)
log.trace("{0}".format(res.url))
title, canonical, self.title = validate.Schema(
validate.parse_html(),
validate.union((
validate.xml_xpath_string(".//head/title[1]/text()"),
validate.xml_xpath_string(
".//head/meta[@res='canonical'][@href][1]/@href"),
validate.xml_xpath_string(
".//head/meta[@property='og:title'][@content][1]/@content"
),
))).validate(res.text)
if canonical == "https://www.facebook.com/login/" or "log in" in title.lower(
):
log.error(
"This URL requires a login or may be accessible from a different IP address."
)
return
for s in self._parse_streams(res):
done = True
yield s
if done:
return
# fallback on to playlist
log.debug("Falling back to playlist regex")
match = self._playlist_re.search(res.text)
playlist = match and match.group(1)
if playlist:
match = self._plurl_re.search(playlist)
if match:
url = match.group(1)
yield "sd", HTTPStream(self.session, url)
return
# fallback to tahoe player url
log.debug("Falling back to tahoe player")
video_id = self.match.group("video_id")
url = self._TAHOE_URL.format(video_id)
data = {
"__a": 1,
"__pc": self._DEFAULT_PC,
"__rev": self._DEFAULT_REV,
"fb_dtsg": "",
}
match = self._pc_re.search(res.text)
if match:
data["__pc"] = match.group(1)
match = self._rev_re.search(res.text)
if match:
data["__rev"] = match.group(1)
match = self._dtsg_re.search(res.text)
if match:
data["fb_dtsg"] = match.group(1)
res = self.session.http.post(
url,
headers={"Content-Type": "application/x-www-form-urlencoded"},
data=urlencode(data).encode("ascii"))
for s in self._parse_streams(res):
yield s
