def test_parse_json(self):
self.assertEqual({}, parse_json("{}"))
self.assertEqual({"test": 1}, parse_json("""{"test": 1}"""))
self.assertEqual({"test": 1}, parse_json("""{"test": 1}""", schema=validate.Schema({"test": 1})))
self.assertRaises(PluginError, parse_json, """{"test: 1}""")
self.assertRaises(IOError, parse_json, """{"test: 1}""", exception=IOError)
self.assertRaises(PluginError, parse_json, """{"test: 1}""" * 10)
def _get_streams(self):
if "eltrecetv.com.ar/vivo" in self.url.lower():
try:
http.headers = {'Referer': self.url,
'User-Agent': useragents.ANDROID}
res = http.get('https://api.iamat.com/metadata/atcodes/eltrece')
yt_id = parse_json(res.text)["atcodes"][0]["context"]["ahora"]["vivo"]["youtubeVideo"]
yt_url = "https://www.youtube.com/watch?v={0}".format(yt_id)
return self.session.streams(yt_url)
except BaseException:
self.logger.info("Live content is temporarily unavailable. Please try again later.")
else:
try:
http.headers = {'Referer': self.url,
'User-Agent': useragents.CHROME}
res = http.get(self.url)
_player_re = re.compile(r'''data-kaltura="([^"]+)"''')
match = _player_re.search(res.text)
if not match:
return
entry_id = parse_json(match.group(1).replace(""", '"'))["entryId"]
hls_url = "https://vodgc.com/p/111/sp/11100/playManifest/entryId/{0}/format/applehttp/protocol/https/a.m3u8".format(entry_id)
return HLSStream.parse_variant_playlist(self.session, hls_url)
except BaseException:
self.logger.error("The requested VOD content is unavailable.")
def _get_streams(self):
"""
Find the streams for web.tv
:return:
"""
headers = {}
res = self.session.http.get(self.url, headers=headers)
headers["Referer"] = self.url
sources = self._sources_re.findall(res.text)
if len(sources):
sdata = parse_json(sources[0], schema=self._sources_schema)
for source in sdata:
self.logger.debug("Found stream of type: {}", source[u'type'])
if source[u'type'] == u"application/vnd.apple.mpegurl":
url = update_scheme(self.url, source[u"src"])
try:
# try to parse the stream as a variant playlist
variant = HLSStream.parse_variant_playlist(self.session, url, headers=headers)
if variant:
for q, s in variant.items():
yield q, s
else:
# and if that fails, try it as a plain HLS stream
yield 'live', HLSStream(self.session, url, headers=headers)
except IOError:
self.logger.warning("Could not open the stream, perhaps the channel is offline")
def _get_vod_stream(self):
vod_url = self.url
if vod_url.endswith('/'):
vod_url = vod_url[:-1]
json_url = '{0}.securevideo.json'.format(vod_url)
res = http.get(json_url)
match = _json_re.search(res.text)
if not match:
return
data = parse_json(match.group(1))
res = http.get(API_VOD.format(data['clientid'], data['mzid']))
data = http.json(res, schema=_stream_schema)
for d in data['targetUrls']:
if d['type'] == 'HDS':
hds_url = d['url']
for s in HDSStream.parse_manifest(self.session, hds_url).items():
yield s
if d['type'] == 'HLS':
hls_url = d['url']
for s in HLSStream.parse_variant_playlist(self.session, hls_url).items():
yield s
def find_vpid(self, url, res=None):
self.logger.debug("Looking for vpid on {0}", url)
# Use pre-fetched page if available
res = res or http.get(url)
m = self.mediator_re.search(res.text)
vpid = m and parse_json(m.group(1), schema=self.mediator_schema)
return vpid
def _find_video_id(self, url):
m = _url_re.match(url)
if m.group("video_id"):
log.debug("Video ID from URL")
return m.group("video_id")
res = self.session.http.get(url)
datam = _ytdata_re.search(res.text)
if datam:
data = parse_json(datam.group(1))
# find the videoRenderer object, where there is a LVE NOW badge
for vid_ep in search_dict(data, 'currentVideoEndpoint'):
video_id = vid_ep.get("watchEndpoint", {}).get("videoId")
if video_id:
log.debug("Video ID from currentVideoEndpoint")
return video_id
for x in search_dict(data, 'videoRenderer'):
for bstyle in search_dict(x.get("badges", {}), "style"):
if bstyle == "BADGE_STYLE_TYPE_LIVE_NOW":
if x.get("videoId"):
log.debug("Video ID from videoRenderer (live)")
return x["videoId"]
if "/embed/live_stream" in url:
for link in itertags(res.text, "link"):
if link.attributes.get("rel") == "canonical":
canon_link = link.attributes.get("href")
if canon_link != url:
log.debug("Re-directing to canonical URL: {0}".format(canon_link))
return self._find_video_id(canon_link)
raise PluginError("Could not find a video on this page")
def _get_streams(self):
res = self.session.http.get(self.url, headers={"User-Agent": useragents.CHROME})
streams = {}
vod_urls = set([])
for match in self._src_re.finditer(res.text):
stream_url = match.group("url")
if "\\/" in stream_url:
# if the URL is json encoded, decode it
stream_url = parse_json("\"{}\"".format(stream_url))
if ".mpd" in stream_url:
streams.update(DASHStream.parse_manifest(self.session, stream_url))
elif ".mp4" in stream_url:
streams[match.group(1)] = HTTPStream(self.session, stream_url)
vod_urls.add(stream_url)
else:
self.logger.debug("Non-dash/mp4 stream: {0}".format(stream_url))
if streams:
return streams
# fallback on to playlist
self.logger.debug("Falling back to playlist regex")
match = self._playlist_re.search(res.text)
playlist = match and match.group(1)
if playlist:
for url in dict(url.group(1) for url in self._plurl_re.finditer(playlist)):
if url not in vod_urls:
streams["sd"] = HTTPStream(self.session, url)
return streams
def _get_streams(self):
res = http.get(self.url)
match = _info_re.search(res.text)
if not match:
return
info = parse_json(match.group(1), schema=_schema)
stream_name = info["mode"]
mp4_url = info.get("mp4_url")
ios_url = info.get("ios_url")
swf_url = info.get("swf_url")
if mp4_url:
stream = HTTPStream(self.session, mp4_url)
yield stream_name, stream
if ios_url:
if urlparse(ios_url).path.endswith(".m3u8"):
streams = HLSStream.parse_variant_playlist(self.session, ios_url)
# TODO: Replace with "yield from" when dropping Python 2.
for stream in streams.items():
yield stream
if swf_url:
stream = self._get_rtmp_stream(swf_url)
if stream:
yield stream_name, stream
def _get_streams(self):
headers = {
"Referer": self.url
}
res = self.session.http.get(self.url, headers=headers)
match = _ddos_re.search(res.text)
if match:
self.logger.debug("Anti-DDOS bypass...")
headers["Cookie"] = match.group(1)
res = self.session.http.get(self.url, headers=headers)
match = _apidata_re.search(res.text)
channel_info = match and parse_json(match.group("data"))
if not channel_info:
self.logger.error("Could not find channel info")
return
self.logger.debug("Found channel info: channelkey={channelkey} pid={streamkey} online={status}",
**channel_info)
if not channel_info['status']:
self.logger.debug("Channel appears to be offline")
streams = {}
for name, url_suffix in QUALITIES.items():
url = HLS_URL_FORMAT.format(channel_info['streamkey'], url_suffix)
if not self._check_stream(url):
continue
streams[name] = HLSStream(self.session, url)
return streams
def _get_movie_data(self):
pres = self._get_page()
match = self._stores_re.search(pres.text)
if match:
self._pdata = parse_json(match.group(1), schema=self._stores_schema)
return self._pdata
def _get_page_config(self):
pres = self._get_page()
match = self._config_re.search(pres.text)
if match:
self._pconfig = parse_json(match.group(1))
return self._pconfig
def _get_streams(self):
res = self.session.http.get(self.url, headers={'User-Agent': useragents.CHROME})
video_search = res.text
video_search = video_search[video_search.index('{"top":{"view":"PlayerContainer","model":{'):]
video_search = video_search[: video_search.index('}]}}') + 4] + "}"
video_url_found_hls = ""
video_url_found_http = ""
json_video_search = parse_json(video_search)
json_video_search_sources = json_video_search["top"]["model"]["videos"][0]["sources"]
self.logger.debug('Video ID found: {0}', json_video_search["top"]["model"]["id"])
for current_video_source in json_video_search_sources:
if "HLS" in current_video_source["type"]:
video_url_found_hls = "http://telefe.com" + current_video_source["url"]
self.logger.debug("HLS content available")
if "HTTP" in current_video_source["type"]:
video_url_found_http = "http://telefe.com" + current_video_source["url"]
self.logger.debug("HTTP content available")
self.session.http.headers = {'Referer': self.url,
'User-Agent': useragents.CHROME,
'X-Requested-With': 'ShockwaveFlash/25.0.0.148'}
if video_url_found_hls:
hls_streams = HLSStream.parse_variant_playlist(self.session, video_url_found_hls)
for s in hls_streams.items():
yield s
if video_url_found_http:
yield "http", HTTPStream(self.session, video_url_found_http)
def get_pdata(self, channel):
"""
Get the params for the post request
:param channel: channel name
:return: "gcp" and "ogn"
"""
res = http.get(self.pdata_url.format(channel=channel))
return parse_json(res.text, schema=self.pdata_schema)
def _get_live_stream(self, channel):
channel = 'vualto_{0}'.format(channel)
_live_json_re = re.compile(r'''"{0}":\s(\173[^\173\175]+\175)'''.format(channel))
res = http.get(API_LIVE)
match = _live_json_re.search(res.text)
if not match:
return
data = parse_json(match.group(1))
hls_url = data['hls']
if hls_url:
for s in HLSStream.parse_variant_playlist(self.session, hls_url).items():
yield s
def _get_live_stream(self, channel):
channel = 'vualto_{0}'.format(channel)
_live_json_re = re.compile(
r'''"{0}":\s(\173[^\173\175]+\175)'''.format(channel))
res = http.get(API_LIVE)
match = _live_json_re.search(res.text)
if not match:
return
data = parse_json(match.group(1))
hls_url = data['hls']
if hls_url:
for s in HLSStream.parse_variant_playlist(self.session,
hls_url).items():
yield s
def _get_streams(self):
page = self.session.http.get(self.url)
m = self._source_re.search(page.text)
if m:
params = ""
data = m.group(1)
log.debug("Source data: {0}".format(data))
if "location.hash.substring" in data:
log.debug("Removing hash substring addition")
data = re.sub(r"\s*\+\s*location.hash.substring\(\d+\)", "", data)
params = urlparse(self.url).fragment
data = self.js_to_json(data)
for stream in parse_json(data):
for s in HLSStream.parse_variant_playlist(self.session, stream['file'], params=params).items():
yield s
else:
log.debug("No match for sources regex")
def find_vpid(self, url, res=None):
"""
Find the Video Packet ID in the HTML for the provided URL
:param url: URL to download, if res is not provided.
:param res: Provide a cached version of the HTTP response to search
:type url: string
:type res: requests.Response
:return: Video Packet ID for a Programme in iPlayer
:rtype: string
"""
log.debug("Looking for vpid on {0}", url)
# Use pre-fetched page if available
res = res or self.session.http.get(url)
m = self.mediator_re.search(res.text)
vpid = m and parse_json(m.group(1), schema=self.mediator_schema)
return vpid
def find_vpid(self, url, res=None):
"""
Find the Video Packet ID in the HTML for the provided URL
:param url: URL to download, if res is not provided.
:param res: Provide a cached version of the HTTP response to search
:type url: string
:type res: requests.Response
:return: Video Packet ID for a Programme in iPlayer
:rtype: string
"""
self.logger.debug("Looking for vpid on {0}", url)
# Use pre-fetched page if available
res = res or http.get(url)
m = self.mediator_re.search(res.text)
vpid = m and parse_json(m.group(1), schema=self.mediator_schema)
return vpid
def _get_streams(self):
res = self.session.http.get(self.url)
data = self._re_stream.search(res.text)
if not data:
return
data = parse_json(base64.b64decode(data.group(1)),
schema=self._schema_data)
for info in data:
log.trace(f'{info!r}')
flv_url = f'{info["sFlvUrl"]}/{info["sStreamName"]}.{info["sFlvUrlSuffix"]}?{info["sFlvAntiCode"]}'
name = f'source_{info["sCdnType"].lower()}'
self.QUALITY_WEIGHTS[name] = info['iPCPriorityRate']
yield name, HTTPStream(self.session, flv_url)
log.debug(f'QUALITY_WEIGHTS: {self.QUALITY_WEIGHTS!r}')
def get_stream(self, slug, js_data):
token_data = {}
token_data['token'] = self.cache.get('token')
token_data['token_type'] = self.cache.get('token_type')
if token_data['token'] and token_data['token_type']:
log.debug('Using cached token')
else:
log.debug('Getting new token')
res = self.session.http.post(
self.login_url,
json=js_data['credentials'],
)
token_data = self.session.http.json(res, schema=self.token_schema)
log.debug('Token={0}'.format(token_data["token"]))
self.cache.set('token', token_data['token'], expires=token_data['expires_in'])
self.cache.set('token_type', token_data['token_type'], expires=token_data['expires_in'])
headers = {'Authorization': '{0} {1}'.format(token_data["token_type"], token_data["token"])}
data = {
'slug': slug,
'type': js_data['type'],
}
res = self.session.http.post(
self.stream_url,
headers=headers,
json=data,
)
encrypted_data = self.session.http.json(
res,
schema=self.encrypted_data_schema
)
stream_data = parse_json(
self.decrypt_data(js_data['cipher_data'], encrypted_data),
schema=self.stream_schema,
)
self.author = stream_data['channel_name']
self.category = stream_data['genres']
self.title = stream_data['meta_title']
return stream_data['live_stream_url']
def _parse_streams(self, res):
_found_stream_url = False
for meta in itertags(res.text, "meta"):
if meta.attributes.get("property") == "og:video:url":
stream_url = html_unescape(meta.attributes.get("content"))
if ".mpd" in stream_url:
for s in DASHStream.parse_manifest(self.session,
stream_url).items():
yield s
_found_stream_url = True
elif ".mp4" in stream_url:
yield "vod", HTTPStream(self.session, stream_url)
_found_stream_url = True
break
else:
log.debug("No meta og:video:url")
if _found_stream_url:
return
for match in self._src_re.finditer(res.text):
stream_url = match.group("url")
if "\\/" in stream_url:
# if the URL is json encoded, decode it
stream_url = parse_json("\"{}\"".format(stream_url))
if ".mpd" in stream_url:
yield from DASHStream.parse_manifest(self.session,
stream_url).items()
elif ".mp4" in stream_url:
yield match.group(1), HTTPStream(self.session, stream_url)
else:
log.debug("Non-dash/mp4 stream: {0}".format(stream_url))
match = self._dash_manifest_re.search(res.text)
if match:
# facebook replaces "<" characters with the substring "\\x3C"
manifest = match.group("manifest").replace("\\/", "/")
manifest = bytes(unquote_plus(manifest),
"utf-8").decode("unicode_escape")
# Ignore unsupported manifests until DASH SegmentBase support is implemented
if "SegmentBase" in manifest:
log.error("Skipped DASH manifest with SegmentBase streams")
else:
yield from DASHStream.parse_manifest(self.session,
manifest).items()
def _get_video_data(self, slug):
m = self.truncate_url_re.search(self.url)
if m and m.group(1):
log.debug("Truncated URL={0}".format(m.group(1)))
else:
raise PluginError("Failed to truncate URL")
res = self.session.http.get(m.group(1))
m = self.json_data_re.search(res.text)
if m and m.group(1):
videos = parse_json(m.group(1), schema=self._video_data_schema)
else:
raise PluginError("Failed to get json_data")
for video in videos:
if 'slug' in videos[video]:
if slug == videos[video]['slug'] and 'id' in videos[video]:
return videos[video]['id']
def _get_prid(self, subtitles=False):
res = http.get(self.url)
bprid = None
# Locate the asset id for the content on the page
for alt, _, prid in self.prid_re.findall(res.text):
if alt and subtitles:
bprid = prid
elif bprid is None:
bprid = prid
if bprid is None:
m = self.react_re.search(res.text)
if m:
data = parse_json(m.group("data").replace(""", '"'))
bprid = data.get("mid")
return bprid
def _find_video_id(self, url):
m = _url_re.match(url)
if m.group("video_id"):
log.debug("Video ID from URL")
return m.group("video_id")
res = self.session.http.get(url)
datam = _ytdata_re.search(res.text)
if datam:
data = parse_json(datam.group(1))
# find the videoRenderer object, where there is a LVE NOW badge
for vid_ep in search_dict(data, 'currentVideoEndpoint'):
video_id = vid_ep.get("watchEndpoint", {}).get("videoId")
if video_id:
log.debug("Video ID from currentVideoEndpoint")
return video_id
for x in search_dict(data, 'videoRenderer'):
if x.get("viewCountText", {}).get("runs"):
if x.get("videoId"):
log.debug("Video ID from videoRenderer (live)")
return x["videoId"]
for bstyle in search_dict(x.get("badges", {}), "style"):
if bstyle == "BADGE_STYLE_TYPE_LIVE_NOW":
if x.get("videoId"):
log.debug("Video ID from videoRenderer (live)")
return x["videoId"]
if "/embed/live_stream" in url:
for link in itertags(res.text, "link"):
if link.attributes.get("rel") == "canonical":
canon_link = link.attributes.get("href")
if canon_link != url:
if canon_link.endswith("v=live_stream"):
log.debug("The video is not available")
break
else:
log.debug(
"Re-directing to canonical URL: {0}".format(
canon_link))
return self._find_video_id(canon_link)
raise PluginError("Could not find a video on this page")
def _get_streams(self):
page = http.get(self.url)
m = self._source_re.search(page.text)
if m:
params = ""
data = m.group(1)
log.debug("Source data: {0}".format(data))
if "location.hash.substring" in data:
log.debug("Removing hash substring addition")
data = re.sub(r"\s*\+\s*location.hash.substring\(\d+\)", "",
data)
params = urlparse(self.url).fragment
data = self.js_to_json(data)
for stream in parse_json(data):
for s in HLSStream.parse_variant_playlist(
self.session, stream['file'], params=params).items():
yield s
else:
log.debug("No match for sources regex")
def _get_show_streams(self, stream_data, show, episode, platform="desktop"):
video_id = parse_json(stream_data.group(1), schema=self.vod_id_schema)
res = http.get(self.vod_api, params={"platform": platform, "id": video_id})
# create a unique list of the stream manifest URLs
streams = []
urldups = []
for stream in parse_xml(res.text, schema=self._vod_api_schema):
if stream["url"] not in urldups:
streams.append(stream)
urldups.append(stream["url"])
mapper = StreamMapper(lambda fmt, strm: strm["url"].endswith(fmt))
mapper.map(".m3u8", self._make_hls_hds_stream, HLSStream.parse_variant_playlist)
mapper.map(".f4m", self._make_hls_hds_stream, HDSStream.parse_manifest, is_akamai=True)
mapper.map(".mp4", lambda s: (s["bitrate"] + "k", HTTPStream(self.session, s["url"])))
for q, s in mapper(streams):
yield q, s
def _get_show_streams(self, stream_data, show, episode, platform="desktop"):
video_id = parse_json(stream_data.group(1), schema=self.vod_id_schema)
res = http.get(self.vod_api, params={"platform": platform, "id": video_id})
# create a unique list of the stream manifest URLs
streams = []
urldups = []
for stream in parse_xml(res.text, schema=self._vod_api_schema):
if stream["url"] not in urldups:
streams.append(stream)
urldups.append(stream["url"])
mapper = StreamMapper(lambda fmt, strm: strm["url"].endswith(fmt))
mapper.map(".m3u8", self._make_hls_hds_stream, HLSStream.parse_variant_playlist)
mapper.map(".f4m", self._make_hls_hds_stream, HDSStream.parse_manifest, is_akamai=True)
mapper.map(".mp4", lambda s: (s["bitrate"]+"k", HTTPStream(self.session, s["url"])))
for q, s in mapper(streams):
yield q, s
def _get_data_from_api(self, res):
_i_video_id = self._re_url.match(self.url).group("video_id")
if _i_video_id is None:
for link in itertags(res.text, "link"):
if link.attributes.get("rel") == "canonical":
try:
_i_video_id = self._re_url.match(link.attributes.get("href")).group("video_id")
except AttributeError:
return
break
else:
return
try:
_i_api_key = re.search(r'"INNERTUBE_API_KEY":\s*"([^"]+)"', res.text).group(1)
except AttributeError:
_i_api_key = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8"
try:
_i_version = re.search(r'"INNERTUBE_CLIENT_VERSION":\s*"([\d\.]+)"', res.text).group(1)
except AttributeError:
_i_version = "1.20210616.1.0"
res = self.session.http.post(
"https://www.youtube.com/youtubei/v1/player",
headers={"Content-Type": "application/json"},
params={"key": _i_api_key},
data=json.dumps({
"videoId": _i_video_id,
"context": {
"client": {
"clientName": "WEB_EMBEDDED_PLAYER",
"clientVersion": _i_version,
"platform": "DESKTOP",
"clientFormFactor": "UNKNOWN_FORM_FACTOR",
"browserName": "Chrome",
},
"user": {"lockedSafetyMode": "false"},
"request": {"useSsl": "true"},
}
}),
)
return parse_json(res.text)
def _get_streams(self):
self.session.http.headers = {"User-Agent": useragents.ANDROID}
res = self.session.http.get(self.url)
iframe_url = self.find_iframe(res)
if iframe_url:
self.logger.debug("Found iframe: {0}", iframe_url)
res = self.session.http.get(iframe_url, headers={"Referer": self.url})
data = self.data_re.search(res.text)
if data:
try:
ovva_url = parse_json(b64decode(data.group(1)).decode("utf8"), schema=self.ovva_data_schema)
stream_url = self.session.http.get(ovva_url, schema=self.ovva_redirect_schema)
except PluginError as e:
self.logger.error("Could not find stream URL: {0}", e)
else:
return HLSStream.parse_variant_playlist(self.session, stream_url)
else:
self.logger.error("Could not find player data.")
def _get_streams(self):
res = self.session.http.get(self.url)
m = self._room_json_re.search(res.text)
if not m:
log.info("Stream currently unavailable.")
return
data = parse_json(m.group(1), schema=self.data_schema)
# 1. some stream url has bitrate ending with t like _1500t.flv
# 2. data["vid"] is required because some stream IDs are to short and
# it could result in a wrong bitrate for source.
bitrate_re = re.compile(r"%s_(\d{3,4})\w?\.flv" % data["vid"])
for d in data["urlArray"]:
url = d["playUrl"]
match = bitrate_re.search(url)
if match:
stream_name = "{0}k".format(int(match.group(1)))
else:
stream_name = "source"
yield stream_name, HTTPStream(self.session, url)
def _get_streams(self):
res = self.session.http.get(self.url)
m = self._stream_data_re.search(res.text)
if m:
data = parse_json(m.group(1))
if data['LivestreamEnabled'] == '1':
account_id = data['LivestreamArgs']['account_id']
event_id = data['LivestreamArgs']['event_id']
log.debug("Found account_id={account_id} and event_id={event_id}".format(account_id=account_id, event_id=event_id))
url = self.API_URL.format(account_id=account_id, event_id=event_id)
api_res = self.session.http.get(url)
api_data = self.session.http.json(api_res)
stream_url = api_data.get('secure_m3u8_url') or api_data.get('m3u8_url')
if stream_url:
return HLSStream.parse_variant_playlist(self.session, stream_url)
else:
log.error("Could not find m3u8_url")
else:
log.error("Stream is offline")
def get_stream_url(self, data):
"""
Get the hls_url from the post request
:param data: dict with "gcp" and "ogn"
:return: hls_url
"""
try:
res = http.post(self.gate_url, headers=self.headers, data=data)
except Exception as e:
if "403" in str(e):
self.logger.error("This Video is Not Available in Your Country.")
raise NoStreamsError(self.url)
r_data = parse_json(res.text)
hls_url = r_data.get("stream")
suffix = r_data.get("suffix")
if hls_url is None and suffix:
hls_url = self.create_hls_url(suffix)
return hls_url
def _get_streams(self):
res = self.session.http.get(self.url)
data = self._re_stream.search(res.text)
if not data:
return
data = parse_json(base64.b64decode(data.group(1)),
schema=self._schema_data)
for info in data:
log.trace('{0!r}'.format(info))
flv_url = '{0}/{1}.{2}?{3}'.format(info["sFlvUrl"],
info["sStreamName"],
info["sFlvUrlSuffix"],
info["sFlvAntiCode"])
name = 'source_{0}'.format(info["sCdnType"].lower())
self.QUALITY_WEIGHTS[name] = info['iPCPriorityRate']
yield name, HTTPStream(self.session, flv_url)
log.debug('QUALITY_WEIGHTS: {0!r}'.format(self.QUALITY_WEIGHTS))
def _get_streams(self):
res = self.session.http.get(self.url)
m = self._room_json_re.search(res.text)
if not m:
log.info("Stream currently unavailable.")
return
data = parse_json(m.group(1), schema=self.data_schema)
# 1. some stream url has bitrate ending with t like _1500t.flv
# 2. data["vid"] is required because some stream IDs are to short and
# it could result in a wrong bitrate for source.
bitrate_re = re.compile(r"%s_(\d{3,4})\w?\.flv" % data["vid"])
for d in data["urlArray"]:
url = d["playUrl"]
match = bitrate_re.search(url)
if match:
stream_name = "{0}k".format(int(match.group(1)))
else:
stream_name = "source"
yield stream_name, HTTPStream(self.session, url)
def get_stream_url(self, data):
"""
Get the hls_url from the post request
:param data: dict with "gcp" and "ogn"
:return: hls_url
"""
try:
res = http.post(self.gate_url, headers=self.headers, data=data)
except Exception as e:
if "403" in str(e):
self.logger.error("This Video is Not Available in Your Country.")
raise NoStreamsError(self.url)
r_data = parse_json(res.text)
hls_url = r_data.get("stream")
suffix = r_data.get("suffix")
if hls_url is None and suffix:
hls_url = self.create_hls_url(suffix)
return hls_url
def _get_live_stream(self, stream_data, show, episode=None):
# parse the stream info as json
stream_info = parse_json(stream_data.group(1), schema=self.live_schema)
# get the stream ID
stream_id = None
show_info = stream_info[u"streams"][show]
if episode:
self.logger.debug("Loading replay of episode: {0}/{1}", show, episode)
for epi in show_info[u"archiveEpisodes"]:
if epi[u"slug"] == episode:
stream_id = epi[u"id"]
elif show_info["isLive"] or not len(show_info[u"archiveEpisodes"]):
self.logger.debug("Loading LIVE streams for: {0}", show)
stream_id = show_info[u"stream"]
else: # off-air
if len(show_info[u"archiveEpisodes"]):
epi = show_info[u"archiveEpisodes"][0]
self.logger.debug("Loading replay of episode: {0}/{1}", show, epi[u"slug"])
stream_id = epi[u"id"]
else:
self.logger.error("This stream is currently offline")
return
if stream_id:
api_url = self.API_URL.format(id=stream_id)
res = http.get(api_url, headers={"User-Agent": useragents.SAFARI_8})
stream_data = http.json(res, schema=self._api_schema)
mapper = StreamMapper(lambda fmt, surl: surl.endswith(fmt))
mapper.map(".m3u8", HLSStream.parse_variant_playlist, self.session)
mapper.map(".f4m", HDSStream.parse_manifest, self.session)
stream_urls = [asset[u"url"] for asset in stream_data[u'data'][u'stream'][u'assets']]
for q, s in mapper(stream_urls):
yield q, s
else:
self.logger.error("Couldn't find the stream ID for this stream: {}".format(show))
def _extract_nonce(cls, http_result):
"""
Given an HTTP response from the sessino endpoint, extract the nonce, so we can "sign" requests with it.
We don't really sign the requests in the traditional sense of a nonce, we just incude them in the auth requests.
:param http_result: HTTP response from the bbc session endpoint.
:type http_result: requests.Response
:return: nonce to "sign" url requests with
:rtype: string
"""
# Extract the redirect URL from the last call
last_redirect_url = urlparse(http_result.history[-1].request.url)
last_redirect_query = dict(parse_qsl(last_redirect_url.query))
# Extract the nonce from the query string in the redirect URL
final_url = urlparse(last_redirect_query['goto'])
goto_url = dict(parse_qsl(final_url.query))
goto_url_query = parse_json(goto_url['state'])
# Return the nonce we can use for future queries
return goto_url_query['nonce']
def _extract_nonce(cls, http_result):
"""
Given an HTTP response from the sessino endpoint, extract the nonce, so we can "sign" requests with it.
We don't really sign the requests in the traditional sense of a nonce, we just incude them in the auth requests.
:param http_result: HTTP response from the bbc session endpoint.
:type http_result: requests.Response
:return: nonce to "sign" url requests with
:rtype: string
"""
# Extract the redirect URL from the last call
last_redirect_url = urlparse(http_result.history[-1].request.url)
last_redirect_query = dict(parse_qsl(last_redirect_url.query))
# Extract the nonce from the query string in the redirect URL
final_url = urlparse(last_redirect_query['goto'])
goto_url = dict(parse_qsl(final_url.query))
goto_url_query = parse_json(goto_url['state'])
# Return the nonce we can use for future queries
return goto_url_query['nonce']
def _get_streams(self):
res = self.session.http.get(self.url,
headers={'User-Agent': useragents.CHROME})
video_search = res.text
video_search = video_search[
video_search.index('{"top":{"view":"PlayerContainer","model":{'):]
video_search = video_search[:video_search.index('}]}}') + 4] + "}"
video_url_found_hls = ""
video_url_found_http = ""
json_video_search = parse_json(video_search)
json_video_search_sources = json_video_search["top"]["model"][
"videos"][0]["sources"]
self.logger.debug('Video ID found: {0}',
json_video_search["top"]["model"]["id"])
for current_video_source in json_video_search_sources:
if "HLS" in current_video_source["type"]:
video_url_found_hls = "http://telefe.com" + current_video_source[
"url"]
self.logger.debug("HLS content available")
if "HTTP" in current_video_source["type"]:
video_url_found_http = "http://telefe.com" + current_video_source[
"url"]
self.logger.debug("HTTP content available")
self.session.http.headers = {
'Referer': self.url,
'User-Agent': useragents.CHROME,
'X-Requested-With': 'ShockwaveFlash/25.0.0.148'
}
if video_url_found_hls:
hls_streams = HLSStream.parse_variant_playlist(
self.session, video_url_found_hls)
for s in hls_streams.items():
yield s
if video_url_found_http:
yield "http", HTTPStream(self.session, video_url_found_http)
def _get_live_stream(self, stream_data, show, episode=None):
# parse the stream info as json
stream_info = parse_json(stream_data.group(1), schema=self.live_schema)
# get the stream ID
stream_id = None
show_info = stream_info[u"streams"][show]
if episode:
self.logger.debug("Loading replay of episode: {0}/{1}", show, episode)
for epi in show_info[u"archiveEpisodes"]:
if epi[u"slug"] == episode:
stream_id = epi[u"id"]
elif show_info.get("isLive") or not len(show_info[u"archiveEpisodes"]):
self.logger.debug("Loading LIVE streams for: {0}", show)
stream_id = show_info[u"stream"]
else: # off-air
if len(show_info[u"archiveEpisodes"]):
epi = show_info[u"archiveEpisodes"][0]
self.logger.debug("Loading replay of episode: {0}/{1}", show, epi[u"slug"])
stream_id = epi[u"id"]
else:
self.logger.error("This stream is currently offline")
return
if stream_id:
api_url = self.API_URL.format(id=stream_id)
res = http.get(api_url, headers={"User-Agent": useragents.SAFARI_8})
stream_data = http.json(res, schema=self._api_schema)
mapper = StreamMapper(lambda fmt, surl: surl.endswith(fmt))
mapper.map(".m3u8", HLSStream.parse_variant_playlist, self.session)
mapper.map(".f4m", HDSStream.parse_manifest, self.session)
stream_urls = [asset[u"url"] for asset in stream_data[u'data'][u'stream'][u'assets']]
for q, s in mapper(stream_urls):
yield q, s
else:
self.logger.error("Couldn't find the stream ID for this stream: {0}".format(show))
def login(self, ptrt_url, context="tvandiplayer"):
# get the site config, to find the signin url
config = http.get(self.config_url, params=dict(ptrt=ptrt_url), schema=self.config_schema)
res = http.get(config["signin_url"],
params=dict(userOrigin=context, context=context),
headers={"Referer": self.url})
m = self.account_locals_re.search(res.text)
if m:
auth_data = parse_json(m.group(1))
res = http.post(self.auth_url,
params=dict(context=auth_data["userOrigin"],
ptrt=auth_data["ptrt"]["value"],
userOrigin=auth_data["userOrigin"],
nonce=auth_data["nonce"]),
data=dict(jsEnabled="false", attempts=0, username=self.get_option("username"),
password=self.get_option("password")))
# redirects to ptrt_url on successful login
if res.url == ptrt_url:
return res
else:
self.logger.error("Could not authenticate, could not find the authentication nonce")
def session_key(self):
"""
Get a cached or new session key, uuid is a random uuid (type 4)
:return:
"""
session_key = self.cache.get("sessionKey")
if session_key:
self.logger.debug("Using cached sessionKey")
return session_key
else:
self.logger.debug("Requesting new sessionKey")
uuid = uuid4()
res = http.get(self.session_url,
params=dict(appKey=self.app_key, uuid=uuid))
data = parse_json(res.text, schema=self.session_schema)
# when to expire the sessionKey, -1 hour for good measure
expires_in = (data["expiration"] -
datetime.now()).total_seconds() - 3600
self.cache.set("sessionKey",
data["sessionKey"],
expires=expires_in)
return data["sessionKey"]
def _get_streams(self):
res = http.get(self.url, headers={"User-Agent": useragents.CHROME})
with open("temp.html", "w") as f:
f.write(res.text)
for match in self._mpd_re.finditer(res.text):
manifest_url = match.group("url")
if "\\/" in manifest_url:
# if the URL is json encoded, decode it
manifest_url = parse_json("\"{}\"".format(manifest_url))
for s in DASHStream.parse_manifest(self.session,
manifest_url).items():
yield s
else:
match = self._playlist_re.search(res.text)
playlist = match and match.group(1)
if playlist:
for url in {
url.group(1)
for url in self._plurl_re.finditer(playlist)
}:
yield "live", HTTPStream(self.session, url)
def _get_prid(self, subtitles=False):
res = http.get(self.url)
bprid = None
# Locate the asset id for the content on the page
for alt, _, prid in self.prid_re.findall(res.text):
if alt and subtitles:
bprid = prid
elif bprid is None:
bprid = prid
if bprid is None:
m = self.react_re.search(res.text)
if m:
data = parse_json(m.group("data").replace(""", '"'))
bprid = data.get("mid")
if bprid is None:
m = self.media_id_re.search(res.text)
if m:
bprid = m.group('media_id')
return bprid
def _get_streams(self):
video_url_found_hls = ""
if "eltrecetv.com.ar/vivo" in self.url.lower():
http.headers = {'Referer': self.url,
'User-Agent': "ArtearPlayer/3.2.42 (Linux;Android 4.4.2) ExoPlayerLib/1.5.9"}
video_url_found_hls = "http://stream.eltrecetv.com.ar/live13/13tv/13tv1/playlist.m3u8"
else:
http.headers = {'Referer': self.url,
'User-Agent': useragents.CHROME}
res = http.get(self.url)
_player_re = re.compile(r'''data-kaltura="([^"]+)"''')
match = _player_re.search(res.text)
if not match:
return
json_video_search = parse_json(match.group(1).replace(""", '"'))
video_url_found_hls = "https://vodgc.com/p/111/sp/11100/playManifest/entryId/" + json_video_search["entryId"] + "/format/applehttp/protocol/https/a.m3u8"
if video_url_found_hls:
hls_streams = HLSStream.parse_variant_playlist(self.session, video_url_found_hls)
for s in hls_streams.items():
yield s
def _find_video_id(self, url):
m = _url_re.match(url)
if m.group("video_id"):
log.debug("Video ID from URL")
return m.group("video_id")
res = http.get(url)
datam = _ytdata_re.search(res.text)
if datam:
data = parse_json(datam.group(1))
# find the videoRenderer object, where there is a LVE NOW badge
for vid_ep in search_dict(data, 'currentVideoEndpoint'):
video_id = vid_ep.get("watchEndpoint", {}).get("videoId")
if video_id:
log.debug("Video ID from currentVideoEndpoint")
return video_id
for x in search_dict(data, 'videoRenderer'):
for bstyle in search_dict(x.get("badges", {}), "style"):
if bstyle == "BADGE_STYLE_TYPE_LIVE_NOW":
if x.get("videoId"):
log.debug("Video ID from videoRenderer (live)")
return x["videoId"]
def _get_streams(self):
res = http.get(self.url, headers={"User-Agent": useragents.CHROME})
streams = {}
vod_urls = set([])
for match in self._src_re.finditer(res.text):
stream_url = match.group("url")
if "\\/" in stream_url:
# if the URL is json encoded, decode it
stream_url = parse_json("\"{}\"".format(stream_url))
if ".mpd" in stream_url:
streams.update(
DASHStream.parse_manifest(self.session, stream_url))
elif ".mp4" in stream_url:
streams[match.group(1)] = HTTPStream(self.session, stream_url)
vod_urls.add(stream_url)
else:
self.logger.debug(
"Non-dash/mp4 stream: {0}".format(stream_url))
if streams:
return streams
# fallback on to playlist
self.logger.debug("Falling back to playlist regex")
match = self._playlist_re.search(res.text)
playlist = match and match.group(1)
if playlist:
for url in {
url.group(1)
for url in self._plurl_re.finditer(playlist)
}:
if url not in vod_urls:
streams["sd"] = HTTPStream(self.session, url)
return streams
def _get_streams(self):
"""
Find the streams for web.tv
:return:
"""
headers = {}
res = http.get(self.url, headers=headers)
headers["Referer"] = self.url
sources = self._sources_re.findall(res.text)
if len(sources):
sdata = parse_json(sources[0], schema=self._sources_schema)
for source in sdata:
self.logger.debug("Found stream of type: {}", source[u'type'])
if source[u'type'] == u"application/vnd.apple.mpegurl":
# if the url has no protocol, assume it is http
url = source[u"src"]
if url.startswith("//"):
url = "http:" + url
try:
# try to parse the stream as a variant playlist
variant = HLSStream.parse_variant_playlist(
self.session, url, headers=headers)
if variant:
for q, s in variant.items():
yield q, s
else:
# and if that fails, try it as a plain HLS stream
yield 'live', HLSStream(self.session,
url,
headers=headers)
except IOError:
self.logger.warning(
"Could not open the stream, perhaps the channel is offline"
)
def _find_channel_video(self):
res = http.get(self.url)
datam = _ytdata_re.search(res.text)
if datam:
data = parse_json(datam.group(1))
# find the videoRenderer object, where there is a LVE NOW badge
for x in search_dict(data, 'videoRenderer'):
for bstyle in search_dict(x.get("badges", {}), "style"):
if bstyle == "BADGE_STYLE_TYPE_LIVE_NOW":
if x.get("videoId"):
self.logger.debug("Found channel video ID via HTML: {0}", x["videoId"])
return x["videoId"]
else:
# fall back on API
self.logger.debug("No channel data, falling back to API")
match = _channelid_re.search(res.text)
if not match:
return
channel_id = match.group(1)
self.logger.debug("Found channel_id: {0}".format(channel_id))
return self._get_channel_video(channel_id)
def recv(self):
data = parse_json(self._ws.recv(), schema=self.api_schema)
log.debug("Received `{0}` command".format(data["cmd"]))
log.trace("{0!r}".format(data))
return data
def _get_stream_data(self, **params):
api_url = self.britecove.format(**params)
res = http.get(api_url, headers={"Accept": "application/json;pk={policy_key}".format(**params)})
return parse_json(res.text, schema=self.britecove_schema)
def _get_data(self, res):
match = re.search(self._re_ytInitialPlayerResponse, res.text)
if not match:
log.debug("Missing initial player response data")
return
return parse_json(match.group(1))
def get_hls_url(self, media_id):
res = http.get(self.api_url, params=dict(media_id=media_id))
try:
return parse_json(res.text, schema=self.api_schema)
except PluginError:
return
def _get_streams(self):
self.session.http.headers.update({'User-Agent': useragents.FIREFOX})
log.debug('Version 2018-07-12')
log.info('This is a custom plugin. ')
match = self._url_re.match(self.url)
username = match.group('username')
user_id = match.group('user_id')
servers = self._get_servers()
chat_servers = servers['chat_servers']
message, php_message = self._websocket_data(username, chat_servers)
if user_id and not username:
data = self._php_fallback(username, user_id, php_message)
else:
log.debug('Attempting to use WebSocket data')
data = self._dict_re.search(message)
if data is None:
raise NoStreamsError(self.url)
data = parse_json(data.group('data'), schema=self._data_schema)
vs = data['vs']
ok_vs = [0, 90]
if vs not in ok_vs:
if vs == 2:
log.info('Model is currently away')
elif vs == 12:
log.info('Model is currently in a private show')
elif vs == 13:
log.info('Model is currently in a group show')
elif vs == 127:
log.info('Model is currently offline')
else:
log.error('Stream status: {0}'.format(vs))
raise NoStreamsError(self.url)
log.debug('VS: {0}'.format(vs))
nm = data['nm']
uid = data['uid']
uid_video = uid + 100000000
camserver = data['u']['camserv']
server, server_type = self._get_camserver(servers, camserver)
if server is None and not user_id:
fallback_data = self._php_fallback(username, user_id, php_message)
camserver = fallback_data['u']['camserv']
server, server_type = self._get_camserver(servers, camserver)
log.info('Username: {0}'.format(nm))
log.info('User ID: {0}'.format(uid))
if not server:
raise PluginError('Missing video server')
log.debug('Video server: {0}'.format(server))
log.debug('Video server_type: {0}'.format(server_type))
if server_type == 'h5video_servers':
DASH_VIDEO_URL = 'https://{0}.myfreecams.com/NxServer/ngrp:mfc_{1}.f4v_desktop/manifest.mpd'.format(
server, uid_video)
HLS_VIDEO_URL = 'https://{0}.myfreecams.com/NxServer/ngrp:mfc_{1}.f4v_mobile/playlist.m3u8'.format(
server, uid_video)
elif server_type == 'wzobs_servers':
DASH_VIDEO_URL = ''
HLS_VIDEO_URL = 'https://{0}.myfreecams.com/NxServer/ngrp:mfc_a_{1}.f4v_mobile/playlist.m3u8'.format(
server, uid_video)
elif server_type == 'ngvideo_servers':
raise PluginError('ngvideo_servers are not supported.')
else:
raise PluginError('Unknow server type.')
log.debug('HLS URL: {0}'.format(HLS_VIDEO_URL))
for s in HLSStream.parse_variant_playlist(self.session,
HLS_VIDEO_URL).items():
yield s
if DASH_VIDEO_URL and self.get_option('dash'):
log.debug('DASH URL: {0}'.format(DASH_VIDEO_URL))
for s in DASHStream.parse_manifest(self.session,
DASH_VIDEO_URL).items():
yield s
def _get_servers(self):
res = self.session.http.get(self.JS_SERVER_URL)
servers = parse_json(res.text)
return servers
def recv(self):
data = parse_json(self._ws.recv(), schema=self.api_schema)
log.debug("Received `{0}` command".format(data["cmd"]))
log.trace("{0!r}".format(data))
return data
def js_to_json(data):
js_re = re.compile(r'(?!<")(\w+):(?!/)')
trimmed = [y.replace("\r", "").strip() for y in data.split(",")]
jsons = ','.join([js_re.sub(r'"\1":', x, count=1) for x in trimmed])
return parse_json(jsons)
