def _get_streams(self):
if "eltrecetv.com.ar/vivo" in self.url.lower():
try:
self.session.http.headers = {
'Referer': self.url,
'User-Agent': useragents.ANDROID
}
res = self.session.http.get('https://api.iamat.com/metadata/atcodes/eltrece')
yt_id = parse_json(res.text)["atcodes"][0]["context"]["ahora"]["vivo"]["youtubeVideo"]
yt_url = "https://www.youtube.com/watch?v={0}".format(yt_id)
return self.session.streams(yt_url)
except BaseException:
log.info("Live content is temporarily unavailable. Please try again later.")
else:
try:
self.session.http.headers = {
'Referer': self.url,
'User-Agent': useragents.CHROME
}
res = self.session.http.get(self.url)
_player_re = re.compile(r'''data-kaltura="([^"]+)"''')
match = _player_re.search(res.text)
if not match:
return
entry_id = parse_json(match.group(1).replace(""", '"'))["entryId"]
hls_url = "https://vodgc.com/p/111/sp/11100/playManifest/entryId/" \
"{0}/format/applehttp/protocol/https/a.m3u8".format(entry_id)
return HLSStream.parse_variant_playlist(self.session, hls_url)
except BaseException:
log.error("The requested VOD content is unavailable.")
def test_parse_json(self):
self.assertEqual({}, parse_json("{}"))
self.assertEqual({"test": 1}, parse_json("""{"test": 1}"""))
self.assertEqual({"test": 1}, parse_json("""{"test": 1}""", schema=validate.Schema({"test": 1})))
self.assertRaises(PluginError, parse_json, """{"test: 1}""")
self.assertRaises(IOError, parse_json, """{"test: 1}""", exception=IOError)
self.assertRaises(PluginError, parse_json, """{"test: 1}""" * 10)
def _get_streams(self):
streamdata = None
if self.get_option("email"):
if self.login(self.get_option("email"), self.get_option("password")):
log.info("Logged in as {0}".format(self.get_option("email")))
self.save_cookies(lambda c: "steamMachineAuth" in c.name)
# Handle steam.tv URLs
if self.matches[1] is not None:
# extract the steam ID from the page
res = self.session.http.get(self.url)
for div in itertags(res.text, 'div'):
if div.attributes.get("id") == "webui_config":
broadcast_data = html_unescape(div.attributes.get("data-broadcast"))
steamid = parse_json(broadcast_data).get("steamid")
self.url = self._watch_broadcast_url + steamid
# extract the steam ID from the URL
steamid = self.match.group(1)
res = self.session.http.get(self.url) # get the page to set some cookies
sessionid = res.cookies.get('sessionid')
while streamdata is None or streamdata["success"] in ("waiting", "waiting_for_start"):
streamdata = self._get_broadcast_stream(steamid,
sessionid=sessionid)
if streamdata["success"] == "ready":
return DASHStream.parse_manifest(self.session, streamdata["url"])
elif streamdata["success"] == "unavailable":
log.error("This stream is currently unavailable")
return
else:
r = streamdata["retry"] / 1000.0
log.info("Waiting for stream, will retry again in {} seconds...".format(r))
time.sleep(r)
def _get_streams(self):
res = self.session.http.get(self.url)
m = self._re_channel_id.search(res.text)
if not m:
return
res = self.session.http.get(
"https://www.rtvs.sk/json/live5f.json",
params={
"c": m.group(1),
"b": "mozilla",
"p": "win",
"f": "0",
"d": "1",
}
)
videos = parse_json(res.text, schema=validate.Schema({
"clip": {
"sources": [{
"src": validate.url(),
"type": str,
}],
}},
validate.get(("clip", "sources")),
validate.filter(lambda n: n["type"] == "application/x-mpegurl"),
))
for video in videos:
yield from HLSStream.parse_variant_playlist(self.session, video["src"]).items()
def _get_streams(self):
headers = {"Referer": self.url}
res = self.session.http.get(self.url, headers=headers)
match = _ddos_re.search(res.text)
if match:
log.debug("Anti-DDOS bypass...")
headers["Cookie"] = match.group(1)
res = self.session.http.get(self.url, headers=headers)
match = _apidata_re.search(res.text)
channel_info = match and parse_json(match.group("data"))
if not channel_info:
log.error("Could not find channel info")
return
log.debug(
"Found channel info: id={id} channelkey={channelkey} pid={streamkey} online={status}"
.format(**channel_info))
if not channel_info['status']:
log.debug("Channel appears to be offline")
streams = {}
for name, url_suffix in QUALITIES.items():
url = HLS_URL_FORMAT.format(channel_info['streamkey'], url_suffix)
if not self._check_stream(url):
continue
streams[name] = HLSStream(self.session, url)
return streams
def _get_streams(self):
"""
Find the streams for web.tv
:return:
"""
headers = {}
res = self.session.http.get(self.url, headers=headers)
headers["Referer"] = self.url
sources = self._sources_re.findall(res.text)
if len(sources):
sdata = parse_json(sources[0], schema=self._sources_schema)
for source in sdata:
log.debug(f"Found stream of type: {source['type']}")
if source["type"] == "application/vnd.apple.mpegurl":
url = update_scheme("https://", source["src"], force=False)
try:
# try to parse the stream as a variant playlist
variant = HLSStream.parse_variant_playlist(
self.session, url, headers=headers)
if variant:
yield from variant.items()
else:
# and if that fails, try it as a plain HLS stream
yield 'live', HLSStream(self.session,
url,
headers=headers)
except OSError:
log.warning(
"Could not open the stream, perhaps the channel is offline"
)
def parse_token(tokenstr):
return parse_json(tokenstr, schema=validate.Schema(
{"chansub": {"restricted_bitrates": validate.all(
[str],
validate.filter(lambda n: not re.match(r"(.+_)?archives|live|chunked", n))
)}},
validate.get(("chansub", "restricted_bitrates"))
))
def find_tvip(self, url, master=False):
log.debug("Looking for {0} tvip on {1}".format("master" if master else "", url))
res = self.session.http.get(url)
m = self.state_re.search(res.text)
data = m and parse_json(m.group(1))
if data:
channel = data.get("channel")
if master:
return channel.get("masterBrand")
return channel.get("id")
def _get_stream_data(self, id):
res = self.session.http.get(self.url)
m = self.json_data_re.search(res.text)
if m and m.group(1):
streams = parse_json(m.group(1), schema=self._stream_data_schema)
else:
raise PluginError("Failed to get json_data")
for stream in streams:
if 'id' in stream:
if id == stream['id'] and 'stream' in stream:
return stream['stream']
def get_stream(self, slug, js_data):
token_data = {}
token_data['token'] = self.cache.get('token')
token_data['token_type'] = self.cache.get('token_type')
if token_data['token'] and token_data['token_type']:
log.debug('Using cached token')
else:
log.debug('Getting new token')
res = self.session.http.post(
self.login_url,
json=js_data['credentials'],
)
token_data = self.session.http.json(res, schema=self.token_schema)
log.debug('Token={0}'.format(token_data["token"]))
self.cache.set('token',
token_data['token'],
expires=token_data['expires_in'])
self.cache.set('token_type',
token_data['token_type'],
expires=token_data['expires_in'])
headers = {
'Authorization':
'{0} {1}'.format(token_data["token_type"], token_data["token"])
}
data = {
'slug': slug,
'type': js_data['type'],
}
res = self.session.http.post(
self.stream_url,
headers=headers,
json=data,
)
encrypted_data = self.session.http.json(
res, schema=self.encrypted_data_schema)
stream_data = parse_json(
self.decrypt_data(js_data['cipher_data'], encrypted_data),
schema=self.stream_schema,
)
self.author = stream_data['channel_name']
self.category = stream_data['genres']
self.title = stream_data['meta_title']
return stream_data['live_stream_url']
def _get_streams(self):
room_id = self.match.group("room_id")
res = self.session.http.get(self.api_url.format(room_id))
data = self._data_re.search(res.text)
if not data:
return
try:
hls_url = parse_json(data.group("data"), schema=self._data_schema)
except Exception:
raise NoStreamsError(self.url)
log.debug("URL={0}".format(hls_url))
return {"live": HLSStream(self.session, hls_url)}
def _parse_streams(self, res):
stream_url = validate.Schema(
validate.parse_html(),
validate.xml_xpath_string(
".//head/meta[@property='og:video:url'][@content][1]/@content")
).validate(res.text)
if not stream_url:
log.debug("No meta og:video:url")
else:
if ".mpd" in stream_url:
for s in DASHStream.parse_manifest(self.session,
stream_url).items():
yield s
return
elif ".mp4" in stream_url:
yield "vod", HTTPStream(self.session, stream_url)
return
for match in self._src_re.finditer(res.text):
stream_url = match.group("url")
if "\\/" in stream_url:
# if the URL is json encoded, decode it
stream_url = parse_json("\"{}\"".format(stream_url))
if ".mpd" in stream_url:
for s in DASHStream.parse_manifest(self.session,
stream_url).items():
yield s
elif ".mp4" in stream_url:
yield match.group(1), HTTPStream(self.session, stream_url)
else:
log.debug("Non-dash/mp4 stream: {0}".format(stream_url))
match = self._dash_manifest_re.search(res.text)
if match:
# facebook replaces "<" characters with the substring "\\x3C"
manifest = match.group("manifest").replace("\\/", "/")
if is_py3:
manifest = bytes(unquote_plus(manifest),
"utf-8").decode("unicode_escape")
else:
manifest = unquote_plus(manifest).decode("string_escape")
# Ignore unsupported manifests until DASH SegmentBase support is implemented
if "SegmentBase" in manifest:
log.error("Skipped DASH manifest with SegmentBase streams")
else:
for s in DASHStream.parse_manifest(self.session,
manifest).items():
yield s
def _get_data_from_api(self, res):
_i_video_id = self.match.group("video_id")
if _i_video_id is None:
try:
_i_video_id = self._schema_canonical(res.text)
except (PluginError, TypeError):
return
try:
_i_api_key = re.search(r'"INNERTUBE_API_KEY":\s*"([^"]+)"',
res.text).group(1)
except AttributeError:
_i_api_key = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8"
try:
_i_version = re.search(
r'"INNERTUBE_CLIENT_VERSION":\s*"([\d\.]+)"',
res.text).group(1)
except AttributeError:
_i_version = "1.20210616.1.0"
res = self.session.http.post(
"https://www.youtube.com/youtubei/v1/player",
headers={"Content-Type": "application/json"},
params={"key": _i_api_key},
data=json.dumps({
"videoId": _i_video_id,
"contentCheckOk": True,
"racyCheckOk": True,
"context": {
"client": {
"clientName": "WEB",
"clientVersion": _i_version,
"platform": "DESKTOP",
"clientScreen": "EMBED",
"clientFormFactor": "UNKNOWN_FORM_FACTOR",
"browserName": "Chrome",
},
"user": {
"lockedSafetyMode": "false"
},
"request": {
"useSsl": "true"
},
}
}),
)
return parse_json(res.text)
def _get_streams(self):
res = self.session.http.get(self.url)
data = self._re_stream.search(res.text)
if not data:
return
data = parse_json(base64.b64decode(data.group(1)), schema=self._schema_data)
for info in data:
log.trace(f'{info!r}')
flv_url = f'{info["sFlvUrl"]}/{info["sStreamName"]}.{info["sFlvUrlSuffix"]}?{info["sFlvAntiCode"]}'
name = f'source_{info["sCdnType"].lower()}'
self.QUALITY_WEIGHTS[name] = info['iPCPriorityRate']
yield name, HTTPStream(self.session, flv_url)
log.debug(f'QUALITY_WEIGHTS: {self.QUALITY_WEIGHTS!r}')
def find_vpid(self, url, res=None):
"""
Find the Video Packet ID in the HTML for the provided URL
:param url: URL to download, if res is not provided.
:param res: Provide a cached version of the HTTP response to search
:type url: string
:type res: requests.Response
:return: Video Packet ID for a Programme in iPlayer
:rtype: string
"""
log.debug(f"Looking for vpid on {url}")
# Use pre-fetched page if available
res = res or self.session.http.get(url)
m = self.mediator_re.search(res.text)
vpid = m and parse_json(m.group(1), schema=self.mediator_schema)
return vpid
def _parse_streams(self, res):
_found_stream_url = False
for meta in itertags(res.text, "meta"):
if meta.attributes.get("property") == "og:video:url":
stream_url = html_unescape(meta.attributes.get("content"))
if ".mpd" in stream_url:
for s in DASHStream.parse_manifest(self.session,
stream_url).items():
yield s
_found_stream_url = True
elif ".mp4" in stream_url:
yield "vod", HTTPStream(self.session, stream_url)
_found_stream_url = True
break
else:
log.debug("No meta og:video:url")
if _found_stream_url:
return
for match in self._src_re.finditer(res.text):
stream_url = match.group("url")
if "\\/" in stream_url:
# if the URL is json encoded, decode it
stream_url = parse_json("\"{}\"".format(stream_url))
if ".mpd" in stream_url:
yield from DASHStream.parse_manifest(self.session,
stream_url).items()
elif ".mp4" in stream_url:
yield match.group(1), HTTPStream(self.session, stream_url)
else:
log.debug("Non-dash/mp4 stream: {0}".format(stream_url))
match = self._dash_manifest_re.search(res.text)
if match:
# facebook replaces "<" characters with the substring "\\x3C"
manifest = match.group("manifest").replace("\\/", "/")
manifest = bytes(unquote_plus(manifest),
"utf-8").decode("unicode_escape")
# Ignore unsupported manifests until DASH SegmentBase support is implemented
if "SegmentBase" in manifest:
log.error("Skipped DASH manifest with SegmentBase streams")
else:
yield from DASHStream.parse_manifest(self.session,
manifest).items()
def _get_video_data(self, slug):
m = self.truncate_url_re.search(self.url)
if m and m.group(1):
log.debug("Truncated URL={0}".format(m.group(1)))
else:
raise PluginError("Failed to truncate URL")
res = self.session.http.get(m.group(1))
m = self.json_data_re.search(res.text)
if m and m.group(1):
videos = parse_json(m.group(1), schema=self._video_data_schema)
else:
raise PluginError("Failed to get json_data")
for video in videos:
if 'slug' in videos[video]:
if slug == videos[video]['slug'] and 'id' in videos[video]:
return videos[video]['id']
def _get_streams(self):
res = self.session.http.get(self.url)
m = self._config_re.search(res.text)
if not m:
log.debug("Unable to find _config_re")
return
stream_info = parse_json(m.group(1), "config JSON",
schema=self._stream_config_schema)
log.trace("stream_info: {0!r}".format(stream_info))
if not (stream_info and stream_info["is_live"]):
log.debug("Stream might be Off Air")
return
m3u8_url = stream_info.get("secure_m3u8_url")
if m3u8_url:
for s in HLSStream.parse_variant_playlist(self.session, m3u8_url).items():
yield s
def _get_streams(self):
res = self.session.http.get(self.url)
m = self._room_json_re.search(res.text)
if not m:
log.info("Stream currently unavailable.")
return
data = parse_json(m.group(1), schema=self.data_schema)
# 1. some stream url has bitrate ending with t like _1500t.flv
# 2. data["vid"] is required because some stream IDs are to short and
# it could result in a wrong bitrate for source.
bitrate_re = re.compile(r"%s_(\d{3,4})\w?\.flv" % data["vid"])
for d in data["urlArray"]:
url = d["playUrl"]
match = bitrate_re.search(url)
if match:
stream_name = "{0}k".format(int(match.group(1)))
else:
stream_name = "source"
yield stream_name, HTTPStream(self.session, url)
def _get_streams(self):
res = self.session.http.get(self.url)
data = self._re_stream.search(res.text)
if not data:
return
data = parse_json(base64.b64decode(data.group(1)),
schema=self._schema_data)
for info in data:
log.trace('{0!r}'.format(info))
flv_url = '{0}/{1}.{2}?{3}'.format(info["sFlvUrl"],
info["sStreamName"],
info["sFlvUrlSuffix"],
info["sFlvAntiCode"])
name = 'source_{0}'.format(info["sCdnType"].lower())
self.QUALITY_WEIGHTS[name] = info['iPCPriorityRate']
yield name, HTTPStream(self.session, flv_url)
log.debug('QUALITY_WEIGHTS: {0!r}'.format(self.QUALITY_WEIGHTS))
def on_message(self, wsapp, data: str):
try:
parsed = parse_json(data, schema=self._schema_cmd)
except PluginError:
log.error(f"Could not parse message: {data[:50]}")
return
cmd: str = parsed["cmd"]
args: List[Dict] = parsed["args"]
log.trace(f"Received '{cmd}' command")
log.trace(f"{args!r}")
handlers = self._MESSAGE_HANDLERS.get(cmd)
if handlers is not None:
for arg in args:
for name, handler in handlers.items():
argdata = arg.get(name)
if argdata is not None:
log.debug(f"Processing '{cmd}' - '{name}'")
handler(self, argdata)
def _get_streams(self):
res = self.session.http.get(self.url,
headers={'User-Agent': useragents.CHROME})
video_search = res.text
video_search = video_search[
video_search.index('{"top":{"view":"PlayerContainer","model":{'):]
video_search = video_search[:video_search.index('}]}}') + 4] + "}"
video_url_found_hls = ""
video_url_found_http = ""
json_video_search = parse_json(video_search)
json_video_search_sources = json_video_search["top"]["model"][
"videos"][0]["sources"]
log.debug('Video ID found: {0}'.format(
json_video_search["top"]["model"]["id"]))
for current_video_source in json_video_search_sources:
if "HLS" in current_video_source["type"]:
video_url_found_hls = "http://telefe.com" + current_video_source[
"url"]
log.debug("HLS content available")
if "HTTP" in current_video_source["type"]:
video_url_found_http = "http://telefe.com" + current_video_source[
"url"]
log.debug("HTTP content available")
self.session.http.headers = {
'Referer': self.url,
'User-Agent': useragents.CHROME,
'X-Requested-With': 'ShockwaveFlash/25.0.0.148'
}
if video_url_found_hls:
hls_streams = HLSStream.parse_variant_playlist(
self.session, video_url_found_hls)
for s in hls_streams.items():
yield s
if video_url_found_http:
yield "http", HTTPStream(self.session, video_url_found_http)
def on_message(self, wsapp, data: str):
log.debug(f"Received: {data}")
message = parse_json(data)
msgtype = message.get("type")
msgdata = message.get("data", {})
if msgtype == "ping":
self.send_pong()
elif msgtype == "stream" and msgdata.get("protocol") == "hls" and msgdata.get("uri"):
self.hls_stream_url = msgdata.get("uri")
self.ready.set()
if self.opened.wait(self.STREAM_OPENED_TIMEOUT):
log.debug("Stream opened, keeping websocket connection alive")
else:
log.info("Closing websocket connection")
self.close()
elif msgtype == "disconnect":
reason = msgdata.get("reason", "Unknown reason")
log.info(f"Received disconnect message: {reason}")
self.close()
def on_message(self, wsapp, data):
# type: (object, str)
try:
parsed = parse_json(data, schema=self._schema_cmd)
except PluginError:
log.error("Could not parse message: {0}".format(data[:50]))
return
cmd = parsed["cmd"]
# type: str
args = parsed["args"]
# type: List[Dict]
log.trace("Received '{0}' command".format(cmd))
log.trace("{0!r}".format(args))
handlers = self._MESSAGE_HANDLERS.get(cmd)
if handlers is not None:
for arg in args:
for name, handler in handlers.items():
argdata = arg.get(name)
if argdata is not None:
log.debug("Processing '{0}' - '{1}'".format(cmd, name))
handler(self, argdata)
def get_hls_url(self, media_id):
res = self.session.http.get(self.api_url,
params=dict(media_id=media_id))
return parse_json(res.text, schema=self.api_schema)
def _get_data_from_regex(res, regex, descr):
match = re.search(regex, res.text)
if not match:
log.debug(f"Missing {descr}")
return
return parse_json(match.group(1))
def _get_streams(self):
res = self.session.http.get(self.url)
for script in itertags(res.text, "script"):
_type = script.attributes.get("type")
if not (_type and _type == "application/json"):
continue
video_url = None
_data_ssr_name = script.attributes.get("data-ssr-name")
if not _data_ssr_name:
continue
log.trace(f"Found _data_ssr_name={_data_ssr_name}")
if _data_ssr_name == "pages/Broadcasts/Broadcasts":
self.title, video_url, is_live = parse_json(
script.text,
schema=validate.Schema(
{
"currentLivestream": {
"is_live": bool,
"title": str,
"stream": validate.url(),
}
}, validate.get("currentLivestream"),
validate.union_get("title", "stream", "is_live")))
if not is_live:
log.error(self._msg_live_offline)
continue
elif _data_ssr_name == "pages/Livestream/Livestream":
self.title, video_url, is_live = parse_json(
script.text,
schema=validate.Schema(
{
"streamIsLive": bool,
"title": str,
"stream": validate.url(),
}, validate.union_get("title", "stream",
"streamIsLive")))
if not is_live:
log.error(self._msg_live_offline)
continue
elif _data_ssr_name in self.vod_keys.keys():
_key = self.vod_keys[_data_ssr_name]
self.title, video_url = parse_json(
script.text,
schema=validate.Schema(
{
_key: {
"title": str,
"aspect_ratios": {
"profiles":
validate.all(
[{
"name": str,
"url": validate.url(),
}],
validate.filter(lambda n: n["name"] ==
"hls_unencrypted"))
}
}
}, validate.get(_key),
validate.union_get(
"title", ("aspect_ratios", "profiles", 0, "url"))))
if video_url is not None:
yield from HLSStream.parse_variant_playlist(
self.session, video_url).items()
break
def recv(self):
data = parse_json(self._ws.recv(), schema=self.api_schema)
log.debug("Received `{0}` command".format(data["cmd"]))
log.trace("{0!r}".format(data))
return data
def json(cls, res, *args, **kwargs):
"""Parses JSON from a response."""
# if an encoding is already set then use the provided encoding
if res.encoding is None:
res.encoding = cls.determine_json_encoding(res.content[:4])
return parse_json(res.text, *args, **kwargs)
