def _get_stream_id(self, channel, ws_url):
ws = websocket.WebSocket()
ws.connect(ws_url)
ws.send(JSON_UID % RANDOM_UID)
ws.send(JSON_CHANNEL % channel)
# STREAM_ID
result = ws.recv()
data = utils.parse_json(result, schema=_schema)
log.debug('DATA 1 {0}'.format(data))
if 'failure' in data:
ws.close()
return False, False
# RTMP CDN
result_2 = ws.recv()
data2 = utils.parse_json(result_2, schema=_schema)
log.debug('DATA 2 {0}'.format(data2))
if 'failure' in data2:
ws.close()
return False, False
# ERROR
result_3 = ws.recv()
data3 = utils.parse_json(result_3, schema=_schema)
log.debug('DATA 3 {0}'.format(data3))
if 'failure' in data3:
ws.close()
return False, False
return data[0], data2[1]
def _get_hls_streams(self, type="live"):
self._authenticate()
if self._check_for_host() and self.options.get("disable_hosting"):
self.logger.info("hosting was disabled by command line option")
return {}
else:
self.logger.info("switching to {}", self.channel)
sig, token = self._access_token(type)
if type == "live":
url = self.usher.channel(self.channel, sig=sig, token=token)
elif type == "video":
url = self.usher.video(self.video_id, nauthsig=sig, nauth=token)
try:
streams = HLSStream.parse_variant_playlist(self.session, url)
except IOError as err:
err = str(err)
if "404 Client Error" in err or "Failed to parse playlist" in err:
return
else:
raise PluginError(err)
try:
token = parse_json(token, schema=_token_schema)
for name in token["restricted_bitrates"]:
if name not in streams:
self.logger.warning("The quality '{0}' is not available "
"since it requires a subscription.",
name)
except PluginError:
pass
return streams
def _get_hls_streams(self, type="live"):
self._authenticate()
if self._check_for_host() and self.options.get("disable_hosting"):
self.logger.info("hosting was disabled by command line option")
return {}
else:
self.logger.info("switching to {}", self.channel)
sig, token = self._access_token(type)
if type == "live":
url = self.usher.channel(self.channel, sig=sig, token=token)
elif type == "video":
url = self.usher.video(self.video_id, nauthsig=sig, nauth=token)
try:
streams = HLSStream.parse_variant_playlist(self.session, url)
except IOError as err:
err = str(err)
if "404 Client Error" in err or "Failed to parse playlist" in err:
return
else:
raise PluginError(err)
try:
token = parse_json(token, schema=_token_schema)
for name in token["restricted_bitrates"]:
if name not in streams:
self.logger.warning(
"The quality '{0}' is not available "
"since it requires a subscription.", name)
except PluginError:
pass
return streams
def _get_streams(self):
streamdata = None
if self.get_option("email"):
if self.login(self.get_option("email"), self.get_option("password")):
log.info("Logged in as {0}".format(self.get_option("email")))
self.save_cookies(lambda c: "steamMachineAuth" in c.name)
# Handle steam.tv URLs
if self._steamtv_url_re.match(self.url) is not None:
# extract the steam ID from the page
res = self.session.http.get(self.url)
for div in itertags(res.text, 'div'):
if div.attributes.get("id") == "webui_config":
broadcast_data = html_unescape(div.attributes.get("data-broadcast"))
steamid = parse_json(broadcast_data).get("steamid")
self.url = self._watch_broadcast_url + steamid
# extract the steam ID from the URL
steamid = self._url_re.match(self.url).group(1)
res = self.session.http.get(self.url) # get the page to set some cookies
sessionid = res.cookies.get('sessionid')
while streamdata is None or streamdata[u"success"] in ("waiting", "waiting_for_start"):
streamdata = self._get_broadcast_stream(steamid,
sessionid=sessionid)
if streamdata[u"success"] == "ready":
return DASHStream.parse_manifest(self.session, streamdata["url"])
elif streamdata[u"success"] == "unavailable":
log.error("This stream is currently unavailable")
return
else:
r = streamdata[u"retry"] / 1000.0
log.info("Waiting for stream, will retry again in {} seconds...".format(r))
time.sleep(r)
def _get_stream_info(self):
res = http.get(self.url)
match = re.search("window.config = ({.+})", res.text)
if match:
config = match.group(1)
return parse_json(config, "config JSON",
schema=_stream_config_schema)
def _get_stream_info(self):
res = http.get(self.url)
match = re.search("window.config = ({.+})", res.text)
if match:
config = match.group(1)
return parse_json(config, "config JSON",
schema=_stream_config_schema)
def _get_streams(self):
streamdata = None
if self.get_option("email"):
if self.login(self.get_option("email"), self.get_option("password")):
log.info("Logged in as {0}".format(self.get_option("email")))
self.save_cookies(lambda c: "steamMachineAuth" in c.name)
# Handle steam.tv URLs
if self._steamtv_url_re.match(self.url) is not None:
# extract the steam ID from the page
res = self.session.http.get(self.url)
for div in itertags(res.text, 'div'):
if div.attributes.get("id") == "webui_config":
broadcast_data = html_unescape(div.attributes.get("data-broadcast"))
steamid = parse_json(broadcast_data).get("steamid")
self.url = self._watch_broadcast_url + steamid
# extract the steam ID from the URL
steamid = self._url_re.match(self.url).group(1)
res = self.session.http.get(self.url) # get the page to set some cookies
sessionid = res.cookies.get('sessionid')
while streamdata is None or streamdata[u"success"] in ("waiting", "waiting_for_start"):
streamdata = self._get_broadcast_stream(steamid,
sessionid=sessionid)
if streamdata[u"success"] == "ready":
return DASHStream.parse_manifest(self.session, streamdata["url"])
elif streamdata[u"success"] == "unavailable":
log.error("This stream is currently unavailable")
return
else:
r = streamdata[u"retry"] / 1000.0
log.info("Waiting for stream, will retry again in {} seconds...".format(r))
time.sleep(r)
def _get_hls_streams(self, stream_type="live"):
self.logger.debug("Getting {0} HLS streams for {1}".format(
stream_type, self.channel))
self._authenticate()
self._hosted_chain.append(self.channel)
if stream_type == "live":
hosted_channel = self._check_for_host()
if hosted_channel and self.options.get("disable_hosting"):
self.logger.info("hosting was disabled by command line option")
elif hosted_channel:
self.logger.info("switching to {0}", hosted_channel)
if hosted_channel in self._hosted_chain:
self.logger.error(
u"A loop of hosted channels has been detected, "
"cannot find a playable stream. ({0})".format(
u" -> ".join(self._hosted_chain +
[hosted_channel])))
return {}
self.channel = hosted_channel
return self._get_hls_streams(stream_type)
# only get the token once the channel has been resolved
sig, token = self._access_token(stream_type)
url = self.usher.channel(self.channel,
sig=sig,
token=token,
fast_bread=True)
elif stream_type == "video":
sig, token = self._access_token(stream_type)
url = self.usher.video(self.video_id, nauthsig=sig, nauth=token)
else:
self.logger.debug(
"Unknown HLS stream type: {0}".format(stream_type))
return {}
try:
# If the stream is a VOD that is still being recorded the stream should start at the
# beginning of the recording
streams = HLSStream.parse_variant_playlist(
self.session, url, force_restart=not stream_type == "live")
except IOError as err:
err = str(err)
if "404 Client Error" in err or "Failed to parse playlist" in err:
return
else:
raise PluginError(err)
try:
token = parse_json(token, schema=_token_schema)
for name in token["restricted_bitrates"]:
if name not in streams:
self.logger.warning(
"The quality '{0}' is not available "
"since it requires a subscription.", name)
except PluginError:
pass
return streams
def parse_token(cls, tokenstr):
return parse_json(tokenstr, schema=validate.Schema(
{"chansub": {"restricted_bitrates": validate.all(
[str],
validate.filter(lambda n: not re.match(r"(.+_)?archives|live|chunked", n))
)}},
validate.get(("chansub", "restricted_bitrates"))
))
def _get_streams(self):
res = http.get(self.url)
match = (_stream_hls_re.search(res.text) or
_stream_data_re.search(res.text))
if not match:
return
stream_url = parse_json(match.group(1))
return HLSStream.parse_variant_playlist(self.session, stream_url)
def _get_streams(self):
res = http.get(self.url)
match = (_stream_hls_re.search(res.text)
or _stream_data_re.search(res.text))
if not match:
return
stream_url = parse_json(match.group(1))
return HLSStream.parse_variant_playlist(self.session, stream_url)
def _find_store(self, res, name):
match = _store_data_re.search(res.text)
if not match:
return
stores_data = parse_json(match.group(1).replace('\\"', '"'),
schema=_store_schema)
if not stores_data:
return
for store in filter(lambda s: s["instanceName"] == name, stores_data):
return store
def _get_hls_streams(self, stream_type="live"):
self.logger.debug("Getting {0} HLS streams for {1}".format(stream_type, self.channel))
self._authenticate()
self._hosted_chain.append(self.channel)
if stream_type == "live":
hosted_channel = self._check_for_host()
if hosted_channel and self.options.get("disable_hosting"):
self.logger.info("hosting was disabled by command line option")
elif hosted_channel:
self.logger.info("switching to {0}", hosted_channel)
if hosted_channel in self._hosted_chain:
self.logger.error(u"A loop of hosted channels has been detected, "
"cannot find a playable stream. ({0})".format(
u" -> ".join(self._hosted_chain + [hosted_channel])))
return {}
self.channel = hosted_channel
return self._get_hls_streams(stream_type)
# only get the token once the channel has been resolved
sig, token = self._access_token(stream_type)
url = self.usher.channel(self.channel, sig=sig, token=token)
elif stream_type == "video":
sig, token = self._access_token(stream_type)
url = self.usher.video(self.video_id, nauthsig=sig, nauth=token)
else:
self.logger.debug("Unknown HLS stream type: {0}".format(stream_type))
return {}
try:
# If the stream is a VOD that is still being recorded the stream should start at the
# beginning of the recording
streams = HLSStream.parse_variant_playlist(self.session, url,
force_restart=not stream_type == "live")
except IOError as err:
err = str(err)
if "404 Client Error" in err or "Failed to parse playlist" in err:
return
else:
raise PluginError(err)
try:
token = parse_json(token, schema=_token_schema)
for name in token["restricted_bitrates"]:
if name not in streams:
self.logger.warning("The quality '{0}' is not available "
"since it requires a subscription.",
name)
except PluginError:
pass
return streams
def _get_hls_streams(self, type="live"):
self.logger.debug("Getting {} HLS streams for {}".format(
type, self.channel))
self._authenticate()
hosted_channel = self._check_for_host()
if hosted_channel and self.options.get("disable_hosting"):
self.logger.info("hosting was disabled by command line option")
return {}
elif hosted_channel:
self.logger.info("switching to {}", hosted_channel)
if hosted_channel in self._hosted_chain:
self._hosted_chain.append(hosted_channel)
self.logger.error(
u"A loop of hosted channels has been detected, "
"cannot find a playable stream. ({})".format(u" -> ".join(
self._hosted_chain)))
return {}
self._hosted_chain.append(hosted_channel)
self.channel = hosted_channel
return self._get_hls_streams(type)
sig, token = self._access_token(type)
if type == "live":
url = self.usher.channel(self.channel, sig=sig, token=token)
elif type == "video":
url = self.usher.video(self.video_id, nauthsig=sig, nauth=token)
else:
self.logger.debug("Unknown HLS stream type: {}".format(type))
return {}
try:
streams = HLSStream.parse_variant_playlist(self.session, url)
except IOError as err:
err = str(err)
if "404 Client Error" in err or "Failed to parse playlist" in err:
return
else:
raise PluginError(err)
try:
token = parse_json(token, schema=_token_schema)
for name in token["restricted_bitrates"]:
if name not in streams:
self.logger.warning(
"The quality '{0}' is not available "
"since it requires a subscription.", name)
except PluginError:
pass
return streams
def _get_streams(self):
room_id = self.match.group("room_id")
res = self.session.http.get(self.api_url.format(room_id))
data = self._data_re.search(res.text)
if not data:
return
try:
hls_url = parse_json(data.group("data"), schema=self._data_schema)
except Exception:
raise NoStreamsError(self.url)
log.debug("URL={0}".format(hls_url))
return {"live": HLSStream(self.session, hls_url)}
def _get_streams(self):
page = http.get(self.url)
match = SOURCES_RE.search(page.text)
if match is None:
return
sources = parse_json(match.group(1))
if "sources" not in sources or not isinstance(sources["sources"], list):
return
for source in sources["sources"]:
if "src" not in source or not source["src"].endswith(".m3u8"):
continue
yield "live", HLSStream(self.session, source["src"], headers={"User-Agent": USER_AGENT})
def _find_channel_id(self, text):
match = self._stream_id_re.search(text)
if match:
return match.group(1)
match = self._site_data_re.search(text)
if match:
r_json = parse_json(match.group("data"))
if r_json:
mlg_channel_id = r_json.get("mlg_channel_id")
if mlg_channel_id:
res = http.get(self.CHANNEL_API.format(mlg_channel_id))
channel_id = http.json(res, schema=self._site_data_schema)
return channel_id
match = self._player_embed_re.search(text)
if match:
return match.group("channel_id")
def _find_channel_id(self, text):
match = self._stream_id_re.search(text)
if match:
return match.group(1)
match = self._site_data_re.search(text)
if match:
r_json = parse_json(match.group("data"))
if r_json:
mlg_channel_id = r_json.get("mlg_channel_id")
if mlg_channel_id:
res = http.get(self.CHANNEL_API.format(mlg_channel_id))
channel_id = http.json(res, schema=self._site_data_schema)
return channel_id
match = self._player_embed_re.search(text)
if match:
return match.group("channel_id")
def _get_streams(self):
res = self.session.http.get(self.url)
m = self._config_re.search(res.text)
if not m:
log.debug("Unable to find _config_re")
return
stream_info = parse_json(m.group(1), "config JSON",
schema=self._stream_config_schema)
log.trace("stream_info: {0!r}".format(stream_info))
if not (stream_info and stream_info["is_live"]):
log.debug("Stream might be Off Air")
return
m3u8_url = stream_info.get("secure_m3u8_url")
if m3u8_url:
for s in HLSStream.parse_variant_playlist(self.session, m3u8_url).items():
yield s
def _get_streams(self):
match = self._url_re.match(self.url)
if not match:
return
room_id = match.group("room_id")
res = http.get(self.api_url.format(room_id))
data = self._data_re.search(res.text)
if not data:
return
try:
hls_url = parse_json(data.group("data"), schema=self._data_schema)
except Exception as e:
raise NoStreamsError(self.url)
self.logger.debug("URL={0}".format(hls_url))
return {"live": HLSStream(self.session, hls_url)}
def _resolve_stream(self):
res = http.get(self.url)
match = _data_stream_re.search(res.text)
if not match:
return
data_stream = match.group(1)
resolve_data = {'stream': data_stream}
res = http.post('http://www-ipv4.nos.nl/livestream/resolve/',
data=json.dumps(resolve_data))
data = http.json(res)
res = http.get(data['url'])
match = _js_re.search(res.text)
if not match:
return
stream_url = parse_json(match.group(1))
return HLSStream.parse_variant_playlist(self.session, stream_url)
def _get_live_stream(self, export_url):
res = http.get(export_url)
match = _live_json_re.search(res.text)
if not match:
return
json = parse_json(match.group(1), schema=_live_schema)
streams = {}
for stream in json["streams"]:
stream_name = stream["quality"]
parsed = urlparse(stream["url"])
stream = RTMPStream(self.session, {
"rtmp": stream["url"],
"app": "{0}?{1}".format(parsed.path[1:], parsed.query),
"playpath": stream["name"],
"swfVfy": SWF_LIVE_URL,
"pageUrl": self.url,
"live": True
})
streams[stream_name] = stream
return streams
def _get_live_stream(self, export_url):
res = http.get(export_url)
match = _live_json_re.search(res.text)
if not match:
return
json = parse_json(match.group(1), schema=_live_schema)
streams = {}
for stream in json["streams"]:
stream_name = stream["quality"]
parsed = urlparse(stream["url"])
stream = RTMPStream(self.session, {
"rtmp": stream["url"],
"app": "{0}?{1}".format(parsed.path[1:], parsed.query),
"playpath": stream["name"],
"swfVfy": SWF_LIVE_URL,
"pageUrl": self.url,
"live": True
})
streams[stream_name] = stream
return streams
def _resolve_stream(self):
res = http.get(self.url)
match = _data_stream_re.search(res.text)
if not match:
return
data_stream = match.group(1)
resolve_data = {
'stream': data_stream
}
res = http.post(
'http://www-ipv4.nos.nl/livestream/resolve/',
data=json.dumps(resolve_data)
)
data = http.json(res)
res = http.get(data['url'])
match = _js_re.search(res.text)
if not match:
return
stream_url = parse_json(match.group(1))
return HLSStream.parse_variant_playlist(self.session, stream_url)
def _find_stream_id(self, text):
match = self._player_config_re.search(text)
if match:
stream_id = parse_json(match.group(1),
schema=self._player_config_schema)
return stream_id
def _get_streams(self):
page = http.get(self.url, schema=_schema)
if not page:
return
pubkey_pem = get_public_key(self.cache, urljoin(self.url, page["clientlibs"]))
if not pubkey_pem:
raise PluginError("Unable to get public key")
flashvars = page["flashvars"]
params = {
"cashPath": int(time.time() * 1000)
}
res = http.get(urljoin(self.url, flashvars["country"]), params=params)
if not res:
return
language = http.xml(res, schema=_language_schema)
api_params = {}
for key in ("ss_id", "mv_id", "device_cd", "ss1_prm", "ss2_prm", "ss3_prm"):
if flashvars.get(key, ""):
api_params[key] = flashvars[key]
aeskey = number.long_to_bytes(random.getrandbits(8 * 32), 32)
params = {
"s": flashvars["s"],
"c": language,
"e": self.url,
"d": aes_encrypt(aeskey, json.dumps(api_params)),
"a": rsa_encrypt(pubkey_pem, aeskey)
}
res = http.get(urljoin(self.url, flashvars["init"]), params=params)
if not res:
return
rtn = http.json(res, schema=_init_schema)
if not rtn:
return
init_data = parse_json(aes_decrypt(aeskey, rtn))
parsed = urlparse(init_data["play_url"])
if parsed.scheme != "https" or not parsed.path.startswith("/i/") or not parsed.path.endswith("/master.m3u8"):
return
hlsstream_url = init_data["play_url"]
streams = HLSStream.parse_variant_playlist(self.session, hlsstream_url)
if "caption_url" in init_data:
if self.get_option("mux_subtitles") and FFMPEGMuxer.is_usable(self.session):
res = http.get(init_data["caption_url"])
srt = http.xml(res, ignore_ns=True, schema=_xml_to_srt_schema)
subfiles = []
metadata = {}
for i, lang, srt in ((i, s[0], s[1]) for i, s in enumerate(srt)):
subfile = tempfile.TemporaryFile()
subfile.write(srt.encode("utf8"))
subfile.seek(0)
subfiles.append(FileStream(self.session, fileobj=subfile))
metadata["s:s:{0}".format(i)] = ["language={0}".format(lang)]
for n, s in streams.items():
yield n, MuxedStream(self.session, s, *subfiles,
maps=list(range(0, len(metadata) + 1)),
metadata=metadata)
return
else:
self.logger.info("Subtitles: {0}".format(init_data["caption_url"]))
for s in streams.items():
yield s
def _find_stream_id(self, text):
match = _player_config_re.search(text)
if match:
stream_id = parse_json(match.group(1),
schema=_player_config_schema)
return stream_id
