class TestOptions(unittest.TestCase):
def setUp(self):
self.options = Options({"a_default": "default"})
def test_options(self):
self.assertEqual(self.options.get("a_default"), "default")
self.assertEqual(self.options.get("non_existing"), None)
self.options.set("a_option", "option")
self.assertEqual(self.options.get("a_option"), "option")
class TestOptions(unittest.TestCase):
def setUp(self):
self.options = Options({
"a_default": "default"
})
def test_options(self):
self.assertEqual(self.options.get("a_default"), "default")
self.assertEqual(self.options.get("non_existing"), None)
self.options.set("a_option", "option")
self.assertEqual(self.options.get("a_option"), "option")
class TestPlugin(Plugin):
options = Options({"a_option": "default"})
@classmethod
def can_handle_url(self, url):
return "test.se" in url
def _get_streams(self):
streams = {}
streams["rtmp"] = RTMPStream(self.session, dict(rtmp="rtmp://test.se"))
streams["hls"] = HLSStream(self.session,
"http://test.se/playlist.m3u8")
streams["http"] = HTTPStream(self.session, "http://test.se/stream")
streams["akamaihd"] = AkamaiHDStream(self.session,
"http://test.se/stream")
streams["240p"] = HTTPStream(self.session, "http://test.se/stream")
streams["360p"] = HTTPStream(self.session, "http://test.se/stream")
streams["1080p"] = HTTPStream(self.session, "http://test.se/stream")
streams["350k"] = HTTPStream(self.session, "http://test.se/stream")
streams["800k"] = HTTPStream(self.session, "http://test.se/stream")
streams["1500k"] = HTTPStream(self.session, "http://test.se/stream")
streams["3000k"] = HTTPStream(self.session, "http://test.se/stream")
streams["480p"] = [
HTTPStream(self.session, "http://test.se/stream"),
RTMPStream(self.session, dict(rtmp="rtmp://test.se"))
]
streams.update(testplugin_support.get_streams(self.session))
return streams
class Twitch(JustinTVPluginBase):
options = Options({
"cookie": None,
"oauth_token": None,
"password": None
})
@classmethod
def can_handle_url(self, url):
return _url_re.match(url)
def __init__(self, url):
JustinTVPluginBase.__init__(self, url)
self.api = TwitchAPI(host="twitch.tv",
beta=self.subdomain == "beta")
def _authenticate(self):
oauth_token = self.options.get("oauth_token")
if oauth_token and not self.api.oauth_token:
self.logger.info("Attempting to authenticate using OAuth token")
self.api.oauth_token = oauth_token
user = self.api.user(schema=_user_schema)
if user:
self.logger.info("Successfully logged in as {0}", user)
else:
self.logger.error("Failed to authenticate, the access token "
"is not valid")
else:
return JustinTVPluginBase._authenticate(self)
def _get_video_streams(self):
self._authenticate()
if self.video_type == "b":
self.video_type = "a"
try:
videos = self.api.videos(self.video_type + self.video_id,
schema=_video_schema)
except PluginError as err:
if "HTTP/1.1 0 ERROR" in str(err):
raise NoStreamsError(self.url)
else:
raise
# Parse the "t" query parameter on broadcasts and adjust
# start offset if needed.
time_offset = self.params.get("t")
if time_offset:
videos["start_offset"] += time_to_offset(self.params.get("t"))
return self._create_playlist_streams(videos)
class Plugin(object):
"""
A plugin can retrieve stream information from the *url* specified.
"""
options = Options()
def __init__(self, url):
self.url = url
self.logger = self.session.logger.new_module("plugin." + self.module)
@classmethod
def can_handle_url(cls, url):
raise NotImplementedError
@classmethod
def set_option(cls, key, value):
cls.options.set(key, value)
@classmethod
def get_option(cls, key):
return cls.options.get(key)
def get_streams(self):
"""
Retrieves and returns a :class:`dict` containing the streams.
The key is the name of the stream, most commonly the quality.
The value is a :class:`Stream` object.
The stream with key *best* is a reference to the stream most likely
to be of highest quality.
"""
streams = self._get_streams()
best = (0, None)
for name, stream in streams.items():
weight = qualityweight(name)
if weight > best[0]:
best = (weight, stream)
if best[1] is not None:
streams["best"] = best[1]
return streams
def _get_streams(self):
raise NotImplementedError
class GomTV(Plugin):
BaseURL = "http://www.gomtv.net"
LiveURL = BaseURL + "/main/goLive.gom"
LoginURL = "https://ssl.gomtv.net/userinfo/loginProcess.gom"
LoginCheckURL = BaseURL + "/forum/list.gom?m=my"
GOXVODURL = BaseURL + "/gox/ggox.gom"
KeyCheckPort = 63800
LoginHeaders = {"Referer": BaseURL}
StreamHeaders = {"User-Agent": "KPeerClient"}
options = Options({
"cookie": None,
"username": None,
"password": None,
})
@classmethod
def can_handle_url(self, url):
return "gomtv.net" in url
def __init__(self, url):
parsed = urlparse(url)
# Attempt to resolve current live URL if main page is passed
if len(parsed.path) <= 1:
url = self.LiveURL
Plugin.__init__(self, url)
def _get_streams(self):
self.rsession = requests.session(prefetch=True)
options = self.options
if options.get("cookie"):
self._authenticate(cookies=options.get("cookie"))
else:
self._authenticate(options.get("username"),
options.get("password"))
if "/vod/" in self.url:
return self._get_vod_streams()
else:
return self._get_live_streams()
def _get_vod_streams(self):
res = urlget(self.url)
flashvars = re.search("FlashVars=\"(.+?)\"", res.text)
if not flashvars:
raise PluginError("Unable to find flashvars on page")
flashvars = parse_qs(flashvars.group(1))
for var in ("vjoinid", "conid", "leagueid"):
if not var in flashvars:
raise PluginError(
("Missing key '{0}' in flashvars").format(var))
streams = {}
qualities = ["SQ", "HQ"]
for quality in qualities:
params = dict(leagueid=flashvars["leagueid"][0],
vjoinid=flashvars["vjoinid"][0],
conid=flashvars["conid"][0],
title="",
ref="",
tmpstamp=int(time.time()),
strLevel=quality)
res = urlget(self.GOXVODURL, params=params, session=self.rsession)
if res.text != "1002" and len(res.text) > 0:
gox = self._parse_gox_file(res.text)
entry = gox[0]
nokey = False
for var in ("NODEIP", "NODEID", "UNO", "USERIP"):
if not var in entry:
nokey = True
if nokey:
self.logger.warning(
"Unable to fetch key, make sure that you have access to this VOD"
)
continue
key = self._check_vod_key(entry["NODEIP"], entry["NODEID"],
entry["UNO"], entry["USERIP"])
streams[quality.lower()] = HTTPStream(
self.session,
gox[0]["REF"],
params=dict(key=key),
headers=self.StreamHeaders)
return streams
def _get_live_streams(self):
streams = {}
qualities = ["HQ", "SQ", "HQTest", "SQTest"]
res = self._get_live_page(self.url)
goxurl = self._find_gox_url(res.text)
if not goxurl:
raise PluginError("Unable to find GOX URL")
for quality in qualities:
# Grab the response of the URL listed on the Live page for a stream
url = goxurl.format(quality=quality)
res = urlget(url, session=self.rsession)
# The response for the GOX XML if an incorrect stream quality is chosen is 1002.
if res.text != "1002" and len(res.text) > 0:
gox = self._parse_gox_file(res.text)
streams[quality.lower()] = HTTPStream(
self.session, gox[0]["REF"], headers=self.StreamHeaders)
return streams
def _authenticate(self, username=None, password=None, cookies=None):
if (username is None or password is None) and cookies is None:
raise PluginError(
"GOMTV.net requires a username and password or a cookie")
if cookies is not None:
for cookie in cookies.split(";"):
try:
name, value = cookie.split("=")
except ValueError:
continue
self.rsession.cookies[name.strip()] = value.strip()
self.logger.info("Attempting to authenticate with cookies")
else:
form = dict(cmd="login",
rememberme="1",
mb_username=username,
mb_password=password)
self.logger.info(
"Attempting to authenticate with username and password")
urlopen(self.LoginURL,
data=form,
headers=self.LoginHeaders,
session=self.rsession)
res = urlget(self.LoginCheckURL, session=self.rsession)
if "Please need login" in res.text:
raise PluginError("Authentication failed")
if "SES_USERNICK" in self.rsession.cookies:
username = self.rsession.cookies["SES_USERNICK"]
self.logger.info(
("Successfully logged in as {0}").format(username))
if username and password:
cookie = ""
for v in ("SES_USERNO", "SES_STATE", "SES_MEMBERNICK",
"SES_USERNICK"):
if v in self.rsession.cookies:
cookie += "{0}={1}; ".format(v, self.rsession.cookies[v])
self.logger.info("Cookie for reusing this session: {0}", cookie)
def _check_vod_key(self, nodeip, nodeid, userno, userip):
try:
conn = socket.create_connection((nodeip, self.KeyCheckPort),
timeout=15)
except socket.error as err:
raise PluginError(
("Failed to connect to key check server: {0}").format(
str(err)))
msg = "Login,0,{userno},{nodeid},{userip}\n".format(nodeid=nodeid,
userno=userno,
userip=userip)
try:
conn.sendall(bytes(msg, "ascii"))
res = conn.recv(4096)
except IOError as err:
raise PluginError(
("Failed to communicate with key check server: {0}").format(
str(err)))
if len(res) == 0:
raise PluginError("Empty response from key check server")
conn.close()
res = str(res, "ascii").strip().split(",")
return res[-1]
def _get_event_url(self, prefix, data):
match = re.search(' \"(.*)\";', data)
if not match:
raise PluginError("Event live page URL not found")
return urljoin(prefix, match.group(1))
def _get_live_page(self, url):
res = urlget(url, session=self.rsession)
# If a special event occurs, we know that the live page response
# will just be some JavaScript that redirects the browser to the
# real live page. We assume that the entireity of this JavaScript
# is less than 200 characters long, and that real live pages are
# more than that.
if len(res.text) < 200:
# Grabbing the real live page URL
url = self._parse_event_url(url, res.text)
res = urlget(url, session=self.rsession)
return res
def _find_gox_url(self, data):
url = None
# Parsing through the live page for a link to the gox XML file.
# Quality is simply passed as a URL parameter e.g. HQ, SQ, SQTest
try:
patternhtml = "[^/]+var.+(http://www.gomtv.net/gox[^;]+;)"
url = re.search(patternhtml, data).group(1)
url = re.sub('\" \+ playType \+ \"', "{quality}", url)
except AttributeError:
raise PluginError(
"Unable to find the majority of the GOMTV.net XML URL on the Live page"
)
# Finding the title of the stream, probably not necessary but
# done for completeness
try:
patterntitle = "this\.title[^;]+;"
title = re.search(patterntitle, data).group(0)
title = re.search('\"(.*)\"', title).group(0)
title = re.sub('"', "", title)
url = re.sub('"\+ tmpThis.title[^;]+;', title, url)
except AttributeError:
raise PluginError(
"Unable to find the stream title on the Live page")
return url
def _get_node_text(self, element):
res = []
for node in element.childNodes:
if node.nodeType == node.TEXT_NODE:
res.append(node.data)
if len(res) == 0:
return None
else:
return "".join(res)
def _parse_gox_file(self, data):
try:
dom = xml.dom.minidom.parseString(data)
except Exception as err:
raise PluginError(("Unable to parse gox file: {0})").format(err))
entries = []
for xentry in dom.getElementsByTagName("ENTRY"):
entry = {}
for child in xentry.childNodes:
if isinstance(child, xml.dom.minidom.Element):
if child.tagName == "REF":
href = child.getAttribute("href")
# SQ and SQTest streams can be gomp2p links, with actual stream address passed as a parameter.
if href.startswith("gomp2p://"):
href, n = re.subn("^.*LiveAddr=", "", href)
href = unquote(href)
entry[child.tagName] = href
else:
entry[child.tagName] = self._get_node_text(child)
entries.append(entry)
return entries
class GomTV(Plugin):
"""
Implements authentication to the GomTV website.
"""
BaseURL = "http://www.gomtv.net"
LiveURL = BaseURL + "/main/goLive.gom"
LoginURL = "https://ssl.gomtv.net/userinfo/loginProcess.gom"
LoginCheckURL = BaseURL + "/forum/list.gom?m=my"
LoginHeaders = {"Referer": BaseURL}
options = Options({
"cookie": None,
"username": None,
"password": None,
})
@classmethod
def can_handle_url(self, url):
return "gomtv.net" in url
def __init__(self, url):
parsed = urlparse(url)
# Attempt to resolve current live URL if main page is passed
if len(parsed.path) <= 1:
url = self.LiveURL
Plugin.__init__(self, url)
def _get_streams(self):
self.rsession = requests.session()
options = self.options
if options.get("cookie"):
self._authenticate(cookies=options.get("cookie"))
else:
self._authenticate(options.get("username"),
options.get("password"))
res = urlget(self.url, session=self.rsession)
player = GomTV3(self.url, res, self.rsession)
streams = {}
if "/vod/" in self.url:
return player.get_vod_streams()
else:
try:
streams.update(player.get_live_streams())
except NoStreamsError:
pass
try:
streams.update(player.get_alt_live_streams())
except NoStreamsError:
pass
try:
streams.update(player.get_limelight_live_streams())
except NoStreamsError:
pass
return streams
def _authenticate(self, username=None, password=None, cookies=None):
if (username is None or password is None) and cookies is None:
raise PluginError(
"GOMTV.net requires a username and password or a cookie")
if cookies is not None:
for cookie in cookies.split(";"):
try:
name, value = cookie.split("=")
except ValueError:
continue
self.rsession.cookies[name.strip()] = value.strip()
self.logger.info("Attempting to authenticate with cookies")
else:
form = dict(cmd="login",
rememberme="1",
mb_username=username,
mb_password=password)
self.logger.info(
"Attempting to authenticate with username and password")
urlopen(self.LoginURL,
data=form,
headers=self.LoginHeaders,
session=self.rsession)
res = urlget(self.LoginCheckURL, session=self.rsession)
if "Please need login" in res.text:
raise PluginError("Authentication failed")
if "SES_USERNICK" in self.rsession.cookies:
username = self.rsession.cookies["SES_USERNICK"]
self.logger.info(
("Successfully logged in as {0}").format(username))
if username and password:
cookie = ""
for v in ("SES_MEMBERNO", "SES_STATE", "SES_MEMBERNICK",
"SES_USERNICK"):
if v in self.rsession.cookies:
cookie += "{0}={1}; ".format(v, self.rsession.cookies[v])
self.logger.info("Cookie for reusing this session: {0}", cookie)
def _parse_event_url(self, prefix, data):
match = re.search(' \"(.*)\";', data)
if not match:
raise PluginError("Event live page URL not found")
return urljoin(prefix, match.group(1))
def _get_live_page(self, res):
# If a special event occurs, we know that the live page response
# will just be some JavaScript that redirects the browser to the
# real live page. We assume that the entireity of this JavaScript
# is less than 200 characters long, and that real live pages are
# more than that.
if len(res.text) < 200:
# Grabbing the real live page URL
url = self._parse_event_url(url, res.text)
res = urlget(url, session=self.rsession)
return res
class PluginBase(Plugin):
options = Options({
"cookie": None,
"password": None,
})
@classmethod
def stream_weight(cls, key):
weight = QUALITY_WEIGHTS.get(key)
if weight:
return weight, "justintv"
return Plugin.stream_weight(key)
def __init__(self, url):
Plugin.__init__(self, url)
try:
match = re.match(URL_PATTERN, url).groupdict()
self.channel = match.get("channel").lower()
self.video_type = match.get("video_type")
self.video_id = match.get("video_id")
self.usher = UsherService(match.get("domain"))
parsed = urlparse(url)
self.params = parse_qsd(parsed.query)
except AttributeError:
self.channel = None
self.params = None
self.video_id = None
self.video_type = None
self.usher = None
def _create_playlist_streams(self, videos):
start_offset = int(videos.get("start_offset", 0))
stop_offset = int(videos.get("end_offset", 0))
streams = {}
for quality, chunks in videos.get("chunks").items():
if not chunks:
if videos.get("restrictions", {}).get(quality) == "chansub":
self.logger.warning(
"The quality '{0}' is not available "
"since it requires a subscription.", quality)
continue
# Rename 'live' to 'source'
if quality == "live":
quality = "source"
chunks_duration = sum(c.get("length") for c in chunks)
# If it's a full broadcast we just use all the chunks
if start_offset == 0 and chunks_duration == stop_offset:
# No need to use the FLV concat if it's just one chunk
if len(chunks) == 1:
url = chunks[0].get("url")
stream = HTTPStream(self.session, url)
else:
chunks = [
HTTPStream(self.session, c.get("url")) for c in chunks
]
stream = FLVPlaylist(self.session,
chunks,
duration=chunks_duration)
else:
try:
stream = self._create_video_clip(chunks, start_offset,
stop_offset)
except StreamError as err:
self.logger.error(
"Error while creating video clip '{0}': {1}", quality,
err)
continue
streams[quality] = stream
return streams
def _create_video_clip(self, chunks, start_offset, stop_offset):
playlist_duration = stop_offset - start_offset
playlist_offset = 0
playlist_streams = []
playlist_tags = []
for chunk in chunks:
chunk_url = chunk.get("url")
chunk_length = chunk.get("length")
chunk_start = playlist_offset
chunk_stop = chunk_start + chunk_length
chunk_stream = HTTPStream(self.session, chunk_url)
if start_offset >= chunk_start and start_offset <= chunk_stop:
headers = extract_flv_header_tags(chunk_stream)
if not headers.metadata:
raise StreamError(
"Missing metadata tag in the first chunk")
metadata = headers.metadata.data.value
keyframes = metadata.get("keyframes")
if not keyframes:
raise StreamError(
"Missing keyframes info in the first chunk")
keyframe_offset = None
keyframe_offsets = keyframes.get("filepositions")
keyframe_times = [
playlist_offset + t for t in keyframes.get("times")
]
for time, offset in izip(keyframe_times, keyframe_offsets):
if time > start_offset:
break
keyframe_offset = offset
if keyframe_offset is None:
raise StreamError("Unable to find a keyframe to seek to "
"in the first chunk")
chunk_headers = dict(
Range="bytes={0}-".format(int(keyframe_offset)))
chunk_stream = HTTPStream(self.session,
chunk_url,
headers=chunk_headers)
playlist_streams.append(chunk_stream)
for tag in headers:
playlist_tags.append(tag)
elif chunk_start >= start_offset and chunk_start < stop_offset:
playlist_streams.append(chunk_stream)
playlist_offset += chunk_length
return FLVPlaylist(self.session,
playlist_streams,
tags=playlist_tags,
duration=playlist_duration)
def _get_streams(self):
if not self.channel:
return
if self.video_id:
return self._get_video_streams()
else:
return self._get_live_streams()
def _authenticate(self):
cookies = self.options.get("cookie")
if cookies and not self.api.oauth_token:
self.logger.info("Attempting to authenticate using cookies")
self.api.add_cookies(cookies)
self.api.oauth_token = self.api.token()
viewer = self.api.viewer_info()
login = viewer.get("login")
if login:
self.logger.info("Successfully logged in as {0}", login)
else:
self.logger.error("Failed to authenticate, your cookies "
"may have expired")
def _access_token(self):
try:
sig, token = self.api.channel_access_token(self.channel)
except PluginError as err:
if "404 Client Error" in str(err):
raise NoStreamsError(self.url)
else:
raise
return sig, token
def _get_live_streams(self):
self._authenticate()
sig, token = self._access_token()
url = self.usher.select(self.channel,
password=self.options.get("password"),
nauthsig=sig,
nauth=token)
try:
streams = HLSStream.parse_variant_playlist(self.session, url)
except IOError as err:
err = str(err)
if "404 Client Error" in err or "Failed to parse playlist" in err:
return
else:
raise PluginError(err)
try:
token = parse_json(token)
chansub = verifyjson(token, "chansub")
restricted_bitrates = verifyjson(chansub, "restricted_bitrates")
for name in filter(
lambda n: not re.match(r"(.+_)?archives|live", n),
restricted_bitrates):
self.logger.warning(
"The quality '{0}' is not available "
"since it requires a subscription.", name)
except PluginError:
pass
return dict(starmap(self._check_stream_name, streams.items()))
def _get_video_streams(self):
pass
def _check_stream_name(self, name, stream):
if name.startswith("iphone"):
name = name.replace("iphone", "")
return name, stream
def setUp(self):
self.options = Options({
"a_default": "default"
})
class JustinTV(Plugin):
options = Options({
"cookie": None
})
APIBaseURL = "http://usher.justin.tv"
StreamInfoURL = APIBaseURL + "/find/{0}.json"
MetadataURL = "http://www.justin.tv/meta/{0}.xml?on_site=true"
SWFURL = "http://www.justin.tv/widgets/live_embed_player.swf"
HLSStreamTokenKey = b"Wd75Yj9sS26Lmhve"
HLSStreamTokenURL = APIBaseURL + "/stream/iphone_token/{0}.json"
HLSPlaylistURL = APIBaseURL + "/stream/multi_playlist/{0}.m3u8"
HLSTranscodeRequest = APIBaseURL + "/stream/transcode_iphone.json"
@classmethod
def can_handle_url(self, url):
return ("justin.tv" in url) or ("twitch.tv" in url)
def _get_channel_name(self, url):
parts = urlparse(url).path.split("/")
if len(parts) >= 2 and len(parts[1]) > 0:
return parts[1].lower()
def _get_metadata(self):
url = self.MetadataURL.format(self.channelname)
cookies = {}
for cookie in self.options.get("cookie").split(";"):
try:
name, value = cookie.split("=")
except ValueError:
continue
cookies[name.strip()] = value.strip()
res = urlget(url, cookies=cookies)
meta = res_xml(res, "metadata XML")
metadata = {}
metadata["access_guid"] = meta.findtext("access_guid")
metadata["login"] = meta.findtext("login")
metadata["title"] = meta.findtext("title")
return metadata
def _authenticate(self):
if self.options.get("cookie") is not None:
self.logger.info("Attempting to authenticate using cookies")
metadata = self._get_metadata()
chansub = metadata.get("access_guid")
login = metadata.get("login")
if login:
self.logger.info("Successfully logged in as {0}", login)
return chansub
# The HTTP support in rtmpdump's SWF verification is extremly
# basic, therefore we have to work around it.
#
# At first it seemed like resolving the 302 redirect was enough,
# but it seems the resolved URLs also redirects sometimes causing
# rtmpdump to fail. Safest to just to do the verification ourself.
def _verify_swf(self):
swfurl = urlresolve(self.SWFURL)
# For some reason the URL returned sometimes contain random
# user-agent/referer query parameters, let's strip them
# so we actually cache.
if "?" in swfurl:
swfurl = swfurl[:swfurl.find("?")]
cachekey = "swf:{0}".format(swfurl)
swfhash, swfsize = self.cache.get(cachekey, (None, None))
if not (swfhash and swfsize):
self.logger.debug("Verifying SWF")
swfhash, swfsize = swfverify(swfurl)
self.cache.set(cachekey, (swfhash, swfsize))
return swfurl, swfhash, swfsize
def _get_rtmp_streams(self):
chansub = self._authenticate()
url = self.StreamInfoURL.format(self.channelname)
params = dict(b_id="true", group="", private_code="null",
p=int(random.random() * 999999),
channel_subscription=chansub, type="any")
self.logger.debug("Fetching stream info")
res = urlget(url, params=params)
json = res_json(res, "stream info JSON")
if not isinstance(json, list):
raise PluginError("Invalid JSON response")
if len(json) == 0:
raise NoStreamsError(self.url)
streams = {}
swfurl, swfhash, swfsize = self._verify_swf()
for info in json:
if not ("connect" in info and "play" in info
and "type" in info):
continue
stream = RTMPStream(self.session, {
"rtmp": ("{0}/{1}").format(info["connect"], info["play"]),
"swfUrl": swfurl,
"swfhash": swfhash,
"swfsize": swfsize,
"live": True
})
if "display" in info:
sname = info["display"]
else:
sname = info["type"]
if "token" in info:
stream.params["jtv"] = info["token"]
else:
self.logger.warning("No token found for stream {0}, this stream may fail to play", sname)
streams[sname.lower()] = stream
return streams
def _get_hls_streams(self):
url = self.HLSStreamTokenURL.format(self.channelname)
try:
res = urlget(url, params=dict(type="iphone", connection="wifi",
allow_cdn="true"), exception=IOError)
except IOError:
self.logger.debug("HLS streams not available")
return {}
json = res_json(res, "stream token JSON")
if not isinstance(json, list):
raise PluginError("Invalid JSON response")
if len(json) == 0:
raise PluginError("No stream token in JSON")
token = verifyjson(json[0], "token")
hashed = hmac.new(self.HLSStreamTokenKey, bytes(token, "utf8"), sha1)
fulltoken = hashed.hexdigest() + ":" + token
url = self.HLSPlaylistURL.format(self.channelname)
try:
params = dict(token=fulltoken, hd="true", allow_cdn="true")
playlist = HLSStream.parse_variant_playlist(self.session, url,
nameprefix="mobile_",
params=params)
except IOError as err:
if "404" not in str(err):
raise PluginError(err)
else:
self.logger.debug("Requesting mobile transcode")
payload = dict(channel=self.channelname, type="iphone")
urlopen(self.HLSTranscodeRequest, data=payload)
return {}
return playlist
def _get_streams(self):
self.channelname = self._get_channel_name(self.url)
if not self.channelname:
raise NoStreamsError(self.url)
streams = {}
if RTMPStream.is_usable(self.session):
try:
rtmpstreams = self._get_rtmp_streams()
for name, stream in rtmpstreams.items():
if "iphone" in name:
name = name.replace("iphone", "mobile_")
streams[name] = stream
except PluginError as err:
self.logger.error("Error when fetching RTMP stream info: {0}", str(err))
else:
self.logger.warning("rtmpdump is not usable, only HLS streams will be available")
try:
hlsstreams = self._get_hls_streams()
for name, stream in hlsstreams.items():
if "iphone" in name:
name = name.replace("iphone", "")
if name in streams:
streams[name] = [streams[name], stream]
else:
streams[name] = stream
except PluginError as err:
self.logger.error("Error when fetching HLS stream info: {0}", str(err))
return streams
class JustinTV(Plugin):
options = Options({
"cookie": None
})
APIBaseURL = "http://usher.justin.tv"
StreamInfoURL = APIBaseURL + "/find/{0}.xml"
MetadataURL = "http://www.justin.tv/meta/{0}.xml?on_site=true"
SWFURL = "http://www.justin.tv/widgets/live_embed_player.swf"
HLSStreamTokenKey = b"Wd75Yj9sS26Lmhve"
HLSStreamTokenURL = APIBaseURL + "/stream/iphone_token/{0}.json"
HLSSPlaylistURL = APIBaseURL + "/stream/multi_playlist/{0}.m3u8"
@classmethod
def can_handle_url(self, url):
return ("justin.tv" in url) or ("twitch.tv" in url)
def _get_channel_name(self, url):
return url.rstrip("/").rpartition("/")[2].lower()
def _get_metadata(self):
url = self.MetadataURL.format(self.channelname)
headers = {}
cookie = self.options.get("cookie")
if cookie:
headers["Cookie"] = cookie
res = urlget(url, headers=headers)
try:
dom = xml.dom.minidom.parseString(res.text)
except Exception as err:
raise PluginError(("Unable to parse config XML: {0})").format(err))
meta = dom.getElementsByTagName("meta")[0]
metadata = {}
metadata["title"] = self._get_node_if_exists(dom, "title")
metadata["access_guid"] = self._get_node_if_exists(dom, "access_guid")
metadata["login"] = self._get_node_if_exists(dom, "login")
return metadata
def _get_node_if_exists(self, dom, name):
elements = dom.getElementsByTagName(name)
if elements and len(elements) > 0:
return self._get_node_text(elements[0])
def _get_node_text(self, element):
res = []
for node in element.childNodes:
if node.nodeType == node.TEXT_NODE:
res.append(node.data)
if len(res) == 0:
return None
else:
return "".join(res)
def _authenticate(self):
chansub = None
if self.options.get("cookie") is not None:
self.logger.info("Attempting to authenticate using cookies")
metadata = self._get_metadata()
chansub = metadata["access_guid"]
if "login" in metadata and metadata["login"] is not None:
self.logger.info("Successfully logged in as {0}", metadata["login"])
return chansub
def _get_rtmp_streams(self):
def clean_tag(tag):
if tag[0] == "_":
return tag[1:]
else:
return tag
chansub = self._authenticate()
url = self.StreamInfoURL.format(self.channelname)
params = dict(b_id="true", group="", private_code="null",
p=int(random.random() * 999999),
channel_subscription=chansub, type="any")
self.logger.debug("Fetching stream info")
res = urlget(url, params=params)
data = res.text
# fix invalid xml
data = re.sub("<(\d+)", "<_\g<1>", data)
data = re.sub("</(\d+)", "</_\g<1>", data)
streams = {}
try:
dom = xml.dom.minidom.parseString(data)
except Exception as err:
raise PluginError(("Unable to parse config XML: {0})").format(err))
nodes = dom.getElementsByTagName("nodes")[0]
if len(nodes.childNodes) == 0:
return streams
swfurl = urlresolve(self.SWFURL)
for node in nodes.childNodes:
info = {}
for child in node.childNodes:
info[child.tagName] = self._get_node_text(child)
if not ("connect" in info and "play" in info):
continue
stream = RTMPStream(self.session, {
"rtmp": ("{0}/{1}").format(info["connect"], info["play"]),
"swfVfy": swfurl,
"live": True
})
sname = clean_tag(node.tagName)
if "token" in info:
stream.params["jtv"] = info["token"]
else:
self.logger.warning("No token found for stream {0}, this stream may fail to play", sname)
streams[sname] = stream
return streams
def _get_hls_streams(self):
url = self.HLSStreamTokenURL.format(self.channelname)
try:
res = urlget(url, params=dict(type="any", connection="wifi"),
exception=IOError)
except IOError:
return {}
if not isinstance(res.json, list):
raise PluginError("Stream info response is not JSON")
if len(res.json) == 0:
raise PluginError("No stream token in JSON")
streams = {}
token = verifyjson(res.json[0], "token")
hashed = hmac.new(self.HLSStreamTokenKey, bytes(token, "utf8"), sha1)
fulltoken = hashed.hexdigest() + ":" + token
url = self.HLSSPlaylistURL.format(self.channelname)
try:
params = dict(token=fulltoken, hd="true")
playlist = HLSStream.parse_variant_playlist(self.session, url,
params=params)
except IOError as err:
raise PluginError(err)
return playlist
def _get_streams(self):
self.channelname = self._get_channel_name(self.url)
if not self.channelname:
raise NoStreamsError(self.url)
streams = {}
if RTMPStream.is_usable(self.session):
try:
rtmpstreams = self._get_rtmp_streams()
streams.update(rtmpstreams)
except PluginError as err:
self.logger.error("Error when fetching RTMP stream info: {0}", str(err))
else:
self.logger.warning("rtmpdump is not usable, only HLS streams will be available")
try:
hlsstreams = self._get_hls_streams()
if len(streams) > 0:
hlssuffix = "_hls"
else:
hlssuffix = ""
for name, stream in hlsstreams.items():
streams[name + hlssuffix] = stream
except PluginError as err:
self.logger.error("Error when fetching HLS stream info: {0}", str(err))
return streams
class GomTV(Plugin):
BaseURL = "http://www.gomtv.net"
LiveURL = BaseURL + "/main/goLive.gom"
LoginURL = "https://ssl.gomtv.net/userinfo/loginProcess.gom"
LoginCheckURL = BaseURL + "/forum/list.gom?m=my"
LoginHeaders = {"Referer": BaseURL}
StreamHeaders = {"User-Agent": "KPeerClient"}
options = Options({
"cookie": None,
"username": None,
"password": None,
})
@classmethod
def can_handle_url(self, url):
return "gomtv.net" in url
def __init__(self, url):
parsed = urlparse(url)
# Attempt to resolve current live URL if main page is passed
if len(parsed.path) <= 1:
url = self.LiveURL
Plugin.__init__(self, url)
def _get_streams(self):
self.rsession = requests.session(prefetch=True)
options = self.options
if options.get("cookie"):
self._authenticate(cookies=options.get("cookie"))
else:
self._authenticate(options.get("username"),
options.get("password"))
streams = {}
qualities = ["HQ", "SQ", "HQTest", "SQTest"]
res = self._get_live_page(self.url)
urls = self._find_stream_urls(res.text)
for quality in qualities:
for url in urls:
# Grab the response of the URL listed on the Live page for a stream
url = url.format(quality=quality)
res = urlget(url, session=self.rsession)
# The response for the GOX XML if an incorrect stream quality is chosen is 1002.
if res.text != "1002" and len(res.text) > 0:
streamurl = self._parse_gox_file(res.text)
streams[quality.lower()] = HTTPStream(
self.session, streamurl, headers=self.StreamHeaders)
return streams
def _authenticate(self, username=None, password=None, cookies=None):
if (username is None or password is None) and cookies is None:
raise PluginError(
"GOMTV.net requires a username and password or a cookie")
if cookies is not None:
for cookie in cookies.split(";"):
try:
name, value = cookie.split("=")
except ValueError:
continue
self.rsession.cookies[name.strip()] = value.strip()
self.logger.info("Attempting to authenticate with cookies")
else:
form = dict(cmd="login",
rememberme="1",
mb_username=username,
mb_password=password)
self.logger.info(
"Attempting to authenticate with username and password")
urlopen(self.LoginURL,
data=form,
headers=self.LoginHeaders,
session=self.rsession)
res = urlget(self.LoginCheckURL, session=self.rsession)
if "Please need login" in res.text:
raise PluginError("Authentication failed")
if "SES_USERNICK" in self.rsession.cookies:
username = self.rsession.cookies["SES_USERNICK"]
self.logger.info(
("Successfully logged in as {0}").format(username))
def _get_event_url(self, prefix, data):
match = re.search(' \"(.*)\";', data)
if not match:
raise PluginError("Event live page URL not found")
return urljoin(prefix, match.group(1))
def _get_live_page(self, url):
res = urlget(url, session=self.rsession)
# If a special event occurs, we know that the live page response
# will just be some JavaScript that redirects the browser to the
# real live page. We assume that the entireity of this JavaScript
# is less than 200 characters long, and that real live pages are
# more than that.
if len(res.text) < 200:
# Grabbing the real live page URL
url = self._parse_event_url(url, res.text)
res = urlget(url, session=self.rsession)
return res
def _find_stream_urls(self, data):
url = None
# Parsing through the live page for a link to the gox XML file.
# Quality is simply passed as a URL parameter e.g. HQ, SQ, SQTest
try:
patternhtml = "[^/]+var.+(http://www.gomtv.net/gox[^;]+;)"
url = re.search(patternhtml, data).group(1)
url = re.sub('\" \+ playType \+ \"', "{quality}", url)
except AttributeError:
raise PluginError(
"Unable to find the majority of the GOMTV.net XML URL on the Live page"
)
# Finding the title of the stream, probably not necessary but
# done for completeness
try:
patterntitle = "this\.title[^;]+;"
title = re.search(patterntitle, data).group(0)
title = re.search('\"(.*)\"', title).group(0)
title = re.sub('"', "", title)
url = re.sub('"\+ tmpThis.title[^;]+;', title, url)
except AttributeError:
raise PluginError(
"Unable to find the stream title on the Live page")
# Check for multiple streams going at the same time, and extract the conid and the title
# Those streams have the class "live_now"
patternlive = '<a\shref=\"/live/index.gom\?conid=(?P<conid>\d+)\"\sclass=\"live_now\"\stitle=\"(?P<title>[^\"]+)'
streams = re.findall(patternlive, data)
if len(streams) > 1:
urls = []
for stream in streams:
# Modify the urlFromHTML according to the user
singleurl = re.sub("conid=\d+", "conid=" + stream[0], url)
singletitlehtml = "+".join(stream[0].split(" "))
singleurl = re.sub("title=[\w|.|+]*",
"title=" + singletitlehtml, singleurl)
urls.append(singleurl)
return urls
else:
if url is None:
return []
else:
return [url]
def _parse_gox_file(self, data):
# Grabbing the gomcmd URL
try:
patternstream = '<REF href="([^"]*)"\s*/>'
match = re.search(patternstream, data).group(1)
except AttributeError:
raise PluginError(
"Unable to find the gomcmd URL in the GOX XML file")
match = match.replace("&", "&")
match = unquote(match)
# SQ and SQTest streams can be gomp2p links, with actual stream address passed as a parameter.
if match.startswith("gomp2p://"):
match, n = re.subn("^.*LiveAddr=", "", match)
# Cosmetics, getting rid of the HTML entity, we don't
# need either of the " character or "
match = match.replace(""", "")
return match
def setUp(self):
self.options = Options({"a_default": "default"})
class Livestation(Plugin):
SWFURL = "http://beta.cdn.livestation.com/player/5.10/livestation-player.swf"
APIURL = "http://tokens.api.livestation.com/channels/{0}/tokens.json?{1}"
LOGINPAGEURL = "http://www.livestation.com/en/users/new"
LOGINPOSTURL = "http://www.livestation.com/en/sessions.json"
options = Options({"email": "", "password": ""})
@classmethod
def can_handle_url(self, url):
return "livestation.com" in url
def _get_rtmp_streams(self, text):
match = re.search("streamer=(rtmp://.+?)&", text)
if not match:
raise PluginError(
("No RTMP streamer found on URL {0}").format(self.url))
rtmp = match.group(1)
match = re.search("<meta content=\"(http://.+?\.swf)\?", text)
if not match:
self.logger.warning(
"Failed to get player SWF URL location on URL {0}", self.url)
else:
self.SWFURL = match.group(1)
self.logger.debug("Found player SWF URL location {0}", self.SWFURL)
match = re.search("<meta content=\"(.+)\" name=\"item-id\" />", text)
if not match:
raise PluginError(
("Missing channel item-id on URL {0}").format(self.url))
res = http.get(self.APIURL.format(match.group(1), time()),
params=dict(output="json"))
json = http.json(res)
if not isinstance(json, list):
raise PluginError("Invalid JSON response")
rtmplist = {}
for jdata in json:
if "stream_name" not in jdata or "type" not in jdata:
continue
if "rtmp" not in jdata["type"]:
continue
playpath = jdata["stream_name"]
if "token" in jdata and jdata["token"]:
playpath += jdata["token"]
if len(json) == 1:
stream_name = "live"
else:
stream_name = jdata["stream_name"]
rtmplist[stream_name] = RTMPStream(
self.session, {
"rtmp": rtmp,
"pageUrl": self.url,
"swfVfy": self.SWFURL,
"playpath": playpath,
"live": True
})
return rtmplist
def _get_hls_streams(self, text):
match = re.search("\"(http://.+\.m3u8)\"", text)
if not match:
raise PluginError(
("No HLS playlist found on URL {0}").format(self.url))
playlisturl = match.group(1)
self.logger.debug("Playlist URL is {0}", playlisturl)
playlist = {}
try:
playlist = HLSStream.parse_variant_playlist(
self.session, playlisturl)
except IOError as err:
raise PluginError(err)
return playlist
def _get_streams(self):
# If email option given, try to login
if self.options.get("email"):
res = http.get(self.LOGINPAGEURL)
match = re.search('<meta content="([^"]+)" name="csrf-token"',
res.text)
if not match:
raise PluginError("Missing CSRF Token: " + self.LOGINPAGEURL)
csrf_token = match.group(1)
email = self.options.get("email")
password = self.options.get("password")
res = http.post(
self.LOGINPOSTURL,
data={
'authenticity_token': csrf_token,
'channel_id': '',
'commit': 'Login',
'plan_id': '',
'session[email]': email,
'session[password]': password,
'utf8': "\xE2\x9C\x93", # Check Mark Character
})
self.logger.debug("Login account info: {0}", res.text)
result = http.json(res)
if result.get('email', 'no-mail') != email:
raise PluginError("Invalid account")
res = http.get(self.url)
streams = {}
if RTMPStream.is_usable(self.session):
try:
rtmpstreams = self._get_rtmp_streams(res.text)
streams.update(rtmpstreams)
except PluginError as err:
self.logger.error("Error when fetching RTMP stream info: {0}",
str(err))
else:
self.logger.warning(
"rtmpdump is not usable, only HLS streams will be available")
try:
hlsstreams = self._get_hls_streams(res.text)
streams.update(hlsstreams)
except PluginError as err:
self.logger.error("Error when fetching HLS stream info: {0}",
str(err))
return streams
class UStreamTV(Plugin):
options = Options({"password": ""})
@classmethod
def can_handle_url(cls, url):
return "ustream.tv" in url
@classmethod
def stream_weight(cls, stream):
match = re.match("mobile_(\w+)", stream)
if match:
weight, group = Plugin.stream_weight(match.group(1))
weight -= 1
group = "mobile_ustream"
elif stream == "recorded":
weight, group = 720, "ustream"
else:
weight, group = Plugin.stream_weight(stream)
return weight, group
def _get_channel_id(self, url):
match = re.search("ustream.tv/embed/(\d+)", url)
if match:
return int(match.group(1))
match = re.search("\"cid\":(\d+)", urlget(url).text)
if match:
return int(match.group(1))
def _get_hls_streams(self, wait_for_transcode=False):
# HLS streams are created on demand, so we may have to wait
# for a transcode to be started.
attempts = wait_for_transcode and 10 or 1
playlist_url = HLS_PLAYLIST_URL.format(self.channel_id)
streams = {}
while attempts and not streams:
try:
streams = HLSStream.parse_variant_playlist(
self.session, playlist_url, nameprefix="mobile_")
except IOError:
# Channel is probably offline
break
attempts -= 1
sleep(3)
return streams
def _create_rtmp_stream(self, cdn, stream_name):
parsed = urlparse(cdn)
options = dict(rtmp=cdn,
app=parsed.path[1:],
playpath=stream_name,
pageUrl=self.url,
swfUrl=SWF_URL,
live=True)
return RTMPStream(self.session, options)
def _get_module_info(self, app, media_id, password=""):
self.logger.debug("Waiting for moduleInfo invoke")
conn = create_ums_connection(app, media_id, self.url, password)
attempts = 3
while conn.connected and attempts:
try:
result = conn.process_packets(invoked_method="moduleInfo",
timeout=30)
except (IOError, librtmp.RTMPError) as err:
raise PluginError("Failed to get stream info: {0}".format(err))
result = validate_module_info(result)
if result:
break
else:
attempts -= 1
conn.close()
return result
def _get_streams_from_rtmp(self):
password = self.options.get("password")
module_info = self._get_module_info("channel", self.channel_id,
password)
if not module_info:
raise NoStreamsError(self.url)
providers = module_info.get("stream")
if providers == "offline":
raise NoStreamsError(self.url)
elif not isinstance(providers, list):
raise PluginError("Invalid stream info: {0}".format(providers))
streams = {}
for provider in filter(valid_provider, providers):
provider_url = provider.get("url")
provider_name = provider.get("name")
provider_streams = provider.get("streams")
for stream_index, stream_info in enumerate(provider_streams):
stream = None
stream_height = int(stream_info.get("height", 0))
stream_name = (stream_info.get("description")
or (stream_height > 0
and "{0}p".format(stream_height)) or "live")
if stream_name in streams:
provider_name_clean = provider_name.replace("uhs_", "")
stream_name += "_alt_{0}".format(provider_name_clean)
if provider_name.startswith("uhs_"):
stream = UHSStream(self.session, self.channel_id, self.url,
provider_name, stream_index, password)
elif (provider_url.startswith("rtmp")
and RTMPStream.is_usable(self.session)):
playpath = stream_info.get("streamName")
stream = self._create_rtmp_stream(provider_url, playpath)
if stream:
streams[stream_name] = stream
return streams
def _get_streams_from_amf(self):
if not RTMPStream.is_usable(self.session):
raise NoStreamsError(self.url)
res = urlget(AMF_URL.format(self.channel_id))
try:
packet = AMFPacket.deserialize(BytesIO(res.content))
except (IOError, AMFError) as err:
raise PluginError("Failed to parse AMF packet: {0}".format(err))
for message in packet.messages:
if message.target_uri == "/1/onResult":
result = message.value
break
else:
raise PluginError("No result found in AMF packet")
streams = {}
stream_name = result.get("streamName")
if stream_name:
cdn = result.get("cdnUrl") or result.get("fmsUrl")
if cdn:
stream = self._create_rtmp_stream(cdn, stream_name)
if "videoCodec" in result and result["videoCodec"][
"height"] > 0:
stream_name = "{0}p".format(
int(result["videoCodec"]["height"]))
else:
stream_name = "live"
streams[stream_name] = stream
else:
self.logger.warning("Missing cdnUrl and fmsUrl from result")
stream_versions = result.get("streamVersions")
if stream_versions:
for version, info in stream_versions.items():
stream_version_cdn = info.get("streamVersionCdn", {})
for name, cdn in filter(valid_cdn, stream_version_cdn.items()):
stream = self._create_rtmp_stream(cdn["cdnStreamUrl"],
cdn["cdnStreamName"])
stream_name = "live_alt_{0}".format(name)
streams[stream_name] = stream
return streams
def _get_live_streams(self):
self.channel_id = self._get_channel_id(self.url)
if not self.channel_id:
raise NoStreamsError(self.url)
streams = defaultdict(list)
if not RTMPStream.is_usable(self.session):
self.logger.warning("rtmpdump is not usable. "
"Not all streams may be available.")
if HAS_LIBRTMP:
desktop_streams = self._get_streams_from_rtmp
else:
self.logger.warning("python-librtmp is not installed. "
"Not all streams may be available.")
desktop_streams = self._get_streams_from_amf
try:
for name, stream in desktop_streams().items():
streams[name].append(stream)
except PluginError as err:
self.logger.error("Unable to fetch desktop streams: {0}", err)
except NoStreamsError:
pass
try:
mobile_streams = self._get_hls_streams(
wait_for_transcode=not streams)
for name, stream in mobile_streams.items():
streams[name].append(stream)
except PluginError as err:
self.logger.error("Unable to fetch mobile streams: {0}", err)
except NoStreamsError:
pass
return streams
def _get_recorded_streams(self, video_id):
streams = {}
if HAS_LIBRTMP:
module_info = self._get_module_info("recorded", video_id)
if not module_info:
raise NoStreamsError(self.url)
providers = module_info.get("stream")
if not isinstance(providers, list):
raise PluginError("Invalid stream info: {0}".format(providers))
for provider in providers:
for stream_info in provider.get("streams"):
bitrate = int(stream_info.get("bitrate", 0))
stream_name = (bitrate > 0 and "{0}k".format(bitrate)
or "recorded")
if stream_name in streams:
stream_name += "_alt"
stream = HTTPStream(self.session,
stream_info.get("streamName"))
streams[stream_name] = stream
else:
self.logger.warning("The proper API could not be used without "
"python-librtmp installed. Stream URL may be "
"incorrect.")
url = RECORDED_URL.format(video_id)
random_hash = "{0:02x}{1:02x}".format(randint(0, 255),
randint(0, 255))
params = dict(hash=random_hash)
stream = HTTPStream(self.session, url, params=params)
streams["recorded"] = stream
return streams
def _get_streams(self):
recorded = re.match(RECORDED_URL_PATTERN, self.url)
if recorded:
return self._get_recorded_streams(recorded.group("video_id"))
else:
return self._get_live_streams()