def decrypt_data(self, cipher_data, encrypted_data):
cipher = AES.new(
bytes(cipher_data['key'], 'utf-8'),
self.encryption_algorithm[cipher_data['algorithm']],
bytes(cipher_data['iv'], 'utf-8'),
)
return unpad(cipher.decrypt(binascii.unhexlify(encrypted_data)), 16, 'pkcs7')
def _parse_streams(self, res):
for match in self._src_re.finditer(res.text):
stream_url = match.group("url")
if "\\/" in stream_url:
# if the URL is json encoded, decode it
stream_url = parse_json("\"{}\"".format(stream_url))
if ".mpd" in stream_url:
for s in DASHStream.parse_manifest(self.session,
stream_url).items():
yield s
elif ".mp4" in stream_url:
yield match.group(1), HTTPStream(self.session, stream_url)
else:
log.debug("Non-dash/mp4 stream: {0}".format(stream_url))
match = self._dash_manifest_re.search(res.text)
if match:
# facebook replaces "<" characters with the substring "\\x3C"
manifest = match.group("manifest").replace("\\/", "/")
if is_py3:
manifest = bytes(unquote_plus(manifest),
"utf-8").decode("unicode_escape")
else:
manifest = unquote_plus(manifest).decode("string_escape")
# Ignore unsupported manifests until DASH SegmentBase support is implemented
if "SegmentBase" in manifest:
log.error("Skipped DASH manifest with SegmentBase streams")
else:
for s in DASHStream.parse_manifest(self.session,
manifest).items():
yield s
def _get_live_streams_data(self, video_id):
client_type = 'huomaomobileh5'
time_now = str(int(time.time()))
token_data = "{0}{1}{2}{3}".format(
video_id,
client_type,
time_now,
self.magic_val,
)
token = hashlib.md5(bytes(token_data, 'utf-8')).hexdigest()
log.debug("Token={0}".format(token))
post_data = {
'cdns': 1,
'streamtype': 'live',
'VideoIDS': video_id,
'from': client_type,
'time': time_now,
'token': token,
}
video_data = self.session.http.post(self.live_data_url, data=post_data)
return self.session.http.json(
video_data,
schema=self._live_data_schema,
)
def _generate_session_token(self, data64):
swfdata = base64.decodestring(bytes(data64, "ascii"))
md5 = hashlib.md5()
md5.update(swfdata)
hash = md5.hexdigest()
if hash in self.TokenGenerators:
generator = self.TokenGenerators[hash](self)
return generator.generate()
else:
raise StreamError(
("No token generator available for hash '{0}'").format(hash))
def _parse_streams(self, res):
_found_stream_url = False
for meta in itertags(res.text, "meta"):
if meta.attributes.get("property") == "og:video:url":
stream_url = html_unescape(meta.attributes.get("content"))
if ".mpd" in stream_url:
for s in DASHStream.parse_manifest(self.session,
stream_url).items():
yield s
_found_stream_url = True
elif ".mp4" in stream_url:
yield "vod", HTTPStream(self.session, stream_url)
_found_stream_url = True
break
else:
log.debug("No meta og:video:url")
if _found_stream_url:
return
for match in self._src_re.finditer(res.text):
stream_url = match.group("url")
if "\\/" in stream_url:
# if the URL is json encoded, decode it
stream_url = parse_json("\"{}\"".format(stream_url))
if ".mpd" in stream_url:
for s in DASHStream.parse_manifest(self.session,
stream_url).items():
yield s
elif ".mp4" in stream_url:
yield match.group(1), HTTPStream(self.session, stream_url)
else:
log.debug("Non-dash/mp4 stream: {0}".format(stream_url))
match = self._dash_manifest_re.search(res.text)
if match:
# facebook replaces "<" characters with the substring "\\x3C"
manifest = match.group("manifest").replace("\\/", "/")
if is_py3:
manifest = bytes(unquote_plus(manifest),
"utf-8").decode("unicode_escape")
else:
manifest = unquote_plus(manifest).decode("string_escape")
# Ignore unsupported manifests until DASH SegmentBase support is implemented
if "SegmentBase" in manifest:
log.error("Skipped DASH manifest with SegmentBase streams")
else:
for s in DASHStream.parse_manifest(self.session,
manifest).items():
yield s
def _parse_streams(self, res):
stream_url = validate.Schema(
validate.parse_html(),
validate.xml_xpath_string(
".//head/meta[@property='og:video:url'][@content][1]/@content")
).validate(res.text)
if not stream_url:
log.debug("No meta og:video:url")
else:
if ".mpd" in stream_url:
for s in DASHStream.parse_manifest(self.session,
stream_url).items():
yield s
return
elif ".mp4" in stream_url:
yield "vod", HTTPStream(self.session, stream_url)
return
for match in self._src_re.finditer(res.text):
stream_url = match.group("url")
if "\\/" in stream_url:
# if the URL is json encoded, decode it
stream_url = parse_json("\"{}\"".format(stream_url))
if ".mpd" in stream_url:
for s in DASHStream.parse_manifest(self.session,
stream_url).items():
yield s
elif ".mp4" in stream_url:
yield match.group(1), HTTPStream(self.session, stream_url)
else:
log.debug("Non-dash/mp4 stream: {0}".format(stream_url))
match = self._dash_manifest_re.search(res.text)
if match:
# facebook replaces "<" characters with the substring "\\x3C"
manifest = match.group("manifest").replace("\\/", "/")
if is_py3:
manifest = bytes(unquote_plus(manifest),
"utf-8").decode("unicode_escape")
else:
manifest = unquote_plus(manifest).decode("string_escape")
# Ignore unsupported manifests until DASH SegmentBase support is implemented
if "SegmentBase" in manifest:
log.error("Skipped DASH manifest with SegmentBase streams")
else:
for s in DASHStream.parse_manifest(self.session,
manifest).items():
yield s
def generate(self):
if not self.stream.swf:
raise StreamError("A SWF URL is required to create session token")
res = self.stream.session.http.get(self.stream.swf,
exception=StreamError)
data = swfdecompress(res.content)
md5 = hashlib.md5()
md5.update(data)
data = bytes(self.stream.sessionid, "ascii") + md5.digest()
sig = hmac.new(b"foo", data, hashlib.sha1)
b64 = base64.encodestring(sig.digest())
token = str(b64, "ascii").replace("\n", "")
return token
def encrypt(self, data):
return base64.b64encode(self.cipher.encrypt(self.pad(bytes(data, "utf-8"))), altchars=b"-_").decode("ascii")
def pad(cls, data):
n = cls.block_size - len(data) % cls.block_size
return data + bytes(chr(cls.block_size - len(data) % cls.block_size), "utf8") * n
def _get_streams(self):
match = url_re.match(self.url)
stream_page_scheme = 'https'
stream_page_domain = match.group(4)
stream_page_path = match.group(5)
country_code = CONST_DEFAULT_COUNTRY_CODE
is_paid_show = False
# create http session and set headers
http_session = http
http_session.headers.update(CONST_HEADERS)
# get swf url and cookies
r = http_session.get(
urlunparse((stream_page_scheme, stream_page_domain,
stream_page_path, '', '', '')))
# redirect to profile page means stream is offline
if '/profile/' in r.url:
raise NoStreamsError(self.url)
if not r.ok:
self.logger.debug("Status code for {}: {}", r.url, r.status_code)
raise NoStreamsError(self.url)
if len(http_session.cookies) == 0:
raise PluginError("Can't get a cookies")
if urlparse(r.url).netloc != stream_page_domain:
# then redirected to regional subdomain
country_code = urlparse(r.url).netloc.split('.')[0].lower()
# time to set variables
baseurl = urlunparse(
(stream_page_scheme, urlparse(r.url).netloc, '', '', '', ''))
amf_gateway_url = urljoin(baseurl, CONST_AMF_GATEWAY_LOCATION)
stream_page_url = urljoin(baseurl, stream_page_path)
match = swf_re.search(r.text)
if match:
swf_url = urljoin(baseurl, match.group())
self.logger.debug("swf url found: {}", swf_url)
else:
# most likely it means that country/region banned
# can try use default swf-url
swf_url = urljoin(baseurl, CONST_DEFAULT_SWF_LOCATION)
self.logger.debug("swf url not found. Will try {}", swf_url)
# create amf query
amf_message = AMFMessage("svDirectAmf.getRoomData", "/1",
[stream_page_path, is_paid_show])
amf_packet = AMFPacket(version=0)
amf_packet.messages.append(amf_message)
# send request and close http-session
r = http_session.post(url=amf_gateway_url,
params={CONST_AMF_GATEWAY_PARAM: country_code},
data=bytes(amf_packet.serialize()))
http_session.close()
if r.status_code != 200:
raise PluginError("unexpected status code for {}: {}", r.url,
r.status_code)
amf_response = AMFPacket.deserialize(BytesIO(r.content))
if len(amf_response.messages
) != 1 or amf_response.messages[0].target_uri != "/1/onResult":
raise PluginError("unexpected response from amf gate")
stream_source_info = amf_msg_schema.validate(
amf_response.messages[0].value)
self.logger.debug("source stream info:\n{}", stream_source_info)
stream_params = {
"live":
True,
"realtime":
True,
"flashVer":
CONST_FLASH_VER,
"swfUrl":
swf_url,
"tcUrl":
stream_source_info['localData']['NC_ConnUrl'],
"rtmp":
stream_source_info['localData']['NC_ConnUrl'],
"pageUrl":
stream_page_url,
"playpath":
"%s?uid=%s" %
(''.join(('stream_', stream_page_path)),
self._get_stream_uid(stream_source_info['userData']['username'])),
"conn": [
"S:{0}".format(stream_source_info['userData']['username']),
"S:{0}".format(
stream_source_info['localData']['NC_AccessKey']), "B:0",
"S:{0}".format(stream_source_info['localData']['dataKey'])
]
}
self.logger.debug("Stream params:\n{}", stream_params)
stream = RTMPStream(self.session, stream_params)
return {'live': stream}
def pad(self, data):
n = self.block_size - len(data) % self.block_size
return data + bytes(chr(self.block_size - len(data) % self.block_size), "utf8") * n
def parse_manifest(cls,
session,
url,
timeout=60,
pvswf=None,
is_akamai=False,
**request_params):
"""Parses a HDS manifest and returns its substreams.
:param url: The URL to the manifest.
:param timeout: How long to wait for data to be returned from
from the stream before raising an error.
:param is_akamai: force adding of the akamai parameters
:param pvswf: URL of player SWF for Akamai HD player verification.
"""
# private argument, should only be used in recursive calls
raise_for_drm = request_params.pop("raise_for_drm", False)
if not request_params:
request_params = {}
request_params["headers"] = request_params.get("headers", {})
request_params["params"] = request_params.get("params", {})
# These params are reserved for internal use
request_params.pop("exception", None)
request_params.pop("stream", None)
request_params.pop("timeout", None)
request_params.pop("url", None)
if "akamaihd" in url or is_akamai:
request_params["params"]["hdcore"] = HDCORE_VERSION
request_params["params"]["g"] = cls.cache_buster_string(12)
res = session.http.get(url, exception=IOError, **request_params)
manifest = session.http.xml(res,
"manifest XML",
ignore_ns=True,
exception=IOError)
if manifest.findtext("drmAdditionalHeader"):
log.debug("Omitting HDS stream protected by DRM: {}", url)
if raise_for_drm:
raise PluginError("{} is protected by DRM".format(url))
log.warning(
"Some or all streams are unavailable as they are protected by DRM"
)
return {}
parsed = urlparse(url)
baseurl = manifest.findtext("baseURL")
baseheight = manifest.findtext("height")
bootstraps = {}
streams = {}
if not baseurl:
baseurl = urljoin(url, os.path.dirname(parsed.path))
if not baseurl.endswith("/"):
baseurl += "/"
for bootstrap in manifest.findall("bootstrapInfo"):
name = bootstrap.attrib.get("id") or "_global"
url = bootstrap.attrib.get("url")
if url:
box = absolute_url(baseurl, url)
else:
data = base64.b64decode(bytes(bootstrap.text, "utf8"))
box = Box.deserialize(BytesIO(data))
bootstraps[name] = box
pvtoken = manifest.findtext("pv-2.0")
if pvtoken:
if not pvswf:
raise IOError("This manifest requires the 'pvswf' parameter "
"to verify the SWF")
params = cls._pv_params(session, pvswf, pvtoken, **request_params)
request_params["params"].update(params)
child_drm = False
for media in manifest.findall("media"):
url = media.attrib.get("url")
bootstrapid = media.attrib.get("bootstrapInfoId", "_global")
href = media.attrib.get("href")
if url and bootstrapid:
bootstrap = bootstraps.get(bootstrapid)
if not bootstrap:
continue
bitrate = media.attrib.get("bitrate")
streamid = media.attrib.get("streamId")
height = media.attrib.get("height")
if height:
quality = height + "p"
elif bitrate:
quality = bitrate + "k"
elif streamid:
quality = streamid
elif baseheight:
quality = baseheight + "p"
else:
quality = "live"
metadata = media.findtext("metadata")
if metadata:
metadata = base64.b64decode(bytes(metadata, "utf8"))
metadata = ScriptData.deserialize(BytesIO(metadata))
else:
metadata = None
stream = HDSStream(session,
baseurl,
url,
bootstrap,
metadata=metadata,
timeout=timeout,
**request_params)
streams[quality] = stream
elif href:
url = absolute_url(baseurl, href)
try:
child_streams = cls.parse_manifest(session,
url,
timeout=timeout,
is_akamai=is_akamai,
raise_for_drm=True,
**request_params)
except PluginError:
child_drm = True
child_streams = {}
for name, stream in child_streams.items():
# Override stream name if bitrate is available in parent
# manifest but not the child one.
bitrate = media.attrib.get("bitrate")
if bitrate and not re.match(r"^(\d+)k$", name):
name = bitrate + "k"
streams[name] = stream
if child_drm:
log.warning(
"Some or all streams are unavailable as they are protected by DRM"
)
return streams
