def test_length(self): assert validate(length(1), [1, 2, 3]) == [1, 2, 3] def invalid_length(): validate(length(2), [1]) self.assertRaises(ValueError, invalid_length)
class RTPPlay(Plugin): _m3u8_re = re.compile( r""" hls:\s*(?:(["'])(?P<string>[^"']+)\1 | decodeURIComponent\((?P<obfuscated>\[.*?])\.join\() """, re.VERBOSE) _schema_hls = validate.Schema( validate.transform(_m3u8_re.search), validate.any( None, validate.all(validate.get("string"), validate.text, validate.any(validate.length(0), validate.url())), validate.all(validate.get("obfuscated"), validate.text, validate.parse_json(), validate.transform(lambda arr: unquote("".join(arr))), validate.url()), validate.all( validate.get("obfuscated_b64"), validate.text, validate.parse_json(), validate.transform(lambda arr: unquote("".join(arr))), validate.transform(lambda b64: b64decode(b64).decode("utf-8")), validate.url()))) def _get_streams(self): self.session.http.headers.update({ "User-Agent": useragents.CHROME, "Referer": self.url }) hls_url = self.session.http.get(self.url, schema=self._schema_hls) if not hls_url: return return HLSStream.parse_variant_playlist(self.session, hls_url)
def test_failure(self, minlength, value): with pytest.raises(validate.ValidationError) as cm: validate.validate(validate.length(minlength + 1), value) assert_validationerror( cm.value, """ ValidationError(length): Minimum length is 4, but value is 3 """)
class DingitTV(Plugin): """ Plugin that supports playing streams from DingIt.tv """ # regex to match the site urls url_re = re.compile( r""" http://www.dingit.tv/( highlight/(?P<highlight_id>\d+)| channel/(?P<broadcaster>\w+)/(?P<channel_id>\d+) )""", re.VERBOSE) # flashvars API url and schema flashvars_url = "http://www.dingit.tv/api/get_player_flashvars" flashvars_schema = validate.Schema( { u"status": 0, u"data": [{ validate.optional("stream"): validate.text, validate.optional("akaurl"): validate.text, validate.optional("pereakaurl"): validate.text, }] }, validate.get("data"), validate.length(1), validate.get(0)) pereakaurl = "http://dingitmedia-vh.akamaihd.net/i/{}/master.m3u8" akaurl = "https://dingmedia1-a.akamaihd.net/processed/delivery/{}70f8b7bc-5ed4-336d-609a-2d2cd86288c6.m3u8" @classmethod def can_handle_url(cls, url): return cls.url_re.match(url) is not None @Plugin.broken() def _get_streams(self): match = self.url_re.match(self.url) res = self.session.http.post( self.flashvars_url, data=dict(broadcaster=match.group("broadcaster") or "Verm", stream_id=match.group("channel_id") or match.group("highlight_id"))) flashvars = self.session.http.json(res, schema=self.flashvars_schema) if flashvars.get("pereakaurl"): url = self.pereakaurl.format( flashvars.get("pereakaurl").strip("/")) return HLSStream.parse_variant_playlist(self.session, url) elif flashvars.get("akaurl"): url = self.akaurl.format(flashvars.get("akaurl").strip("/")) return HLSStream.parse_variant_playlist(self.session, url) elif flashvars.get("stream"): self.logger.error("OctoStreams are not currently supported")
class RTPPlay(Plugin): _url_re = re.compile(r"https?://www\.rtp\.pt/play/") _m3u8_re = re.compile(r""" hls\s*:\s*(?: (["'])(?P<string>[^"']*)\1 | decodeURIComponent\s*\((?P<obfuscated>\[.*?])\.join\( | atob\s*\(\s*decodeURIComponent\s*\((?P<obfuscated_b64>\[.*?])\.join\( ) """, re.VERBOSE) _schema_hls = validate.Schema( validate.transform(lambda text: next(reversed(list(RTPPlay._m3u8_re.finditer(text))), None)), validate.any( None, validate.all( validate.get("string"), str, validate.any( validate.length(0), validate.url() ) ), validate.all( validate.get("obfuscated"), str, validate.transform(lambda arr: unquote("".join(parse_json(arr)))), validate.url() ), validate.all( validate.get("obfuscated_b64"), str, validate.transform(lambda arr: unquote("".join(parse_json(arr)))), validate.transform(lambda b64: b64decode(b64).decode("utf-8")), validate.url() ) ) ) @classmethod def can_handle_url(cls, url): return cls._url_re.match(url) is not None def _get_streams(self): self.session.http.headers.update({"User-Agent": useragents.CHROME, "Referer": self.url}) hls_url = self.session.http.get(self.url, schema=self._schema_hls) if hls_url: return HLSStream.parse_variant_playlist(self.session, hls_url)
_url_re = re.compile( r""" http(s)?://(\w+\.)? (?P<domain>vaughnlive|breakers|instagib|vapers).tv (/embed/video)? /(?P<channel>[^/&?]+) """, re.VERBOSE) _swf_player_re = re.compile( r'swfobject.embedSWF\("(/\d+/swf/[0-9A-Za-z]+\.swf)"') _schema = validate.Schema( validate.any( validate.all(u"<error></error>", validate.transform(lambda x: None)), validate.all( validate.transform(lambda s: s.split(";")), validate.length(3), validate.union({ "server": validate.all(validate.get(0), validate.text), "token": validate.all( validate.get(1), validate.text, validate.startswith(":mvnkey-"), validate.transform(lambda s: s[len(":mvnkey-"):])), "ingest": validate.all(validate.get(2), validate.text) })))) class VaughnLive(Plugin): @classmethod
def invalid_length(): validate(length(2), [1])
class UHSClient(object): """ API Client, reverse engineered by observing the interactions between the web browser and the ustream servers. """ API_URL = "http://r{0}-1-{1}-{2}-{3}.ums.ustream.tv" APP_ID, APP_VERSION = 2, 1 api_schama = validate.Schema([{"args": [object], "cmd": validate.text}]) connect_schama = validate.Schema([{ "args": validate.all([{ "host": validate.text, "connectionId": validate.text }], validate.length(1)), "cmd": "tracking" }], validate.length(1), validate.get(0), validate.get("args"), validate.get(0)) module_info_schema = validate.Schema( [validate.get("stream")], validate.filter(lambda r: r is not None)) def __init__(self, session, media_id, application, **options): self.session = session http.headers.update({"User-Agent": useragents.IPHONE_6}) self.logger = session.logger.new_module("plugin.ustream.apiclient") self.media_id = media_id self.application = application self.referrer = options.pop("referrer", None) self._host = None self.rsid = self.generate_rsid() self.rpin = self.generate_rpin() self._connection_id = None self._app_id = options.pop("app_id", self.APP_ID) self._app_version = options.pop("app_version", self.APP_VERSION) self._cluster = options.pop("cluster", "live") self._password = options.pop("password") def connect(self, **options): result = self.send_command(type="viewer", appId=self._app_id, appVersion=self._app_version, rsid=self.rsid, rpin=self.rpin, referrer=self.referrer, media=str(self.media_id), application=self.application, schema=self.connect_schama, password=self._password) self._host = "http://{0}".format(result["host"]) self._connection_id = result["connectionId"] self.logger.debug("Got new host={0}, and connectionId={1}", self._host, self._connection_id) return True def poll(self, schema=None, retries=5, timeout=5.0): stime = time.time() try: r = self.send_command(connectionId=self._connection_id, schema=schema, retries=retries, timeout=timeout) except PluginError as err: self.logger.debug("poll took {0:.2f}s: {1}", time.time() - stime, err) else: self.logger.debug("poll took {0:.2f}s", time.time() - stime) return r def generate_rsid(self): return "{0:x}:{1:x}".format(randint(0, 1e10), randint(0, 1e10)) def generate_rpin(self): return "_rpin.{0}".format(randint(0, 1e15)) def send_command(self, schema=None, retries=5, timeout=5.0, **args): res = http.get(self.host, params=args, headers={ "Referer": self.referrer, "User-Agent": useragents.IPHONE_6 }, retries=retries, timeout=timeout, retry_max_backoff=0.5) return http.json(res, schema=schema or self.api_schama) @property def host(self): host = self._host or self.API_URL.format(randint( 0, 0xffffff), self.media_id, self.application, "lp-" + self._cluster) return urljoin(host, "/1/ustream")
""", re.VERBOSE) _live_schema = validate.Schema( { "livestream": [{ "media_user_name": validate.text, validate.optional("media_hosted_media"): object, "media_is_live": validate.all(validate.text, validate.transform(int), validate.transform(bool)), "media_id": validate.text }], }, validate.get("livestream"), validate.length(1), validate.get(0)) _player_schema = validate.Schema({ "clip": { "baseUrl": validate.any(None, validate.text), "bitrates": validate.all( validate.filter(lambda b: b.get("url") and b.get("label")), [{ "label": validate.text, "url": validate.text, }], ) }, validate.optional("playlist"): [{ validate.optional("connectionProvider"):
"breakers": "btv", "vapers": "vtv", "vaughnlive": "live", } _url_re = re.compile(""" http(s)?://(\w+\.)? (?P<domain>vaughnlive|breakers|instagib|vapers).tv /(?P<channel>[^/&?]+) """, re.VERBOSE) _swf_player_re = re.compile('swfobject.embedSWF\("(/\d+/swf/[0-9A-Za-z]+\.swf)"') _schema = validate.Schema( validate.transform(lambda s: s.split(";")), validate.length(3), validate.union({ "server": validate.all( validate.get(0), validate.text ), "token": validate.all( validate.get(1), validate.text, validate.startswith(":mvnkey-"), validate.transform(lambda s: s[len(":mvnkey-"):]) ), "ingest": validate.all( validate.get(2), validate.text )
)? (?: /recorded/(?P<video_id>\d+) )? """, re.VERBOSE) _channel_id_re = re.compile("\"channelId\":(\d+)") HLS_PLAYLIST_URL = ("http://iphone-streaming.ustream.tv" "/uhls/{0}/streams/live/iphone/playlist.m3u8") RECORDED_URL = "http://tcdn.ustream.tv/video/{0}" RTMP_URL = "rtmp://r{0}-1-{1}-channel-live.ums.ustream.tv:1935/ustream" SWF_URL = "http://static-cdn1.ustream.tv/swf/live/viewer.rsl:505.swf" UHS_GET_HOST_URL = 'http://tcdn.ustream.tv/{0}/?protocol=http' UHS_DEFAULT_HOST = 'http://uhs-live-default.ustream.tv' _module_info_schema = validate.Schema(list, validate.length(1), validate.get(0), dict) _amf3_array = validate.Schema( validate.any( validate.all( {int: object}, validate.transform(lambda a: list(a.values())), ), list)) _recorded_schema = validate.Schema({ validate.optional("stream"): validate.all(_amf3_array, [{ "name": validate.text, "streams": validate.all( _amf3_array,
_live_schema = validate.Schema( { "livestream": [{ "media_user_name": validate.text, validate.optional("media_hosted_media"): object, "media_is_live": validate.all( validate.text, validate.transform(int), validate.transform(bool) ), "media_id": validate.text }], }, validate.get("livestream"), validate.length(1), validate.get(0) ) _player_schema = validate.Schema( { "clip": { "baseUrl": validate.any(None, validate.text), "bitrates": validate.all( validate.filter(lambda b: b.get("url") and b.get("label")), [{ "label": validate.text, "url": validate.text, }], ) }, validate.optional("playlist"): [{
def test_success(self, minlength, value): assert validate.validate(validate.length(minlength), value)
/recorded/(?P<video_id>\d+) )? """, re.VERBOSE) _channel_id_re = re.compile("\"channelId\":(\d+)") HLS_PLAYLIST_URL = ( "http://iphone-streaming.ustream.tv" "/uhls/{0}/streams/live/iphone/playlist.m3u8" ) RECORDED_URL = "http://tcdn.ustream.tv/video/{0}" RTMP_URL = "rtmp://r{0}-1-{1}-channel-live.ums.ustream.tv:1935/ustream" SWF_URL = "http://static-cdn1.ustream.tv/swf/live/viewer.rsl:505.swf" _module_info_schema = validate.Schema( list, validate.length(1), validate.get(0), dict ) _amf3_array = validate.Schema( validate.any( validate.all( {int: object}, validate.transform(lambda a: list(a.values())), ), list ) ) _recorded_schema = validate.Schema({ validate.optional("stream"): validate.all( _amf3_array,
class RTE(Plugin): VOD_API_URL = 'http://www.rte.ie/rteavgen/getplaylist/?type=web&format=json&id={0}' LIVE_API_URL = 'http://feeds.rasset.ie/livelistings/playlist' _url_re = re.compile( r'http://www\.rte\.ie/player/[a-z0-9]+/(?:show/[a-z-]+-[0-9]+/(?P<video_id>[0-9]+)|live/(?P<channel_id>[0-9]+))' ) _vod_api_schema = validate.Schema({ 'current_date': validate.text, 'shows': validate.Schema( list, validate.length(1), validate.get(0), validate.Schema({ 'valid_start': validate.text, 'valid_end': validate.text, 'media:group': validate.Schema( list, validate.length(1), validate.get(0), validate.Schema( { 'hls_server': validate.url(), 'hls_url': validate.text, 'hds_server': validate.url(), 'hds_url': validate.text, # API returns RTMP streams that don't seem to work, ignore them # 'url': validate.any( # validate.url(scheme="rtmp"), # validate.url(scheme="rtmpe") # ) }, validate.transform(lambda x: [ x['hls_server'] + x['hls_url'], x['hds_server'] + x['hds_url'] ])), ), }), ) }) _live_api_schema = validate.Schema( validate.xml_findall('.//{http://search.yahoo.com/mrss/}content'), [ validate.all(validate.xml_element(attrib={'url': validate.url()}), validate.get('url')) ]) _live_api_iphone_schema = validate.Schema( list, validate.length(1), validate.get(0), validate.Schema({'fullUrl': validate.any(validate.url(), 'none')}, validate.get('fullUrl'))) @classmethod def can_handle_url(cls, url): return RTE._url_re.match(url) def _get_streams(self): match = self._url_re.match(self.url) video_id = match.group('video_id') if video_id is not None: # VOD res = http.get(self.VOD_API_URL.format(video_id)) stream_data = http.json(res, schema=self._vod_api_schema) # Check whether video format is expired current_date = datetime.strptime(stream_data['current_date'], '%Y-%m-%dT%H:%M:%S.%f') valid_start = datetime.strptime( stream_data['shows']['valid_start'], '%Y-%m-%dT%H:%M:%S') valid_end = datetime.strptime(stream_data['shows']['valid_end'], '%Y-%m-%dT%H:%M:%S') if current_date < valid_start or current_date > valid_end: self.logger.error( 'Failed to access stream, may be due to expired content') return streams = stream_data['shows']['media:group'] else: # Live channel_id = match.group('channel_id') # Get live streams for desktop res = http.get(self.LIVE_API_URL, params={'channelid': channel_id}) streams = http.xml(res, schema=self._live_api_schema) # Get HLS streams for Iphone res = http.get(self.LIVE_API_URL, params={ 'channelid': channel_id, 'platform': 'iphone' }) stream = http.json(res, schema=self._live_api_iphone_schema) if stream != 'none': streams.append(stream) for stream in streams: if '.f4m' in stream: for s in HDSStream.parse_manifest(self.session, stream).items(): yield s if '.m3u8' in stream: for s in HLSStream.parse_variant_playlist( self.session, stream).items(): yield s
CONST_HEADERS[ 'User-Agent'] = 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) ' CONST_HEADERS[ 'User-Agent'] += 'Chrome/54.0.2840.99 Safari/537.36 OPR/41.0.2353.69' url_re = re.compile(r"(http(s)?://)?(\w{2}.)?(bongacams.com)/([\w\d_-]+)") swf_re = re.compile(r"/swf/\w+/\w+.swf\?cache=\d+") amf_msg_schema = validate.Schema({ "status": "success", "userData": { "username": validate.text }, "localData": { "NC_ConnUrl": validate.url(scheme="rtmp"), "NC_AccessKey": validate.length(32), "dataKey": validate.length(32), }, "performerData": { "username": validate.text, } }) class bongacams(Plugin): @classmethod def can_handle_url(self, url): return url_re.match(url) def _get_stream_uid(self, username): m = md5(username.encode('utf-8') + str(time.time()).encode('utf-8'))