def load_live_m3u8(url): """ the stream is live stream. so we use sleep to simulate player. but not perfact! """ global __lenth__ m = m3u8.load(url) __lenth__ = now = d = 0 i = 0 m3u8_live_stopper() while True: if stop: print('stopped!!') raise StopIteration if i < len(m.segments): delta = d -( time.time() - now) if (delta) > 0: time.sleep(delta) segurl = m.segments[i].absolute_uri now = time.time() d = m.segments[i].duration i += 1 __lenth__ += 1 yield segurl else: i = 0 delta = d -( time.time() - now) if (delta) > 0: time.sleep(d - (time.time() - now)) m = m3u8.load(url) now = time.time() d = 0
def get_video(link, filename): """Send Video link to downloader.""" if link.startswith('https://udemy-adaptive-streaming'): if os.path.exists(filename): # neccessary when ffmpeg not used print('{0} already exists'.format(filename)) return if use_ffmpeg: check_downloaded(link, filename, 'ffmpeg', do_not_resume=True) else: print('\nThis lecture has m3u8 file.\nPlease install "ffmpeg" and use argument "--use-ffmpeg"\nFor better video') variant_m3u8 = m3u8.load(link) if variant_m3u8.is_variant: uri = variant_m3u8.playlists[0].uri m3u8_obj = m3u8.load(uri) total_item = len(m3u8_obj.files) for num, url in enumerate(m3u8_obj.files, 1): tmp_name = str(num) + '.part' if os.path.exists(tmp_name): # use for safety os.remove(tmp_name) print('\nDownloading {0} of {1} parts:'.format(num, total_item)) check_downloaded(url, tmp_name, downloader, do_not_resume=True) tmp_list = [str(num) + '.part' for num, _ in enumerate(m3u8_obj.files, 1)] merge_file(filename, tmp_list) else: print("Failed to process m3u8") else: check_downloaded(link, filename, downloader)
def __init__(self, url): self.url = url try: playlist = m3u8.load(url) while playlist.is_variant: # multiple bitrates videos self.url = playlist.base_uri + playlist.playlists[0].uri playlist = m3u8.load(self.url) except: raise NameError("OpenError") else: self.media_seq = playlist.media_sequence self.current_seq = playlist.media_sequence for segment in playlist.segments: print self.current_seq, segment.uri self.current_seq += 1
def test_raise_timeout_exception_if_timeout_happens_when_loading_from_uri(): try: obj = m3u8.load(playlists.TIMEOUT_SIMPLE_PLAYLIST_URI, timeout=1) except: assert True else: assert False
def test_load_should_create_object_from_file_with_relative_segments(): base_uri = os.path.dirname(playlists.RELATIVE_PLAYLIST_FILENAME) obj = m3u8.load(playlists.RELATIVE_PLAYLIST_FILENAME) expected_key_abspath = '%s/key.bin' % os.path.dirname(base_uri) expected_key_path = '../key.bin' expected_ts1_abspath = '%s/entire1.ts' % base_uri expected_ts1_path = '/entire1.ts' expected_ts2_abspath = '%s/entire2.ts' % os.path.dirname(base_uri) expected_ts2_path = '../entire2.ts' expected_ts3_abspath = '%s/entire3.ts' % os.path.dirname(os.path.dirname(base_uri)) expected_ts3_path = '../../entire3.ts' expected_ts4_abspath = '%s/entire4.ts' % base_uri expected_ts4_path = 'entire4.ts' assert isinstance(obj, m3u8.M3U8) assert expected_key_path == obj.key.uri assert expected_key_abspath == obj.key.absolute_uri assert expected_ts1_path == obj.segments[0].uri assert expected_ts1_abspath == obj.segments[0].absolute_uri assert expected_ts2_path == obj.segments[1].uri assert expected_ts2_abspath == obj.segments[1].absolute_uri assert expected_ts3_path == obj.segments[2].uri assert expected_ts3_abspath == obj.segments[2].absolute_uri assert expected_ts4_path == obj.segments[3].uri assert expected_ts4_abspath == obj.segments[3].absolute_uri
def test_load_should_create_object_from_uri_with_relative_segments(): obj = m3u8.load(playlists.RELATIVE_PLAYLIST_URI) urlparsed = url_parser.urlparse(playlists.RELATIVE_PLAYLIST_URI) base_uri = os.path.normpath(urlparsed.path + '/..') prefix = urlparsed.scheme + '://' + urlparsed.netloc expected_key_abspath = '%s%s/key.bin' % (prefix, os.path.normpath(base_uri + '/..')) expected_key_path = '../key.bin' expected_ts1_abspath = '%s/entire1.ts' % (prefix) expected_ts1_path = '/entire1.ts' expected_ts2_abspath = '%s%sentire2.ts' % (prefix, os.path.normpath(base_uri + '/..') + '/') expected_ts2_path = '../entire2.ts' expected_ts3_abspath = '%s%sentire3.ts' % (prefix, os.path.normpath(base_uri + '/../..')) expected_ts3_path = '../../entire3.ts' expected_ts4_abspath = '%s%sentire4.ts' % (prefix, base_uri + '/') expected_ts4_path = 'entire4.ts' assert isinstance(obj, m3u8.M3U8) assert expected_key_path == obj.key.uri assert expected_key_abspath == obj.key.absolute_uri assert expected_ts1_path == obj.segments[0].uri assert expected_ts1_abspath == obj.segments[0].absolute_uri assert expected_ts2_path == obj.segments[1].uri assert expected_ts2_abspath == obj.segments[1].absolute_uri assert expected_ts3_path == obj.segments[2].uri assert expected_ts3_abspath == obj.segments[2].absolute_uri assert expected_ts4_path == obj.segments[3].uri assert expected_ts4_abspath == obj.segments[3].absolute_uri
def update_for_iframes(url): """ Returns an updated master playlist and new I-frame playlists """ try: master_playlist = m3u8.load(url) except IOError: raise PlaylistLoadError('Invalid url') if not master_playlist or not master_playlist.is_variant: raise BadPlaylistError('Not a variant playlist') master_playlist.iframe_playlists[:] = [] uri = url.split('/')[-1] result = {'master_uri': uri, 'master_content': None, 'iframe_playlists': []} for playlist in master_playlist.playlists: iframe_playlist, data = create_iframe_playlist(playlist) if iframe_playlist is None or data is None: continue master_playlist.add_iframe_playlist(iframe_playlist) result['iframe_playlists'].append(data) result['master_content'] = master_playlist.dumps() return result
def run(self): segments = m3u8.load(self.playlist_uri).segments self.segment_duration = segments[0].duration self.playlist = [str(seg).split("\n")[1] for seg in segments] self.running = True self.startup() self.start_polling_and_fetching_chunks()
def adjustSegmentTimes(self, force=False): """ Read through the ts files in this segment's directory and calculate end time. This is used when restarting video because it died. """ if force or not self.endTime: dir = os.path.join(settings.RECORDED_VIDEO_DIR_BASE, util.getSegmentPath(self.episode.shortName, self.source.name, self.segNumber)) videoChunks = glob("%s/*.ts" % dir) videoChunks = sorted(videoChunks, key = lambda chunk: int(re.sub(".+prog_index-(\d+).ts", "\\1", chunk))) if len(videoChunks) > 0: (mode, ino, dev, nlink, uid, gid, size, atime, mtime, ctime) = os.stat(videoChunks[0]) try: index = m3u8.load('%s/%s' % (dir, self.indexFileName)) m3u8segment = index.segments[0] duration = m3u8segment.duration except: print "NO INDEX.M3U8 FILE for segment %s %d" % (self.episode.shortName, self.segNumber) recordingUtil.stopRecordingAndCleanSegments(self.source, videoChunks) return (None, None) startTime = mtime - duration startDT = datetime.fromtimestamp(startTime, pytz.utc) (mode, ino, dev, nlink, uid, gid, size, atime, mtime, ctime) = os.stat(videoChunks[-1]) endTime = mtime endDT = datetime.fromtimestamp(endTime, pytz.utc) print "Segment: Start: %s End: %s" % (startDT, endDT) self.startTime = startDT self.endTime = endDT self.save() return (self.startTime, self.endTime)
def play(self): try: while True: playlist = m3u8.load(self.url) if playlist.media_sequence == self.media_seq: time.sleep(1) continue self.media_seq = playlist.media_sequence target_duration = playlist.target_duration if self.current_seq < self.media_seq: print "ERROR : missing segment" self.current_seq = self.media_seq index = -1 for segment in playlist.segments: index += 1 if self.media_seq + index < self.current_seq: continue seg_url = "%s%s" % (playlist.base_uri, segment.uri) response = urllib2.urlopen(seg_url) buf = response.read() if not os.path.isdir(os.path.dirname(segment.uri)): os.mkdir(os.path.dirname(segment.uri)) f = open(segment.uri, "w") f.write(buf) f.close print "index: ", self.media_seq + index, ", uri: ", segment.uri, ", size ", len(buf) self.current_seq += 1 time.sleep(target_duration - 1) except: print "Error load m3u8 playlist: %s" % self.url
def updateIndexFilePrefix(indexFileSuffix, subst, flightName): """ This is truncating the last n rows from the m3u8 file and reappending the end and the metadata at the top. This fakes our delay """ indexFilePath = settings.DATA_ROOT + indexFileSuffix segmentDirectoryUrl = settings.DATA_URL + os.path.dirname(indexFileSuffix) try: videoDelayInSecs = getClassByName(settings.XGDS_VIDEO_DELAY_AMOUNT_METHOD)(flightName) if videoDelayInSecs > 0: (videoDelayInSegments, m3u8_index) = getSegmentsFromEndForDelay(videoDelayInSecs-30, indexFilePath) else: m3u8_index = m3u8.load(indexFilePath) videoDelayInSegments = 0 segments = m3u8_index.segments if len(segments) > 0: if segments[0].duration > 100: del segments[0] if videoDelayInSegments > 0 and len(segments) > videoDelayInSegments: del segments[-videoDelayInSegments:] for s in segments: s.uri = str(segmentDirectoryUrl) + '/' + s.uri return m3u8_index.dumps() except: traceback.print_exc() traceback.print_stack() return segmentDirectoryUrl
def check_streams(variant_streams, base_uri, verbose): result_msg = "BaseURI=" result_msg += base_uri result_msg += " >> " result_code = 0 for stream in variant_streams: result = "" stream_url = "" stream_url = base_uri stream_url += stream.uri try: stream_playlist = m3u8.load(stream_url) result = "OK" result_msg += stream.uri result_msg += ":" result_msg += result result_msg += ", " except IOError as error: result = str(error) result_code = 2 result_msg += stream.uri result_msg += ":" result_msg += result result_msg += ", " # print stream_url if verbose: print "\t%s: %s" % (stream.uri, result) return (result_code, result_msg[:-2])
def get_publishpoint_streams(self, video_id, stream_type=None, game_type=None): """Return the URL for a stream.""" streams = {} self.get_current_season_and_week() # set cookies url = self.servlets_url + '/publishpoint' if video_id == 'nfl_network': post_data = {'id': '1', 'type': 'channel', 'nt': '1'} elif video_id == 'redzone': post_data = {'id': '2', 'type': 'channel', 'nt': '1'} elif stream_type == 'game': post_data = {'id': video_id, 'type': stream_type, 'nt': '1', 'gt': game_type} else: post_data = {'id': video_id, 'type': stream_type, 'nt': '1'} headers = {'User-Agent': 'iPad'} m3u8_data = self.make_request(url=url, method='post', payload=post_data, headers=headers) m3u8_dict = xmltodict.parse(m3u8_data)['result'] self.log('NFL Dict %s' %m3u8_dict) m3u8_url = m3u8_dict['path'].replace('_ipad', '') temp_path, temp_parm = m3u8_url.split('?', 1) m3u8_header = {'Cookie' : 'nlqptid=' + temp_parm, 'User-Agent' : 'Safari/537.36 Mozilla/5.0 AppleWebKit/537.36 Chrome/31.0.1650.57', 'Accept-encoding' : 'identity', 'Connection' : 'keep-alive'} m3u8_obj = m3u8.load(m3u8_url) if m3u8_obj.is_variant: # if this m3u8 contains links to other m3u8s for playlist in m3u8_obj.playlists: bitrate = str(int(playlist.stream_info.bandwidth[:playlist.stream_info.bandwidth.find(' ')])/100) streams[bitrate] = m3u8_url[:m3u8_url.rfind('/') + 1] + playlist.uri + '?' + m3u8_url.split('?')[1] + '|' + urllib.urlencode(m3u8_header) else: streams['only available'] = m3u8_url return streams
def get_publishpoint_streams(self, video_id, stream_type=None, game_type=None): """Return the URL of a live stream.""" streams = {} self.get_current_season_and_week() # set cookies url = self.servlets_url + '/publishpoint' if video_id == 'nfl_network': post_data = {'id': '1', 'type': 'channel', 'nt': '1'} elif video_id == 'redzone': post_data = {'id': '2', 'type': 'channel', 'nt': '1'} elif stream_type == 'game': post_data = {'id': video_id, 'type': stream_type, 'nt': '1', 'gt': game_type} else: post_data = {'id': video_id, 'type': stream_type, 'nt': '1'} headers = {'User-Agent': 'iPad'} m3u8_data = self.make_request(url=url, method='post', payload=post_data, headers=headers) m3u8_dict = xmltodict.parse(m3u8_data)['result'] self.log('NFL Dict %s' %m3u8_dict) m3u8_url = m3u8_dict['path'].replace('adaptive://', 'http://') m3u8_url = m3u8_dict['path'].replace('_ipad', '') m3u8_obj = m3u8.load(m3u8_url) if m3u8_obj.is_variant: # if this m3u8 contains links to other m3u8s for playlist in m3u8_obj.playlists: bitrate = str(int(playlist.stream_info.bandwidth[:playlist.stream_info.bandwidth.find(' ')])/100) streams[bitrate] = m3u8_url[:m3u8_url.rfind('/') + 1] + playlist.uri + '?' + m3u8_url.split('?')[1] else: streams['only available'] = m3u8_url return streams
def parse_master_playlist(uri): master_url = uri playlists = [] iframe_playlists = [] media_lists = [] # m3u8_content = bytes.decode(is_m3u8_file(uri)) # variant_m3u8 = m3u8.loads(m3u8_content) if is_m3u8_file(uri): variant_m3u8 = m3u8.load(uri) variant_m3u8_base_uri = variant_m3u8.base_uri print("Master_m3u8_base_uri:" + variant_m3u8_base_uri + '\n') print("Start to analyse the Master M3U8 file:") if variant_m3u8.is_variant: print('It\'s a variant m3u8.') playlist_count = 0 playlists = [] for playlist in variant_m3u8.playlists: playlist_uri = playlist.absolute_uri playlist_path = playlist.base_path playlist_tuple = (playlist_uri, playlist_path) if playlist_tuple not in playlists: playlists.append(playlist_tuple) playlist_count += 1 print("There are " + str(len(playlists)) + " play lists.") print(playlists) else: print("It\'s not a variant m3u8.\n") return master_url if variant_m3u8.iframe_playlists: iframe_playlist_count = 0 for iframe_playlist in variant_m3u8.iframe_playlists: iframe_playlist_count += 1 iframe_playlist_uri = iframe_playlist.absolute_uri iframe_playlist_path = iframe_playlist.base_path iframe_playlists.append((iframe_playlist_uri, iframe_playlist_path)) print("There are " + str(iframe_playlist_count) + " iframe playlists:") print(iframe_playlists) else: print("There is no iframe playlist.") if variant_m3u8.media: media_list_count = 0 for media_list in variant_m3u8.media: if media_list.uri: media_list_uri = media_list.absolute_uri media_list_path = media_list.base_path media_list_tuple = (media_list_uri, media_list_path) if media_list_tuple not in media_lists: media_lists.append(media_list_tuple) media_list_count += 1 print("There are " + str(media_list_count) + " media lists:") print(media_lists) else: print("There is no media in m3u8.") return playlists, iframe_playlists, media_lists
def parse_m3u8_novar(url, obj=None): """Parse m3u8 without variable bitrates and return durations and URLs. The return value is a (float, str) tuple of duration and URL. """ m3u8_obj = obj if obj else m3u8.load(url) return [(seg.duration, m3u8_obj.base_uri + seg.uri) for seg in m3u8_obj.segments]
def get_urls_from_m3u8(url): playlist = m3u8.load(url) result = [] if playlist.is_variant: for item in playlist.playlists: result += [(item.base_uri + item.uri, item.stream_info.bandwidth)] result = sorted(result, key = lambda x: x[1]) return result
def load_playlist(m3u8_uri): '''load m3u8 URI and extract playlists ''' logger.debug('Load playlist %s' % m3u8_uri) playlist = m3u8.load(m3u8_uri) if playlist.is_variant: return digest_variant_playlist(playlist, m3u8_uri) else: return digest_singel_playlist(playlist, m3u8_uri)
def get_playlist(playlist_url, fmt): m3u8_obj = m3u8.load(playlist_url) for playlist in m3u8_obj.playlists: if str(playlist.stream_info.resolution).find(fmt) > 0: return playlist.uri return None
def main(): """ Simple command-line program for checking the stream bandwidths to ensure they are what is expected. """ args = get_args() if args.verbose: args.timestamp = True ref_bandwidths = args.bandwidths.split() urls = get_urls(args) result = (0, "OK") result_code = 0 if len(urls) < 1: print >> sys.stderr, render_date_iso8601(), "No valid URLs to process." exit(2) for url in urls: if verify_url(url) == True: try: m3u8_obj = m3u8.load(url) if m3u8_obj.is_variant: if args.verbose: print >> sys.stdout, "Checking URL:", url if args.variance_percent: result = check_variance_bandwidths(m3u8_obj.playlists, ref_bandwidths, args.variance_percent, args.verbose, args.unordered) else: if args.unordered: result = check_unordered_bandwidths(m3u8_obj.playlists, ref_bandwidths, args.verbose) else: result = check_ordered_bandwidths(m3u8_obj.playlists, ref_bandwidths, args.verbose) result_code = set_return_code(result_code, result[0]) if result[0] == 0: print_brief(args.timestamp, sys.stdout, "OK: URL=", url) elif result[0] == 1: print_brief(args.timestamp, sys.stdout, "WARNING: URL=", url, ">>", result[1][:-1]) else: print_brief(args.timestamp, sys.stdout, "CRITICAL: URL=", url, ">>", result[1][:-1]) else: print_brief(args.timestamp, sys.stdout, "CRITICAL: URL=", url, ">> Does not contain any variant playlists") result_code = 2 except IOError as error: print_brief(args.timestamp, sys.stdout, "CRITICAL: URL=", url, ">>", error) result_code = 2 else: print_brief(args.timestamp, sys.stdout, "CRITICAL: URL=", url, ">> Not a valid URL") result_code = 2 if result_code != 0: exit(result_code) return 0
def load_m3u8_playlist(url): stream_types = [] streams = {} m = m3u8.load(url).playlists for l in m: stream_types.append(str(l.stream_info.bandwidth)) streams[str(l.stream_info.bandwidth)] = {'container': 'm3u8', 'video_profile': str(l.stream_info.bandwidth), 'src' : [l.absolute_uri], 'size': 0} stream_types.sort() return stream_types, streams
def list_formats(url): html = open_url(url) parsed_html = BeautifulSoup(html) playlist_url = get_playlist_url(parsed_html) m3u8_obj = m3u8.load(playlist_url) for playlist in m3u8_obj.playlists: print playlist.stream_info.resolution
def find_best_video(uri): playlist = m3u8.load(uri) if not playlist.is_variant: return playlist best_stream = playlist.playlists[0] for stream in playlist.playlists: if stream.stream_info.bandwidth == 'max' or stream.stream_info.bandwidth > best_stream.stream_info.bandwidth: best_stream = stream return find_best_video(best_stream.absolute_uri)
def test_create_iframe_playlist(self): iframe_playlist_uri = 'bigbuckbunny-400k-iframes.m3u8' iframe_playlist_content = IFRAME_PLAYLIST_400K master_playlist = m3u8.load( SAMPLES_PATH + 'original_video/bigbuckbunny.m3u8' ) _, results = create_iframe_playlist(master_playlist.playlists[0]) self.assertEqual(iframe_playlist_uri, results['uri']) self.assertEqual(iframe_playlist_content, results['content'])
def get_vod(self, vod_id): urls = [] token, sig = self.get_token_and_signature(vod_id) quality_playlist_url = USHER_API_VOD.format(vod_id=vod_id, sig=sig, token=token) variant_playlist = m3u8.load(quality_playlist_url) for p in variant_playlist.playlists: quality = p.media[0].name uri = p.uri urls.append({'url': uri, 'quality': quality}) return urls
def __init__(self, grabber, filename, pid): super(Program, self).__init__() self.pid = pid self.title = filename self.grabber = grabber self.filename = os.path.basename(filename) self.m3 = m3u8.load(filename)
def analitzaSubManifestSilent(m, test_ts): u=None try: if m.uri.startswith('http'): url = m.uri else: url = m.base_uri + "/" + m.uri u = m3u8.load(url) except urllib2.HTTPError, e: print " ERROR: Codi HTTP " + str(e.code) + ": " + url
def take_chunks(self): token_url = (TwitchAPI.API_DOMAIN + TwitchAPI.API + '/vods/' + self.vod_id + '/access_token') with urllib.request.urlopen(token_url) as response: token_json = json.loads(response.read().decode('utf-8')) quality_playlist_url = (TwitchAPI.USHER_DOMAIN + '/vod/' + self.vod_id + '?nauthsig=' + token_json['sig'] + '&nauth=' + token_json['token'] + '&allow_source=true') variant_playlist = m3u8.load(quality_playlist_url) try: source_playlist_url = next(i.uri for i in variant_playlist.playlists if i.media[0].group_id == 'chunked') except StopIteration: print('Source(Chunked) playlist have not found.') sys.exit(0) chunks_m3u8 = m3u8.load(source_playlist_url) chunks_rel_path = chunks_m3u8.segments.uri self.chunks = list(map(lambda x: urljoin(source_playlist_url, x), chunks_rel_path))
def _load_media_playlist(media_playlist_url): for retry in range(0, 5): try: return m3u8.load(media_playlist_url) except _RETRIABLE_EXCEPTIONS: time.sleep((2 ** retry) + (random.randint(0, 1000) / 1000)) logging.warning('Retrying to load: ' + media_playlist_url) except urllib2.HTTPError as e: if e.code == 404: time.sleep((2 ** retry) + (random.randint(0, 1000) / 1000)) logging.warning('Retrying to load: ' + media_playlist_url)
def getSegmentsFromEndForDelay(delayTime, indexPath): index = m3u8.load(indexPath) segList = index.segments segCount = 0 totalTime = 0 for s in reversed(segList): totalTime += s.duration segCount += 1 if totalTime >= delayTime: break return segCount, index
def test_load_should_create_object_from_uri(): obj = m3u8.load(playlists.SIMPLE_PLAYLIST_URI) assert isinstance(obj, m3u8.M3U8) assert 5220 == obj.target_duration assert 'http://media.example.com/entire.ts' == obj.segments[0].uri
# Fetch vod index url = _index_api_url.format(video_id) payload = { 'nauth': data['token'], 'nauthsig': data['sig'], 'allow_source': True, 'allow_spectre': False, "player": "twitchweb", "p": int(random() * 999999), "allow_audio_only": True, "type": "any" } r = requests.get(url, params=payload, headers=common_headers) m = m3u8.loads(r.content) index_url = m.playlists[0].uri index = m3u8.load(index_url) # Get the piece we need position = 0 chunks = [] for seg in index.segments: # Add duration of current segment position += seg.duration # Check if we have gotten to the start of the clip if position < int(app.pargs.start): continue # Extract clip name and byte range p = re.match(_chunk_re, seg.absolute_uri)
def get_m3u8(self): self.m3u8 = m3u8.load(self.m3u8_url)
def generate(): try: played_chunk_urls = [] while self.tuner.tuner_lock.locked(): playlist = m3u8.load(channelUri) segments = playlist.segments if len(played_chunk_urls): newsegments = 0 for segment in segments: if segment.absolute_uri not in played_chunk_urls: newsegments += 1 self.fhdhr.logger.info( "Refreshing m3u8, Loaded %s new segments." % str(newsegments)) else: self.fhdhr.logger.info("Loaded %s segments." % str(len(segments))) if playlist.keys != [None]: keys = [{ "url": key.uri, "method": key.method, "iv": key.iv } for key in playlist.keys if key] else: keys = [None for i in range(0, len(segments))] for segment, key in zip(segments, keys): chunkurl = segment.absolute_uri if chunkurl not in played_chunk_urls: played_chunk_urls.append(chunkurl) if (not self.stream_args["duration"] == 0 and not time.time() < self.stream_args["time_end"]): self.fhdhr.logger.info( "Requested Duration Expired.") self.tuner.close() chunk = self.fhdhr.web.session.get( chunkurl).content if not chunk: break # raise TunerError("807 - No Video Data") if key: keyfile = self.fhdhr.web.session.get( key["url"]).content cryptor = AES.new(keyfile, AES.MODE_CBC, keyfile) chunk = cryptor.decrypt(chunk) self.fhdhr.logger.info( "Passing Through Chunk: %s" % chunkurl) yield chunk if playlist.target_duration: time.sleep(int(playlist.target_duration)) self.fhdhr.logger.info( "Connection Closed: Tuner Lock Removed") except GeneratorExit: self.fhdhr.logger.info("Connection Closed.") except Exception as e: self.fhdhr.logger.info("Connection Closed: " + str(e)) finally: self.tuner.close()
print "Got flight %s(%s)" % (flightObj.name, flightObj.uuid) print "Segment path: %s/%s/Video/Recordings/Segment*" % (FLIGHT_BASE, opts.flight) segmentDirs = glob("%s/%s/Video/Recordings/Segment*" % (FLIGHT_BASE, opts.flight)) segmentDirs = sorted(segmentDirs) print "Found %d Segment directories" % len(segmentDirs) for i, dir in enumerate(segmentDirs): videoChunks = glob("%s/*.ts" % dir) videoChunks = sorted( videoChunks, key=lambda chunk: int(re.sub(".+prog_index-(\d+).ts", "\\1", chunk))) if len(videoChunks) > 0: (mode, ino, dev, nlink, uid, gid, size, atime, mtime, ctime) = os.stat(videoChunks[0]) index = m3u8.load('%s/%s' % (dir, 'prog_index.m3u8')) m3u8segment = index.segments[0] duration = m3u8segment.duration # print "duration %f " % duration # print "mtime %d " % mtime # print "mdt %s " % datetime.fromtimestamp(mtime, pytz.utc) startTime = float(mtime) - duration startDT = datetime.fromtimestamp(startTime, pytz.utc) (mode, ino, dev, nlink, uid, gid, size, atime, mtime, ctime) = os.stat(videoChunks[-1]) endTime = mtime endDT = datetime.fromtimestamp(endTime, pytz.utc) print "Segment%03d: Start: %s End: %s" % (i, startDT, endDT)
def find_m3u8_playlist(itunes_url: str, user_agent: str = None) -> MovieData: """ Scrape an iTunes page to find the URL of the M3U8 playlist. Args: itunes_url (str): URL of an iTunes movie page to scrape. user_agent (str, optional): User-Agent string to use for scraping. Defaults to None. Raises: InvalidURL: An inavlid iTunes URL was provided. ConnectionError: A connection error occurred while trying to request the page. HTTPError: An error while trying to download m3u8 playlist data. PageLoadError: The page did not load properly. Returns: MovieData: A MovieData (NamedTuple) object with movie's name, and an M3U8 object of the playlist if the playlist is found. None otherwise. """ # Check whether URL is valid if re.match(ITUNES_STORE_REGEX, itunes_url) is None: raise InvalidURL(f"{itunes_url} is not a valid iTunes movie URL.") site_page: BeautifulSoup = BeautifulSoup(session().get(itunes_url, headers={"User-Agent": user_agent}).text, "lxml") movie_metadata: Union[Tag, NavigableString, None] = site_page.find("script", attrs={"name": "schema:movie", "type": 'application/ld+json'}) if not isinstance(movie_metadata, Tag): raise PageLoadError("The page did not load properly.") # Convert to dictionary structure movie_metadata_dict: dict = json.loads(str(movie_metadata.contents[0]).strip()) media_type: str = movie_metadata_dict['@type'] movie_title: str = html.unescape(movie_metadata_dict['name']) if media_type != "Movie": raise InvalidURL("The provided iTunes URL is not for a movie.") # Scrape a dictionary on the webpage for playlists data playlists_data_tag: Union[Tag, NavigableString, None] = site_page.find("script", attrs={"id": "shoebox-ember-data-store", "type": "fastboot/shoebox"}) # fastboot/shoebox data could not be found if not isinstance(playlists_data_tag, Tag): raise PageLoadError("fastboot/shoebox data could not be found.") # Convert to dictionary structure playlists_data: dict[str, dict] = json.loads(str(playlists_data_tag.contents[0]).strip()) # Loop safely over different structures to find a matching playlist for key in playlists_data.keys(): if isinstance(playlists_data[key].get("included"), list): for item in playlists_data[key]["included"]: if (isinstance(item.get("type"), str) and item["type"] == "offer" and isinstance(item.get("attributes"), dict) and isinstance(item["attributes"].get("assets"), list) and len(item["attributes"]["assets"]) > 0 and isinstance(item["attributes"]["assets"][0], dict) and isinstance(item["attributes"]["assets"][0].get("hlsUrl"), str)): m3u8_url: str = item["attributes"]["assets"][0]["hlsUrl"] try: playlist: M3U8 = m3u8.load(m3u8_url) # If m3u8 playlist is invalid, skip it except ValueError: continue except HTTPError: continue # Assure playlist is for the correct movie if iSubRip.is_playlist_valid(playlist, movie_title): return MovieData(movie_title, playlist) return MovieData(movie_title, None)
def get(self, path): user_id = self.request.get('user_id') if not user_id: raise ValueError("Request param 'user_id' is required.") # For ease of testing we let the world access it self.response.headers[_CORS_ORIGIN_HEADER] = '*' media_url = urlparse.urljoin(_HLS_MEDIA_BASE_URL, path) if path.endswith(_HLS_PLAYLIST_EXTENSION): playlist_m3u8 = m3u8.load(media_url) if playlist_m3u8.is_variant: self.response.out.write(playlist_m3u8.dumps()) else: # Process lookahead segments in the consumed playlist and return # the ad info if one was choosen for the current user in session. ad_info = self._process_lookahead_segments( media_url, user_id, playlist=playlist_m3u8) logging.debug("Processed lookahead segments in %s." % media_url) # If an ad is returned for the consumed playlist, we also # process lookahead segments in the alternative playlists. if ad_info is not None: master_playlist = m3u8.load( self._get_master_playlist_url(media_url)) for playlist in master_playlist.playlists: # Current playlist is already processed if playlist.absolute_uri != media_url: self._process_lookahead_segments( playlist.absolute_uri, user_id, ad_info=ad_info) logging.debug( "Processed lookahead segments in %s." % playlist.absolute_uri) # We pass user ID down to the segments for segment in playlist_m3u8.segments: segment.uri = self._update_query_string(segment.uri, user_id=user_id) # After we update all the segment URIs playlist_content = playlist_m3u8.dumps() for segment in playlist_m3u8.segments: if segment.cue_out: segment_path = self._get_segment_path(segment) # Get the task name from a previous HlsHandler request task_name = memcache.get(segment_path, namespace=user_id) if task_name is not None: hls_cue_out_start = playlist_content.find( _HLS_EXT_CUE_OUT + "\n") hls_cue_out_end = hls_cue_out_start + len( _HLS_EXT_CUE_OUT + "\n") vast_url = self.uri_for('vast', task_name=task_name, _full=True) # VAST URL tag with the format of '#EXT-X-VAST-URL:<vast_url>' hls_ext_vast_url_tag = "%s:%s" % ( _HLS_EXT_VAST_URL, vast_url) # M3U8 Python library has no support for our custom tag, # and it also doesn't let us add comments. So we instead # do a string manipulation to populate the VAST URL tag. playlist_content = ( playlist_content[:hls_cue_out_end] + hls_ext_vast_url_tag + "\n" + playlist_content[hls_cue_out_end:]) self.response.out.write(playlist_content) else: # Get the task name from a previous HlsHandler request task_name = memcache.get(path, namespace=user_id) if task_name is not None: # Get the injected ad segment URL from InjectionWorker if # available, otherwise redirect to the original media URL redirect_dict = memcache.get(task_name) or {} redirect_url = redirect_dict.get(path, media_url) else: # We redirect to the original media URL (no ad injection) redirect_url = media_url self.redirect(redirect_url)
def _process_lookahead_segments(self, playlist_url, user_id, playlist=None, ad_info=None): if playlist is None: playlist = m3u8.load(playlist_url) # Index of the first and the oldest lookahead segment lookahead_start = self._find_lookahead_start(playlist) # Iterate until there are no lookahead segments while lookahead_start < len(playlist.segments): # Process the lookahead segments backwards segment_index = len(playlist.segments) - 1 segment = playlist.segments[segment_index] if segment.cue_out: ad_start_path = self._get_segment_path(segment) # Get the task name from a previous HlsHandler request task_name = memcache.get(ad_start_path, namespace=user_id) # Skip if the task is already added to the queue if task_name is None: if ad_info is None: # Get the ad source and the ad duration pair from the Studio UI ad_info_args = memcache.get(segment.absolute_uri, namespace=user_id) if ad_info_args is not None: ad_info = AdInfo(*ad_info_args) # Request ad from DFP if one is not specified if ad_info is None: dfp_response = self._request_ad_from_dfp( self._get_master_playlist_path(playlist_url)) self._copy_cookies(dfp_response, self.response) ad_elem = etree.fromstring(dfp_response.content) if ad_elem is not None: ad_info = self._get_ad_info(ad_elem) # DFP may also return no ads, so we have to check for it if ad_info is not None: task_name = self._generate_task_name(segment, ad_info) ad_start_seq = playlist.media_sequence + segment_index ad_segment_path = ad_start_path ad_segment_seq = ad_start_seq while ad_info.duration > timedelta(seconds=0): # Pass the task name to the subsequent HlsHandler requests memcache.set(ad_segment_path, task_name, time=_TTL_1_HOUR, namespace=user_id) ad_segment_path = ad_segment_path.replace( str(ad_segment_seq), str(ad_segment_seq + 1)) ad_segment_seq += 1 ad_info.duration -= timedelta( seconds=playlist.target_duration) try: taskqueue.add(queue_name=_TASKQUEUE_NAME, name=task_name, url=_TASKQUEUE_URL, params={ 'playlist_url': playlist_url, 'ad_source': ad_info.source, 'ad_start_seq': ad_start_seq, 'ad_start_time': _AD_START_TIME }) except (taskqueue.TaskAlreadyExistsError, taskqueue.TombstonedTaskError) as e: logging.debug("Task '%s' already queued." % task_name) finally: if ad_info.vast is not None: vast_cache_key = _VAST_CACHE_KEY % task_name vast_content = etree.tostring(ad_info.vast) memcache.set(vast_cache_key, vast_content, time=_TTL_1_HOUR) # Drop the last segment playlist.segments.pop() return ad_info
def download_news_video_and_content(news_id, base_dir, chunk_size=32 * 1024, video_dir="video", asset_dir="assets", audio_dir="audio"): video_dir = os.path.join(base_dir, video_dir) asset_dir = os.path.join(base_dir, asset_dir) audio_dir = os.path.join(base_dir, audio_dir) makedirs(video_dir) makedirs(asset_dir) makedirs(audio_dir) text_path = os.path.join(asset_dir, "{}.txt".format(news_id)) original_text_path = os.path.join(asset_dir, "original-{}.txt".format(news_id)) video_path = os.path.join(video_dir, "{}.ts".format(news_id)) audio_path = os.path.join(audio_dir, "{}.wav".format(news_id)) params = { 'NJC': 'NJC400', 'NID': news_id, # NB11515152 'CD': 'A0100', } response = requests.request( method='GET', url=BASE_URL, params=params, ) soup = soupify(response.text) article_contents = soup.find_all('article_contents') assert len(article_contents) == 1, \ "# of <article_contents> of {} should be 1: {}".format(news_id, response.text) text = soupify(article_contents[0].text).get_text() # remove <div> with open(original_text_path, "w", -1, "utf-8") as f: f.write(text) with open(text_path, "w", -1, "utf-8") as f: from nltk import sent_tokenize text = re.sub(r'\[.{0,80} :\s.+]', '', text) # remove quote text = re.sub(r'☞.+http.+\)', '', text) # remove quote text = re.sub(r'\(https?:\/\/.*[\r\n]*\)', '', text) # remove url sentences = sent_tokenize(text) sentences = [ sent for sentence in sentences for sent in sentence.split('\n') if sent ] new_texts = [] for sent in sentences: sent = sent.strip() sent = re.sub(r'\([^)]*\)', '', sent) #sent = re.sub(r'\<.{0,80}\>', '', sent) sent = sent.replace('…', '.') new_texts.append(sent) f.write("\n".join([sent for sent in new_texts if sent])) vod_paths = soup.find_all('vod_path') assert len(vod_paths) == 1, \ "# of <vod_path> of {} should be 1: {}".format(news_id, response.text) if not os.path.exists(video_path): redirect_url = soup.find_all('vod_path')[0].text list_url = m3u8.load(redirect_url).playlists[0].absolute_uri video_urls = [ segment.absolute_uri for segment in m3u8.load(list_url).segments ] with open(video_path, "wb") as f: for url in video_urls: response = requests.get(url, stream=True) total_size = int(response.headers.get('content-length', 0)) for chunk in response.iter_content(chunk_size): if chunk: # filter out keep-alive new chunks f.write(chunk) if not os.path.exists(audio_path): encoder = get_encoder_name() command = "{} -y -loglevel panic -i {} -ab 160k -ac 2 -ar 44100 -vn {}".\ format(encoder, video_path, audio_path) subprocess.call(command, shell=True) return True
def get_next_clip(self, current_clip_name=None): """ """ # Get current folder current_folder = int(self.valid_folders[self.current_folder_index]) clipname, clip_start_time = datetime_utils.get_clip_name_from_unix_time( self.folder_name.replace("_", "-"), self.current_clip_start_time) # if real_time execution mode is specified if self.real_time: # sleep till enough time has elapsed now = datetime.utcnow() time_to_sleep = (current_clip_name - now).total_seconds() if time_to_sleep < 0: print("Issue with timing") if time_to_sleep > 0: time.sleep(time_to_sleep) # read in current m3u8 file # stream_url for the current AWS folder stream_url = "{}/hls/{}/live.m3u8".format((self.stream_base), (current_folder)) stream_obj = m3u8.load(stream_url) num_total_segments = len(stream_obj.segments) num_segments_in_wav_duration = math.ceil( self.polling_interval_in_seconds / stream_obj.target_duration) # calculate the start index by computing the current time - start of current folder segment_start_index = math.ceil( datetime_utils.get_difference_between_times_in_seconds( self.current_clip_start_time, current_folder) / stream_obj.target_duration) segment_end_index = segment_start_index + num_segments_in_wav_duration if segment_end_index > num_total_segments: # move to the next folder and increment the current_clip_start_time to the new self.current_folder_index += 1 self.current_clip_start_time = self.valid_folders[ self.current_folder_index] return None, None, None # Can get the whole segment so update the clip_start_time for the next clip # We do this before we actually do the pulling in case there is a problem with this clip self.current_clip_start_time = datetime_utils.add_interval_to_unix_time( self.current_clip_start_time, self.polling_interval_in_seconds) # Create tmp path to hold .ts segments tmp_path = "tmp_path" os.makedirs(tmp_path, exist_ok=True) file_names = [] for i in range(segment_start_index, segment_end_index): audio_segment = stream_obj.segments[i] base_path = audio_segment.base_uri file_name = audio_segment.uri audio_url = base_path + file_name try: scraper.download_from_url(audio_url, tmp_path) file_names.append(file_name) except Exception: print("Skipping", audio_url, ": error.") # concatentate all .ts files with ffmpeg hls_file = (clipname + ".ts") audio_file = (clipname + ".wav") wav_file_path = os.path.join(self.wav_dir, audio_file) filenames_str = " ".join(file_names) concat_ts_cmd = "cd {tp} && cat {fstr} > {hls_file}".format( tp=tmp_path, fstr=filenames_str, hls_file=hls_file) os.system(concat_ts_cmd) # read the concatenated .ts and write to wav stream = ffmpeg.input(os.path.join(tmp_path, Path(hls_file))) stream = ffmpeg.output(stream, wav_file_path) ffmpeg.run(stream, quiet=False) # clear the tmp_path os.system(f'rm -rf {tmp_path}') # If we're in demo mode, we need to fake timestamps to make it seem like the date range is real-time if current_clip_name: clipname, _ = get_readable_clipname( self.folder_name.replace("_", "-"), current_clip_name) # rename wav file full_new_clip_path = os.path.join(self.wav_dir, clipname + ".wav") os.rename(wav_file_path, full_new_clip_path) wav_file_path = full_new_clip_path # change clip_start_time - this has to be in UTC so that the email can be in PDT clip_start_time = current_clip_name.isoformat() + "Z" # Get new index return wav_file_path, clip_start_time, current_clip_name
def verify_playlist_link(url, timeout, indent=1, check_first_N_only=5): ''' ''' nice_print('Loading playlist: {0}'.format(url), indent=indent, debug=True) if indent > 6: nice_print('ERROR nested playlist too deep', indent=indent) return False # one nasty server 301-redirected the .m3u8 file to a .mp3 file, ouch. This catches that: try: m3u8_head = requests.head(url, timeout=(timeout, timeout), allow_redirects=False) if 300 <= m3u8_head.status_code < 310: m3u8_head2 = requests.head(url, timeout=(timeout, timeout), allow_redirects=True) url_redirected = m3u8_head2.history[0].headers['Location'] extension = urlparse(url_redirected).path.split(".")[-1] if extension not in ("m3u8", "m3u"): nice_print( 'ERROR m3u8-playlist 30x-redirected to "{0}"-filetype. Skipping this.' .format(extension), indent=indent, debug=False) return False except Exception as e: nice_print('ERROR loading redirected playlist: {0}'.format( str(e)[:100]), indent=indent, debug=True) return False try: m3u8_obj = m3u8.load(url, timeout=timeout) except Exception as e: nice_print('ERROR loading playlist: {0}'.format(str(e)[:100]), indent=indent, debug=True) return False if 0 == len(m3u8_obj.data['playlists']) + len(m3u8_obj.data['segments']): nice_print('ERROR: playlist is empty.', indent=indent, debug=True) return False for nested_playlist in m3u8_obj.data['playlists']: if nested_playlist['uri'].startswith(('https://', 'http://')): nested_url = nested_playlist['uri'] else: nested_url = '{0}{1}'.format(m3u8_obj.base_uri, nested_playlist['uri']) return verify_playlist_link(nested_url, timeout=timeout, indent=indent + 1) counter = 0 for segment in m3u8_obj.data['segments']: if segment['uri'].startswith(('https://', 'http://')): url = segment['uri'] else: url = '{0}{1}'.format(m3u8_obj.base_uri, segment['uri']) if not verify_video_link(url, timeout=timeout): return False # first occurring bad one declares this whole list bad. Is that a good choice? counter += 1 if counter >= check_first_N_only: msg = 'OK: skipping tests of remaining {0} entries because we have {1} good files already in this playlist' nice_print(msg.format( len(m3u8_obj.data['segments']) - counter, counter), indent=indent, debug=True) return True return True
def Get_ts_Playlist(m3u8_URL): m3u8_obj = m3u8.load(m3u8_URL) playlist = [el['uri'] for el in m3u8_obj.data['segments']] return playlist
def loadM3u8(url): playlist = m3u8.load(url) for i in range(len(playlist.segments)): playlist.segments[i].uri=playlist.segments[i].absolute_uri return playlist.dumps()
def test_load_should_remember_redirect(): obj = m3u8.load(playlists.REDIRECT_PLAYLIST_URI) urlparsed = url_parser.urlparse(playlists.SIMPLE_PLAYLIST_URI) assert urlparsed.scheme + '://' + urlparsed.netloc + "/" == obj.base_uri
import m3u8 _path = input('Enter m3u8 path:') playlist = m3u8.load(_path) # playlist = m3u8.load('https://dplusindia-google.prod-vod.h264.io/d6e4176a-8384-4b8f-98f1-9688721421c4/x-goog-token=Expires=1590180862&KeyName=prod-sign-url-key&Signature=E_6EtuTzw6abggGHrOfxUkKaWcs/Master.m3u8') # playlist = m3u8.load('https://test-streams.mux.dev/x36xhzz/x36xhzz.m3u8') # playlist = m3u8.load('https://bitdash-a.akamaihd.net/content/sintel/hls/playlist.m3u8') # playlist = m3u8.load('https://test-streams.mux.dev/x36xhzz/url_2/193039199_mp4_h264_aac_ld_7.m3u8') # this is an absolute filename # playlist = m3u8.load('../sample-stream-playlists/sintel/playlist.m3u8') # print(playlist.segments) # print(playlist.target_duration) # print(playlist.media) # print(playlist.playlists) # print(playlist.playlists[0].stream_info.resolution) # print(playlist.playlists[0].media[0].type) # print(playlist.files) # if you want to write a file from its content # playlist.dump('playlist.m3u8') if playlist.playlists: print('%s playlists found...' % len(playlist.playlists)) count = 0 for pl in playlist.playlists: count += 1 print('%d - Resolution: %s Bandwidth: %s' %
def test_load(): m3u8_obj = m3u8.load( './index-v1-a1.m3u8') # this could also be an absolute filename print(m3u8_obj.segments) print(m3u8_obj.target_duration) print(m3u8_obj.keys)
def get_uri_from_m3u8(self, file_name): print("正在解析真实下载地址..." + str(file_name)) m3u8Obj = m3u8.load(file_name) print("解析完成.") return m3u8Obj.segments
def load_m3u8(url): urls = [] m = m3u8.load(url) for seg in m.segments: urls.append(seg.absolute_uri) return urls
def _collectSegments(self): logger.info('Downloading and parsing HLS manifest from %s' % self.manifesturi) m3u8_obj = m3u8.load(self.manifesturi) if not m3u8_obj.is_variant: raise Exception('%s is not a master manifest' % self.manifesturi) listlengths = [] for mediaplaylist in m3u8_obj.playlists: url = urlparse(self.manifesturi) mediauri = mediaplaylist.uri if mediaplaylist.uri[0] == "/": mediauri = url.scheme + "://" + url.hostname + mediaplaylist.uri debug.log('Building segment list from %s' % mediauri) try: logger.info('Downloading segment playlist from %s' % mediauri) bw = mediaplaylist.stream_info.average_bandwidth if not bw: bw = mediaplaylist.stream_info.bandwidth segmentlist = SegmentList(mediauri, str(bw), self.tmpdir) except Exception as e: logger.error('Failed to download: %s' % str(e)) else: logger.info('Segment playlist from %s downloaded and parsed' % mediauri) self.bitrates.append(segmentlist) listlengths.append(segmentlist.getLength()) if len(self.bitrates) == 0: raise Exception('No segment playlists that could be downloaded was found') # This is to handle the edge case where the segmentlists differs in length and start segment # A special case that actually should not happened debug.log('Shortest list length %d' % min(listlengths)) debug.log('Longest list length %d' % max(listlengths)) headsegments = {} for segmentlist in self.bitrates: if segmentlist.getFirstSegment() in headsegments: headsegments[segmentlist.getFirstSegment()] += 1 else: headsegments[segmentlist.getFirstSegment()] = 1 debug.log(headsegments) # Find start segment winner winner = sorted(headsegments.items(), key=operator.itemgetter(1), reverse=True)[0][0] # Make sure all bitrates starts with the same segment if len(headsegments.keys()) > 1: debug.log('First segment differs and we have chosen %s as winner' % winner) for segmentlist in self.bitrates: if segmentlist.getFirstSegment() != winner: segmentlist.removeFirstSegment() # Make sure that we have the same length on all bitrates segmentlengths = {} for segmentlist in self.bitrates: length = segmentlist.getLength() if length in segmentlengths: segmentlengths[length] += 1 else: segmentlengths[length] = 1 shortestlength = sorted(segmentlengths.items(), key=operator.itemgetter(0))[0][0] debug.log(shortestlength) for segmentlist in self.bitrates: length = segmentlist.getLength() if length > shortestlength: segmentlist.removeLastSegment() # Sanity check firstsegments = {} for segmentlist in self.bitrates: debug.log('First segment: %s of (%d)' % (segmentlist.getFirstSegment(), segmentlist.getLength())) if segmentlist.getFirstSegment() in firstsegments: firstsegments[segmentlist.getFirstSegment()] += 1 else: firstsegments[segmentlist.getFirstSegment()] = 1 debug.log('Keys %d' % len(firstsegments.keys())) if len(firstsegments.keys()) > 1: debug.log(firstsegments) logger.warning("First segment in segment lists differs")
#!/usr/bin/env python3 """ Use a playlist to stream files from m3u8: https://github.com/globocom/m3u8 pip install m3u8 Example from Akamai: https://learn.akamai.com/en-us/webhelp/media-services-live/media-services-live-4-user-guide/GUID-0A50253F-0B1B-406B-A8C9-3788CB42F950.html """ from pathlib import Path import pylivestream.api as pls import m3u8 playlist_m3u8 = Path(__file__).parent / "local.m3u8" playlist = m3u8.load(str(playlist_m3u8)) files = playlist.files for file in files: pls.stream_file( ini_file=None, websites="localhost", assume_yes=True, video_file=file, )
from kivy.uix.videoplayer import VideoPlayer import m3u8 import requests #http://bsn246u6-i.akamaihd.net/hls/live/237201/Live2N24CMS/03.m3u8 m3u8_obj = m3u8.load( 'http://bsn246u6-i.akamaihd.net/hls/live/237201/Live2N24CMS/03.m3u8') print str(m3u8_obj) print m3u8_obj.segments print m3u8_obj.target_duration t = requests.get( 'http://bsn246u6-i.akamaihd.net/hls/live/237201/Live2N24CMS/20161004T125034-03-227477.ts' ) print t player = VideoPlayer( source= 'http://bsn246u6-i.akamaihd.net/hls/live/237201/Live2N24CMS/20161004T125034-03-227630.ts' ) player.state = 'play'
def get(self): if not self.stream_args["duration"] == 0: self.stream_args[ "time_end"] = self.stream_args["duration"] + time.time() if not self.stream_args["true_content_type"].startswith( tuple(["application/", "text/"])): self.fhdhr.logger.info("Direct Stream of %s URL: %s" % (self.stream_args["true_content_type"], self.stream_args["channelUri"])) req = self.fhdhr.web.session.get(self.stream_args["channelUri"], stream=True) def generate(): try: chunk_counter = 1 while self.tuner.tuner_lock.locked(): for chunk in req.iter_content( chunk_size=self.chunksize): if (not self.stream_args["duration"] == 0 and not time.time() < self.stream_args["time_end"]): req.close() self.fhdhr.logger.info( "Requested Duration Expired.") self.tuner.close() if not chunk: break # raise TunerError("807 - No Video Data") self.fhdhr.logger.info( "Passing Through Chunk #%s with size %s" % (chunk_counter, self.chunksize)) yield chunk chunk_counter += 1 self.fhdhr.logger.info( "Connection Closed: Tuner Lock Removed") except GeneratorExit: self.fhdhr.logger.info("Connection Closed.") except Exception as e: self.fhdhr.logger.info("Connection Closed: " + str(e)) finally: req.close() self.tuner.close() # raise TunerError("806 - Tune Failed") else: self.fhdhr.logger.info("Detected stream URL is m3u8: %s" % self.stream_args["true_content_type"]) channelUri = self.stream_args["channelUri"] while True: videoUrlM3u = m3u8.load(channelUri) if len(videoUrlM3u.playlists): channelUri = videoUrlM3u.playlists[0].absolute_uri else: break def generate(): try: played_chunk_urls = [] while self.tuner.tuner_lock.locked(): playlist = m3u8.load(channelUri) segments = playlist.segments if len(played_chunk_urls): newsegments = 0 for segment in segments: if segment.absolute_uri not in played_chunk_urls: newsegments += 1 self.fhdhr.logger.info( "Refreshing m3u8, Loaded %s new segments." % str(newsegments)) else: self.fhdhr.logger.info("Loaded %s segments." % str(len(segments))) if playlist.keys != [None]: keys = [{ "url": key.uri, "method": key.method, "iv": key.iv } for key in playlist.keys if key] else: keys = [None for i in range(0, len(segments))] for segment, key in zip(segments, keys): chunkurl = segment.absolute_uri if chunkurl not in played_chunk_urls: played_chunk_urls.append(chunkurl) if (not self.stream_args["duration"] == 0 and not time.time() < self.stream_args["time_end"]): self.fhdhr.logger.info( "Requested Duration Expired.") self.tuner.close() chunk = self.fhdhr.web.session.get( chunkurl).content if not chunk: break # raise TunerError("807 - No Video Data") if key: keyfile = self.fhdhr.web.session.get( key["url"]).content cryptor = AES.new(keyfile, AES.MODE_CBC, keyfile) chunk = cryptor.decrypt(chunk) self.fhdhr.logger.info( "Passing Through Chunk: %s" % chunkurl) yield chunk if playlist.target_duration: time.sleep(int(playlist.target_duration)) self.fhdhr.logger.info( "Connection Closed: Tuner Lock Removed") except GeneratorExit: self.fhdhr.logger.info("Connection Closed.") except Exception as e: self.fhdhr.logger.info("Connection Closed: " + str(e)) finally: self.tuner.close() # raise TunerError("806 - Tune Failed") return generate()
def test_load_non_exist_file(): m3u8_obj = m3u8.load( './non_exist.m3u8') # this could also be an absolute filename print(m3u8_obj.segments) print(m3u8_obj.target_duration) print(m3u8_obj.keys)
def downts_thread(url=None,save_dir=None,processInfo=None): # 获取频道名称 channel_name = url.split('/')[-2] listfile_name = url.split('/')[-1] # 初始化提取信息 processInfo = {channel_name: {'latest': '-', 'action': 'init process', 'info': 'init process'}} showProcessInfo(processInfo) # 新建文件夹 save_dir = os.path.join(save_dir, channel_name) if not os.path.exists(save_dir): os.makedirs(save_dir) # list_file_name list_file_name = channel_name + "_" + listfile_name # list_file_full_path list_file_full_path = os.path.join(save_dir, list_file_name) pre_md5 = "" # 文件校验码 while 1: djv = Dejavu(config) # 保存ts流列表文件(cctv15_01.m3u8) processInfo[channel_name]['action'] = 'downloading' processInfo[channel_name]['info'] = 'downloading file:' + list_file_name showProcessInfo(processInfo) # save(url,list_file_full_path,True) download_to_file(url,list_file_full_path,True) #并计算校验码 current_md5 = getFileMD5(list_file_full_path) # 当前文件校验码 # 对比是否有变化 if current_md5==pre_md5: processInfo[channel_name]['action'] = 'thread sleeping' processInfo[channel_name]['info'] = 'there is no new media ,'+ channel_name+'-thread is sleeping' showProcessInfo(processInfo) #休息一下继续开始 sleep(Config.TS_SLEEP); continue # 提取文件列表 m3u8_obj = m3u8.load(url) # 间隔时间 gap_time = m3u8_obj.target_duration base_URL = url[:url.rfind('/')+1] for index,file in enumerate(m3u8_obj.files): # 文件的网络路径 file_url = base_URL + file # 获取开始时间 ts_start_time = getTS_Format_Time(m3u8_obj.program_date_time, int(index * int(gap_time))) # 得到完整文件名 file_temp_name = channel_name+"_"+ts_start_time+'.ts' file_full_path = os.path.join(save_dir,file_temp_name) song_name = os.path.splitext(os.path.basename(file_temp_name))[0] if not djv.isSongFingerPrinted(song_name): #下载该文件 processInfo[channel_name]['action'] = 'downloading' processInfo[channel_name]['info'] = 'downloading file:' + file_temp_name showProcessInfo(processInfo) download_to_file(file_url,file_full_path,False) # 调用指纹提取程序提取指纹 processInfo[channel_name]['action'] = 'fingerprinting' processInfo[channel_name]['info'] = 'fingerpinting file:' + file_temp_name showProcessInfo(processInfo) djv.fingerprint_file(file_full_path) else: processInfo[channel_name]['action'] = 'skip fingerprinted' processInfo[channel_name]['info'] = file_temp_name + 'has fingerprinted' showProcessInfo(processInfo) # 更新信息 processInfo[channel_name]['action'] = 'update-info' processInfo[channel_name]['info'] = channel_name+' has updated to ' + ts_start_time processInfo[channel_name]['latest'] = ts_start_time showProcessInfo(processInfo) # 清理文件 processInfo[channel_name]['action'] = 'delete ts file' processInfo[channel_name]['info'] = 'deleting file:' + file_temp_name showProcessInfo(processInfo) if os.path.exists(file_full_path): os.remove(file_full_path) # 提取完成更新文件校验码 pre_md5 = current_md5 # 是否需要清理历史数据 # 休息一下继续开始 processInfo[channel_name]['action'] = 'thread sleeping' processInfo[channel_name]['info'] = 'playlist has been stored,' + channel_name + '-thread is sleeping' showProcessInfo(processInfo) if os.path.exists(list_file_full_path): os.remove(list_file_full_path) sleep(Config.TS_SLEEP)
htmlfile = cloudscraper.create_scraper(browser='firefox', delay=10).get(url) result = re.search("https://.+m3u8", htmlfile.text) m3u8url = result[0] m3u8urlList = m3u8url.split('/') m3u8urlList.pop(-1) downloadurl = '/'.join(m3u8urlList) # 儲存 m3u8 file 至資料夾 m3u8file = os.path.join(folderPath, dirName + '.m3u8') urllib.request.urlretrieve(m3u8url, m3u8file) # In[5]: # 得到 m3u8 file裡的 URI和 IV m3u8obj = m3u8.load(m3u8file) m3u8uri = '' m3u8iv = '' for key in m3u8obj.keys: if key: m3u8uri = key.uri m3u8iv = key.iv # 儲存 ts網址 in tsList tsList = [] for seg in m3u8obj.segments: tsUrl = downloadurl + '/' + seg.uri tsList.append(tsUrl) # In[6]:
for url in videoUrls: sleep(randint(1, 2)) r = requests.get(baseUrl + url) soup = BeautifulSoup(r.text, 'lxml') videoUrl = '/'.join(soup.find('video')['src'].split('/')[:-1]) baseName = videoUrl.split('/')[-1] if not isdir(baseName): if exists(baseName): remove(baseName) mkdir(baseName) # Get list of audio clips and save them if not exists(baseName + '.wav'): clips = [] audioIndex = m3u8.load(videoUrl + '/audio/zho/zho.m3u8') if audioIndex.files: for fileAudio in audioIndex.files: if not exists(join(baseName, fileAudio)): sleep(randint(3, 5)) print(videoUrl + '/audio/zho/' + fileAudio) r = requests.get(videoUrl + '/audio/zho/' + fileAudio) with open(join(baseName, fileAudio), 'wb') as f: f.write(r.content) FFmpeg(global_options='-y', inputs={ join(baseName, fileAudio): None }, outputs={ join(baseName, fileAudio + '.wav'): None }).run()
def get_station_stream_uri(self, station_id): print("Getting station info for " + station_id + "...") try: videoUrlReq = urllib.request.Request('https://api.locastnet.org/api/watch/station/' + str(station_id) + '/' + self.current_location['latitude'] + '/' + self.current_location['longitude'], headers={'Content-Type': 'application/json', 'authorization': 'Bearer ' + self.current_token, 'User-agent': 'Mozilla/5.0'}) videoUrlOpn = urllib.request.urlopen(videoUrlReq) videoUrlRes = json.load(videoUrlOpn) videoUrlOpn.close() except urllib.error.URLError as urlError: print("Error when getting the video URL: " + str(urlError.reason)) return False except urllib.error.HTTPError as httpError: print("Error when getting the video URL: " + str(httpError.reason)) return False except: videoUrlReqErr = sys.exc_info()[0] print("Error when getting the video URL: " + videoUrlReqErr.message) return False print("Determining best video stream for " + station_id + "...") bestStream = None # find the heighest stream url resolution and save it to the list videoUrlM3u = m3u8.load(videoUrlRes['streamUrl'], headers={'authorization': 'Bearer ' + self.current_token, 'User-agent': 'Mozilla/5.0'}) print("Found " + str(len(videoUrlM3u.playlists)) + " Playlists") if len(videoUrlM3u.playlists) > 0: for videoStream in videoUrlM3u.playlists: if bestStream is None: bestStream = videoStream elif ((videoStream.stream_info.resolution[0] > bestStream.stream_info.resolution[0]) and (videoStream.stream_info.resolution[1] > bestStream.stream_info.resolution[1])): bestStream = videoStream elif ((videoStream.stream_info.resolution[0] == bestStream.stream_info.resolution[0]) and (videoStream.stream_info.resolution[1] == bestStream.stream_info.resolution[1]) and (videoStream.stream_info.bandwidth > bestStream.stream_info.bandwidth)): bestStream = videoStream if bestStream is not None: print(station_id + " will use " + str(bestStream.stream_info.resolution[0]) + "x" + str(bestStream.stream_info.resolution[1]) + " resolution at " + str(bestStream.stream_info.bandwidth) + "bps") return bestStream.absolute_uri else: print("No variant streams found for this station. Assuming single stream only.") return videoUrlRes['streamUrl']
def process_playback_url(playback_url, auth_string): logging.debug('Playback url %s' % playback_url) stream_quality = str(get_setting('StreamQuality')) bitrate_limit = int(get_setting('BitrateLimit')) logging.debug('Stream Quality %s' % stream_quality) try: m3u8_obj = m3u8.load(playback_url) except Exception as e: logging.error('Unable to load m3u8 %s' % e) playback_url += '|' + auth_string item = xbmcgui.ListItem(path=playback_url) return setResolvedUrl(plugin.handle, True, item) success = True use_inputstream_addon = not get_setting_as_bool('DisableInputStream') if not use_inputstream_addon: if m3u8_obj.is_variant: stream_options = list() bandwidth_key = 'bandwidth' m3u8_obj.playlists.sort( key=lambda playlist: playlist.stream_info.bandwidth, reverse=True) m3u8_obj.data['playlists'].sort(key=lambda playlist: int(playlist[ 'stream_info'][bandwidth_key]), reverse=True) stream_quality_index = str(get_setting('StreamQualityIndex')) stream_index = None should_ask = False try: stream_index = int(stream_quality_index) if stream_index < 0 or stream_index >= len(m3u8_obj.playlists): should_ask = True except ValueError: should_ask = True if '0' == stream_quality: # Best stream_index = 0 should_ask = False for playlist in m3u8_obj.data['playlists']: stream_info = playlist['stream_info'] bandwidth = int(stream_info[bandwidth_key]) / 1024 if bandwidth <= bitrate_limit: break stream_index += 1 elif '2' == stream_quality: # Ask everytime should_ask = True if should_ask: for playlist in m3u8_obj.data['playlists']: stream_info = playlist['stream_info'] resolution = stream_info['resolution'] frame_rate = stream_info[ 'frame_rate'] if 'frame_rate' in stream_info else 30.0 bandwidth = int(int(stream_info[bandwidth_key]) / 1024) if 'average_bandwidth' in stream_info: logging.debug('bandwidth: %s average bandwidth: %s' % (stream_info['bandwidth'], stream_info['average_bandwidth'])) stream_options.append( get_string(30450) % (resolution, frame_rate, bandwidth)) dialog = xbmcgui.Dialog() stream_index = dialog.select(get_string(30440), stream_options) if stream_index < 0: success = False else: set_setting('StreamQualityIndex', value=str(stream_index)) uri = m3u8_obj.playlists[stream_index].uri logging.debug('Chose stream %d; %s' % (stream_index, uri)) if 'http' not in uri[0:4]: index_of_last_slash = playback_url.rfind('/') uri = playback_url[0:index_of_last_slash] + '/' + uri item = xbmcgui.ListItem(path=uri + '|' + auth_string) setResolvedUrl(plugin.handle, success, item) else: item = xbmcgui.ListItem(path=playback_url) setResolvedUrl(plugin.handle, success, item) else: logging.debug('Using inputstream.adaptive addon') item = xbmcgui.ListItem(path=playback_url) item.setProperty('inputstreamaddon', 'inputstream.adaptive') item.setProperty('inputstream.adaptive.manifest_type', 'hls') setResolvedUrl(plugin.handle, success, item)
def handle(self, *args, **options): m3u8_obj = m3u8.load(settings.BASE_DIR + '/importar/{}'.format(options['filename'])) x = 1 for channel in m3u8_obj.segments: name = channel.title.split(',')[-1] name = name[:255] try: group_title = re.findall(r'group-title=\"(.*)\" ', channel.title)[0] group_title = group_title[:255] except IndexError as e: group_title = '' try: logo = re.findall(r'tvg-logo=\"(.*)\" ', channel.title)[0] logo = logo[:255] except IndexError as e: logo = '' print('sem logo e:', e) uri = channel.uri uri = uri[:255] print('* {}'.format(x)) try: canal = Canal.objects.get(uri=uri) if options['test_stream']: if canal.status == 200: response = requests.get(uri, stream=True) size = 0 status = 0 for chunk in response.iter_content(256): size += len(chunk) if size > 512: print('size: ', size) status = 200 break if status == 0: status = response.status_code canal.status = status canal.save() print('ID: {} - status: {}'.format(canal.id, status)) print('ID: {}'.format(canal.id)) except Canal.DoesNotExist: try: print('testing: {} ({})'.format(uri, name)) response = requests.get(uri, stream=True, timeout=60) headers = requests.head(uri, stream=True, timeout=60).headers content_type = headers.get('content-type') print('Content-type: ', content_type) if content_type == 'application/vnd.apple.mpegurl': status = 200 elif content_type == 'application/vnd.apple.mpegURL': status = 200 elif content_type == 'video/mp4': status = 200 elif content_type == 'application/x-mpegURL': status = 200 elif content_type == 'application/vnd.apple.mpegurl, application/vnd.apple.mpegurl': status = 200 elif content_type == 'video/mp2t': status = 200 elif content_type == 'audio/aacp': # RADIO status = 1 elif content_type == 'audio/mpeg': # RADIO status = 1 elif content_type == 'audio/x-mpegurl': # RADIO status = 1 elif content_type == 'text/html; charset=UTF-8': status = 404 elif content_type == 'text/html; charset=utf-8': status = 404 elif content_type == 'text/html;charset=UTF-8': status = 404 elif content_type == 'image/png': status = 404 elif content_type == 'image/jpeg': status = 404 elif content_type == 'application/xml': status = 404 elif content_type == 'text/plain': status = 404 elif content_type == 'text/html': status = 404 elif content_type == 'application/octet-stream': status = 2 elif content_type is None: status = 0 else: status = 404 exit(0) if status == 200: size = 0 status = 0 for chunk in response.iter_content(256): size += len(chunk) if size > 4096: print('size: ', size) status = 200 break if status == 0: status = response.status_code canal = Canal(name=name, logo=logo, uri=uri, group_title=group_title, status=status) canal.save() print( '#EXTINF:{}, tvg-id="{}" tvg-name="{}" tvg-logo="{}" group-title="{}",{}\n{}' .format(canal.id, name, name, logo, group_title, name, uri)) else: canal = Canal(name=name, logo=logo, uri=uri, group_title=group_title, status=status) canal.save() print('status: ', status) except requests.exceptions.RequestException as e: print(e) canal = Canal(name=name, logo=logo, uri=uri, group_title=group_title, status=404) canal.save() except DataError as e: print(e) canal = Canal(name=name, logo=logo, uri=uri, group_title=group_title, status=404) canal.save() except Exception as e: print(e) canal = Canal(name=name, logo=logo, uri=uri, group_title=group_title, status=status) canal.save() x += 1
""" pip install m3u8 first, please """ import m3u8 import time import urllib2 url = "http://192.168.7.40:20119/live/cctvnews/encoder/0/playlist.m3u8" media_sequence = 0 playlist = m3u8.load(url) while playlist.is_variant: url = playlist.base_uri + "/" + playlist.playlists[0].uri playlist = m3u8.load(url) current_sequence = media_sequence = playlist.media_sequence for segment in playlist.segments: print current_sequence, segment.uri current_sequence += 1 while True: playlist = m3u8.load(url) if playlist.media_sequence == media_sequence: time.sleep(1) continue media_sequence = playlist.media_sequence target_duration = playlist.target_duration if current_sequence < media_sequence: print "ERROR : missing segment"