def load_live_m3u8(url):
"""
the stream is live stream. so we use sleep to simulate player. but not perfact!
"""
global __lenth__
m = m3u8.load(url)
__lenth__ = now = d = 0
i = 0
m3u8_live_stopper()
while True:
if stop:
print('stopped!!')
raise StopIteration
if i < len(m.segments):
delta = d -( time.time() - now)
if (delta) > 0:
time.sleep(delta)
segurl = m.segments[i].absolute_uri
now = time.time()
d = m.segments[i].duration
i += 1
__lenth__ += 1
yield segurl
else:
i = 0
delta = d -( time.time() - now)
if (delta) > 0:
time.sleep(d - (time.time() - now))
m = m3u8.load(url)
now = time.time()
d = 0
def get_video(link, filename):
"""Send Video link to downloader."""
if link.startswith('https://udemy-adaptive-streaming'):
if os.path.exists(filename): # neccessary when ffmpeg not used
print('{0} already exists'.format(filename))
return
if use_ffmpeg:
check_downloaded(link, filename, 'ffmpeg', do_not_resume=True)
else:
print('\nThis lecture has m3u8 file.\nPlease install "ffmpeg" and use argument "--use-ffmpeg"\nFor better video')
variant_m3u8 = m3u8.load(link)
if variant_m3u8.is_variant:
uri = variant_m3u8.playlists[0].uri
m3u8_obj = m3u8.load(uri)
total_item = len(m3u8_obj.files)
for num, url in enumerate(m3u8_obj.files, 1):
tmp_name = str(num) + '.part'
if os.path.exists(tmp_name): # use for safety
os.remove(tmp_name)
print('\nDownloading {0} of {1} parts:'.format(num, total_item))
check_downloaded(url, tmp_name, downloader, do_not_resume=True)
tmp_list = [str(num) + '.part' for num, _ in enumerate(m3u8_obj.files, 1)]
merge_file(filename, tmp_list)
else:
print("Failed to process m3u8")
else:
check_downloaded(link, filename, downloader)
def __init__(self, url):
self.url = url
try:
playlist = m3u8.load(url)
while playlist.is_variant: # multiple bitrates videos
self.url = playlist.base_uri + playlist.playlists[0].uri
playlist = m3u8.load(self.url)
except:
raise NameError("OpenError")
else:
self.media_seq = playlist.media_sequence
self.current_seq = playlist.media_sequence
for segment in playlist.segments:
print self.current_seq, segment.uri
self.current_seq += 1
def test_raise_timeout_exception_if_timeout_happens_when_loading_from_uri():
try:
obj = m3u8.load(playlists.TIMEOUT_SIMPLE_PLAYLIST_URI, timeout=1)
except:
assert True
else:
assert False
def test_load_should_create_object_from_file_with_relative_segments():
base_uri = os.path.dirname(playlists.RELATIVE_PLAYLIST_FILENAME)
obj = m3u8.load(playlists.RELATIVE_PLAYLIST_FILENAME)
expected_key_abspath = '%s/key.bin' % os.path.dirname(base_uri)
expected_key_path = '../key.bin'
expected_ts1_abspath = '%s/entire1.ts' % base_uri
expected_ts1_path = '/entire1.ts'
expected_ts2_abspath = '%s/entire2.ts' % os.path.dirname(base_uri)
expected_ts2_path = '../entire2.ts'
expected_ts3_abspath = '%s/entire3.ts' % os.path.dirname(os.path.dirname(base_uri))
expected_ts3_path = '../../entire3.ts'
expected_ts4_abspath = '%s/entire4.ts' % base_uri
expected_ts4_path = 'entire4.ts'
assert isinstance(obj, m3u8.M3U8)
assert expected_key_path == obj.key.uri
assert expected_key_abspath == obj.key.absolute_uri
assert expected_ts1_path == obj.segments[0].uri
assert expected_ts1_abspath == obj.segments[0].absolute_uri
assert expected_ts2_path == obj.segments[1].uri
assert expected_ts2_abspath == obj.segments[1].absolute_uri
assert expected_ts3_path == obj.segments[2].uri
assert expected_ts3_abspath == obj.segments[2].absolute_uri
assert expected_ts4_path == obj.segments[3].uri
assert expected_ts4_abspath == obj.segments[3].absolute_uri
def test_load_should_create_object_from_uri_with_relative_segments():
obj = m3u8.load(playlists.RELATIVE_PLAYLIST_URI)
urlparsed = url_parser.urlparse(playlists.RELATIVE_PLAYLIST_URI)
base_uri = os.path.normpath(urlparsed.path + '/..')
prefix = urlparsed.scheme + '://' + urlparsed.netloc
expected_key_abspath = '%s%s/key.bin' % (prefix, os.path.normpath(base_uri + '/..'))
expected_key_path = '../key.bin'
expected_ts1_abspath = '%s/entire1.ts' % (prefix)
expected_ts1_path = '/entire1.ts'
expected_ts2_abspath = '%s%sentire2.ts' % (prefix, os.path.normpath(base_uri + '/..') + '/')
expected_ts2_path = '../entire2.ts'
expected_ts3_abspath = '%s%sentire3.ts' % (prefix, os.path.normpath(base_uri + '/../..'))
expected_ts3_path = '../../entire3.ts'
expected_ts4_abspath = '%s%sentire4.ts' % (prefix, base_uri + '/')
expected_ts4_path = 'entire4.ts'
assert isinstance(obj, m3u8.M3U8)
assert expected_key_path == obj.key.uri
assert expected_key_abspath == obj.key.absolute_uri
assert expected_ts1_path == obj.segments[0].uri
assert expected_ts1_abspath == obj.segments[0].absolute_uri
assert expected_ts2_path == obj.segments[1].uri
assert expected_ts2_abspath == obj.segments[1].absolute_uri
assert expected_ts3_path == obj.segments[2].uri
assert expected_ts3_abspath == obj.segments[2].absolute_uri
assert expected_ts4_path == obj.segments[3].uri
assert expected_ts4_abspath == obj.segments[3].absolute_uri
def update_for_iframes(url):
"""
Returns an updated master playlist and new I-frame playlists
"""
try:
master_playlist = m3u8.load(url)
except IOError:
raise PlaylistLoadError('Invalid url')
if not master_playlist or not master_playlist.is_variant:
raise BadPlaylistError('Not a variant playlist')
master_playlist.iframe_playlists[:] = []
uri = url.split('/')[-1]
result = {'master_uri': uri,
'master_content': None,
'iframe_playlists': []}
for playlist in master_playlist.playlists:
iframe_playlist, data = create_iframe_playlist(playlist)
if iframe_playlist is None or data is None:
continue
master_playlist.add_iframe_playlist(iframe_playlist)
result['iframe_playlists'].append(data)
result['master_content'] = master_playlist.dumps()
return result
def run(self):
segments = m3u8.load(self.playlist_uri).segments
self.segment_duration = segments[0].duration
self.playlist = [str(seg).split("\n")[1] for seg in segments]
self.running = True
self.startup()
self.start_polling_and_fetching_chunks()
def adjustSegmentTimes(self, force=False):
""" Read through the ts files in this segment's directory
and calculate end time.
This is used when restarting video because it died.
"""
if force or not self.endTime:
dir = os.path.join(settings.RECORDED_VIDEO_DIR_BASE, util.getSegmentPath(self.episode.shortName, self.source.name, self.segNumber))
videoChunks = glob("%s/*.ts" % dir)
videoChunks = sorted(videoChunks, key = lambda chunk: int(re.sub(".+prog_index-(\d+).ts", "\\1", chunk)))
if len(videoChunks) > 0:
(mode, ino, dev, nlink, uid, gid, size, atime, mtime, ctime) = os.stat(videoChunks[0])
try:
index = m3u8.load('%s/%s' % (dir, self.indexFileName))
m3u8segment = index.segments[0]
duration = m3u8segment.duration
except:
print "NO INDEX.M3U8 FILE for segment %s %d" % (self.episode.shortName, self.segNumber)
recordingUtil.stopRecordingAndCleanSegments(self.source, videoChunks)
return (None, None)
startTime = mtime - duration
startDT = datetime.fromtimestamp(startTime, pytz.utc)
(mode, ino, dev, nlink, uid, gid, size, atime, mtime, ctime) = os.stat(videoChunks[-1])
endTime = mtime
endDT = datetime.fromtimestamp(endTime, pytz.utc)
print "Segment: Start: %s End: %s" % (startDT, endDT)
self.startTime = startDT
self.endTime = endDT
self.save()
return (self.startTime, self.endTime)
def play(self):
try:
while True:
playlist = m3u8.load(self.url)
if playlist.media_sequence == self.media_seq:
time.sleep(1)
continue
self.media_seq = playlist.media_sequence
target_duration = playlist.target_duration
if self.current_seq < self.media_seq:
print "ERROR : missing segment"
self.current_seq = self.media_seq
index = -1
for segment in playlist.segments:
index += 1
if self.media_seq + index < self.current_seq:
continue
seg_url = "%s%s" % (playlist.base_uri, segment.uri)
response = urllib2.urlopen(seg_url)
buf = response.read()
if not os.path.isdir(os.path.dirname(segment.uri)):
os.mkdir(os.path.dirname(segment.uri))
f = open(segment.uri, "w")
f.write(buf)
f.close
print "index: ", self.media_seq + index, ", uri: ", segment.uri, ", size ", len(buf)
self.current_seq += 1
time.sleep(target_duration - 1)
except:
print "Error load m3u8 playlist: %s" % self.url
def updateIndexFilePrefix(indexFileSuffix, subst, flightName):
""" This is truncating the last n rows from the m3u8 file and reappending the end and the metadata at the top.
This fakes our delay
"""
indexFilePath = settings.DATA_ROOT + indexFileSuffix
segmentDirectoryUrl = settings.DATA_URL + os.path.dirname(indexFileSuffix)
try:
videoDelayInSecs = getClassByName(settings.XGDS_VIDEO_DELAY_AMOUNT_METHOD)(flightName)
if videoDelayInSecs > 0:
(videoDelayInSegments, m3u8_index) = getSegmentsFromEndForDelay(videoDelayInSecs-30,
indexFilePath)
else:
m3u8_index = m3u8.load(indexFilePath)
videoDelayInSegments = 0
segments = m3u8_index.segments
if len(segments) > 0:
if segments[0].duration > 100:
del segments[0]
if videoDelayInSegments > 0 and len(segments) > videoDelayInSegments:
del segments[-videoDelayInSegments:]
for s in segments:
s.uri = str(segmentDirectoryUrl) + '/' + s.uri
return m3u8_index.dumps()
except:
traceback.print_exc()
traceback.print_stack()
return segmentDirectoryUrl
def check_streams(variant_streams, base_uri, verbose):
result_msg = "BaseURI="
result_msg += base_uri
result_msg += " >> "
result_code = 0
for stream in variant_streams:
result = ""
stream_url = ""
stream_url = base_uri
stream_url += stream.uri
try:
stream_playlist = m3u8.load(stream_url)
result = "OK"
result_msg += stream.uri
result_msg += ":"
result_msg += result
result_msg += ", "
except IOError as error:
result = str(error)
result_code = 2
result_msg += stream.uri
result_msg += ":"
result_msg += result
result_msg += ", "
# print stream_url
if verbose:
print "\t%s: %s" % (stream.uri, result)
return (result_code, result_msg[:-2])
def get_publishpoint_streams(self, video_id, stream_type=None, game_type=None):
"""Return the URL for a stream."""
streams = {}
self.get_current_season_and_week() # set cookies
url = self.servlets_url + '/publishpoint'
if video_id == 'nfl_network':
post_data = {'id': '1', 'type': 'channel', 'nt': '1'}
elif video_id == 'redzone':
post_data = {'id': '2', 'type': 'channel', 'nt': '1'}
elif stream_type == 'game':
post_data = {'id': video_id, 'type': stream_type, 'nt': '1', 'gt': game_type}
else:
post_data = {'id': video_id, 'type': stream_type, 'nt': '1'}
headers = {'User-Agent': 'iPad'}
m3u8_data = self.make_request(url=url, method='post', payload=post_data, headers=headers)
m3u8_dict = xmltodict.parse(m3u8_data)['result']
self.log('NFL Dict %s' %m3u8_dict)
m3u8_url = m3u8_dict['path'].replace('_ipad', '')
temp_path, temp_parm = m3u8_url.split('?', 1)
m3u8_header = {'Cookie' : 'nlqptid=' + temp_parm, 'User-Agent' : 'Safari/537.36 Mozilla/5.0 AppleWebKit/537.36 Chrome/31.0.1650.57', 'Accept-encoding' : 'identity', 'Connection' : 'keep-alive'}
m3u8_obj = m3u8.load(m3u8_url)
if m3u8_obj.is_variant: # if this m3u8 contains links to other m3u8s
for playlist in m3u8_obj.playlists:
bitrate = str(int(playlist.stream_info.bandwidth[:playlist.stream_info.bandwidth.find(' ')])/100)
streams[bitrate] = m3u8_url[:m3u8_url.rfind('/') + 1] + playlist.uri + '?' + m3u8_url.split('?')[1] + '|' + urllib.urlencode(m3u8_header)
else:
streams['only available'] = m3u8_url
return streams
def get_publishpoint_streams(self, video_id, stream_type=None, game_type=None):
"""Return the URL of a live stream."""
streams = {}
self.get_current_season_and_week() # set cookies
url = self.servlets_url + '/publishpoint'
if video_id == 'nfl_network':
post_data = {'id': '1', 'type': 'channel', 'nt': '1'}
elif video_id == 'redzone':
post_data = {'id': '2', 'type': 'channel', 'nt': '1'}
elif stream_type == 'game':
post_data = {'id': video_id, 'type': stream_type, 'nt': '1', 'gt': game_type}
else:
post_data = {'id': video_id, 'type': stream_type, 'nt': '1'}
headers = {'User-Agent': 'iPad'}
m3u8_data = self.make_request(url=url, method='post', payload=post_data, headers=headers)
m3u8_dict = xmltodict.parse(m3u8_data)['result']
self.log('NFL Dict %s' %m3u8_dict)
m3u8_url = m3u8_dict['path'].replace('adaptive://', 'http://')
m3u8_url = m3u8_dict['path'].replace('_ipad', '')
m3u8_obj = m3u8.load(m3u8_url)
if m3u8_obj.is_variant: # if this m3u8 contains links to other m3u8s
for playlist in m3u8_obj.playlists:
bitrate = str(int(playlist.stream_info.bandwidth[:playlist.stream_info.bandwidth.find(' ')])/100)
streams[bitrate] = m3u8_url[:m3u8_url.rfind('/') + 1] + playlist.uri + '?' + m3u8_url.split('?')[1]
else:
streams['only available'] = m3u8_url
return streams
def parse_master_playlist(uri):
master_url = uri
playlists = []
iframe_playlists = []
media_lists = []
# m3u8_content = bytes.decode(is_m3u8_file(uri))
# variant_m3u8 = m3u8.loads(m3u8_content)
if is_m3u8_file(uri):
variant_m3u8 = m3u8.load(uri)
variant_m3u8_base_uri = variant_m3u8.base_uri
print("Master_m3u8_base_uri:" + variant_m3u8_base_uri + '\n')
print("Start to analyse the Master M3U8 file:")
if variant_m3u8.is_variant:
print('It\'s a variant m3u8.')
playlist_count = 0
playlists = []
for playlist in variant_m3u8.playlists:
playlist_uri = playlist.absolute_uri
playlist_path = playlist.base_path
playlist_tuple = (playlist_uri, playlist_path)
if playlist_tuple not in playlists:
playlists.append(playlist_tuple)
playlist_count += 1
print("There are " + str(len(playlists)) + " play lists.")
print(playlists)
else:
print("It\'s not a variant m3u8.\n")
return master_url
if variant_m3u8.iframe_playlists:
iframe_playlist_count = 0
for iframe_playlist in variant_m3u8.iframe_playlists:
iframe_playlist_count += 1
iframe_playlist_uri = iframe_playlist.absolute_uri
iframe_playlist_path = iframe_playlist.base_path
iframe_playlists.append((iframe_playlist_uri, iframe_playlist_path))
print("There are " + str(iframe_playlist_count) + " iframe playlists:")
print(iframe_playlists)
else:
print("There is no iframe playlist.")
if variant_m3u8.media:
media_list_count = 0
for media_list in variant_m3u8.media:
if media_list.uri:
media_list_uri = media_list.absolute_uri
media_list_path = media_list.base_path
media_list_tuple = (media_list_uri, media_list_path)
if media_list_tuple not in media_lists:
media_lists.append(media_list_tuple)
media_list_count += 1
print("There are " + str(media_list_count) + " media lists:")
print(media_lists)
else:
print("There is no media in m3u8.")
return playlists, iframe_playlists, media_lists
def parse_m3u8_novar(url, obj=None):
"""Parse m3u8 without variable bitrates and return durations and URLs.
The return value is a (float, str) tuple of duration and URL.
"""
m3u8_obj = obj if obj else m3u8.load(url)
return [(seg.duration, m3u8_obj.base_uri + seg.uri)
for seg in m3u8_obj.segments]
def get_urls_from_m3u8(url):
playlist = m3u8.load(url)
result = []
if playlist.is_variant:
for item in playlist.playlists:
result += [(item.base_uri + item.uri, item.stream_info.bandwidth)]
result = sorted(result, key = lambda x: x[1])
return result
def load_playlist(m3u8_uri):
'''load m3u8 URI and extract playlists '''
logger.debug('Load playlist %s' % m3u8_uri)
playlist = m3u8.load(m3u8_uri)
if playlist.is_variant:
return digest_variant_playlist(playlist, m3u8_uri)
else:
return digest_singel_playlist(playlist, m3u8_uri)
def get_playlist(playlist_url, fmt): m3u8_obj = m3u8.load(playlist_url) for playlist in m3u8_obj.playlists: if str(playlist.stream_info.resolution).find(fmt) > 0: return playlist.uri return None
def main():
"""
Simple command-line program for checking the stream bandwidths to ensure they are what is expected.
"""
args = get_args()
if args.verbose:
args.timestamp = True
ref_bandwidths = args.bandwidths.split()
urls = get_urls(args)
result = (0, "OK")
result_code = 0
if len(urls) < 1:
print >> sys.stderr, render_date_iso8601(), "No valid URLs to process."
exit(2)
for url in urls:
if verify_url(url) == True:
try:
m3u8_obj = m3u8.load(url)
if m3u8_obj.is_variant:
if args.verbose:
print >> sys.stdout, "Checking URL:", url
if args.variance_percent:
result = check_variance_bandwidths(m3u8_obj.playlists, ref_bandwidths, args.variance_percent, args.verbose, args.unordered)
else:
if args.unordered:
result = check_unordered_bandwidths(m3u8_obj.playlists, ref_bandwidths, args.verbose)
else:
result = check_ordered_bandwidths(m3u8_obj.playlists, ref_bandwidths, args.verbose)
result_code = set_return_code(result_code, result[0])
if result[0] == 0:
print_brief(args.timestamp, sys.stdout, "OK: URL=", url)
elif result[0] == 1:
print_brief(args.timestamp, sys.stdout, "WARNING: URL=", url, ">>", result[1][:-1])
else:
print_brief(args.timestamp, sys.stdout, "CRITICAL: URL=", url, ">>", result[1][:-1])
else:
print_brief(args.timestamp, sys.stdout, "CRITICAL: URL=", url, ">> Does not contain any variant playlists")
result_code = 2
except IOError as error:
print_brief(args.timestamp, sys.stdout, "CRITICAL: URL=", url, ">>", error)
result_code = 2
else:
print_brief(args.timestamp, sys.stdout, "CRITICAL: URL=", url, ">> Not a valid URL")
result_code = 2
if result_code != 0:
exit(result_code)
return 0
def load_m3u8_playlist(url):
stream_types = []
streams = {}
m = m3u8.load(url).playlists
for l in m:
stream_types.append(str(l.stream_info.bandwidth))
streams[str(l.stream_info.bandwidth)] = {'container': 'm3u8', 'video_profile': str(l.stream_info.bandwidth), 'src' : [l.absolute_uri], 'size': 0}
stream_types.sort()
return stream_types, streams
def list_formats(url): html = open_url(url) parsed_html = BeautifulSoup(html) playlist_url = get_playlist_url(parsed_html) m3u8_obj = m3u8.load(playlist_url) for playlist in m3u8_obj.playlists: print playlist.stream_info.resolution
def find_best_video(uri):
playlist = m3u8.load(uri)
if not playlist.is_variant:
return playlist
best_stream = playlist.playlists[0]
for stream in playlist.playlists:
if stream.stream_info.bandwidth == 'max' or stream.stream_info.bandwidth > best_stream.stream_info.bandwidth:
best_stream = stream
return find_best_video(best_stream.absolute_uri)
def test_create_iframe_playlist(self):
iframe_playlist_uri = 'bigbuckbunny-400k-iframes.m3u8'
iframe_playlist_content = IFRAME_PLAYLIST_400K
master_playlist = m3u8.load(
SAMPLES_PATH + 'original_video/bigbuckbunny.m3u8'
)
_, results = create_iframe_playlist(master_playlist.playlists[0])
self.assertEqual(iframe_playlist_uri, results['uri'])
self.assertEqual(iframe_playlist_content, results['content'])
def get_vod(self, vod_id):
urls = []
token, sig = self.get_token_and_signature(vod_id)
quality_playlist_url = USHER_API_VOD.format(vod_id=vod_id, sig=sig, token=token)
variant_playlist = m3u8.load(quality_playlist_url)
for p in variant_playlist.playlists:
quality = p.media[0].name
uri = p.uri
urls.append({'url': uri, 'quality': quality})
return urls
def __init__(self, grabber, filename, pid):
super(Program, self).__init__()
self.pid = pid
self.title = filename
self.grabber = grabber
self.filename = os.path.basename(filename)
self.m3 = m3u8.load(filename)
def analitzaSubManifestSilent(m, test_ts):
u=None
try:
if m.uri.startswith('http'):
url = m.uri
else:
url = m.base_uri + "/" + m.uri
u = m3u8.load(url)
except urllib2.HTTPError, e:
print " ERROR: Codi HTTP " + str(e.code) + ": " + url
def take_chunks(self):
token_url = (TwitchAPI.API_DOMAIN +
TwitchAPI.API +
'/vods/' + self.vod_id + '/access_token')
with urllib.request.urlopen(token_url) as response:
token_json = json.loads(response.read().decode('utf-8'))
quality_playlist_url = (TwitchAPI.USHER_DOMAIN +
'/vod/' + self.vod_id +
'?nauthsig=' + token_json['sig'] +
'&nauth=' + token_json['token'] +
'&allow_source=true')
variant_playlist = m3u8.load(quality_playlist_url)
try:
source_playlist_url = next(i.uri for i in variant_playlist.playlists if i.media[0].group_id == 'chunked')
except StopIteration:
print('Source(Chunked) playlist have not found.')
sys.exit(0)
chunks_m3u8 = m3u8.load(source_playlist_url)
chunks_rel_path = chunks_m3u8.segments.uri
self.chunks = list(map(lambda x: urljoin(source_playlist_url, x), chunks_rel_path))
def _load_media_playlist(media_playlist_url):
for retry in range(0, 5):
try:
return m3u8.load(media_playlist_url)
except _RETRIABLE_EXCEPTIONS:
time.sleep((2 ** retry) + (random.randint(0, 1000) / 1000))
logging.warning('Retrying to load: ' + media_playlist_url)
except urllib2.HTTPError as e:
if e.code == 404:
time.sleep((2 ** retry) + (random.randint(0, 1000) / 1000))
logging.warning('Retrying to load: ' + media_playlist_url)
def getSegmentsFromEndForDelay(delayTime, indexPath):
index = m3u8.load(indexPath)
segList = index.segments
segCount = 0
totalTime = 0
for s in reversed(segList):
totalTime += s.duration
segCount += 1
if totalTime >= delayTime:
break
return segCount, index
def test_load_should_create_object_from_uri():
obj = m3u8.load(playlists.SIMPLE_PLAYLIST_URI)
assert isinstance(obj, m3u8.M3U8)
assert 5220 == obj.target_duration
assert 'http://media.example.com/entire.ts' == obj.segments[0].uri
# Fetch vod index
url = _index_api_url.format(video_id)
payload = {
'nauth': data['token'],
'nauthsig': data['sig'],
'allow_source': True,
'allow_spectre': False,
"player": "twitchweb",
"p": int(random() * 999999),
"allow_audio_only": True,
"type": "any"
}
r = requests.get(url, params=payload, headers=common_headers)
m = m3u8.loads(r.content)
index_url = m.playlists[0].uri
index = m3u8.load(index_url)
# Get the piece we need
position = 0
chunks = []
for seg in index.segments:
# Add duration of current segment
position += seg.duration
# Check if we have gotten to the start of the clip
if position < int(app.pargs.start):
continue
# Extract clip name and byte range
p = re.match(_chunk_re, seg.absolute_uri)
def get_m3u8(self):
self.m3u8 = m3u8.load(self.m3u8_url)
def generate():
try:
played_chunk_urls = []
while self.tuner.tuner_lock.locked():
playlist = m3u8.load(channelUri)
segments = playlist.segments
if len(played_chunk_urls):
newsegments = 0
for segment in segments:
if segment.absolute_uri not in played_chunk_urls:
newsegments += 1
self.fhdhr.logger.info(
"Refreshing m3u8, Loaded %s new segments." %
str(newsegments))
else:
self.fhdhr.logger.info("Loaded %s segments." %
str(len(segments)))
if playlist.keys != [None]:
keys = [{
"url": key.uri,
"method": key.method,
"iv": key.iv
} for key in playlist.keys if key]
else:
keys = [None for i in range(0, len(segments))]
for segment, key in zip(segments, keys):
chunkurl = segment.absolute_uri
if chunkurl not in played_chunk_urls:
played_chunk_urls.append(chunkurl)
if (not self.stream_args["duration"] == 0
and not time.time() <
self.stream_args["time_end"]):
self.fhdhr.logger.info(
"Requested Duration Expired.")
self.tuner.close()
chunk = self.fhdhr.web.session.get(
chunkurl).content
if not chunk:
break
# raise TunerError("807 - No Video Data")
if key:
keyfile = self.fhdhr.web.session.get(
key["url"]).content
cryptor = AES.new(keyfile, AES.MODE_CBC,
keyfile)
chunk = cryptor.decrypt(chunk)
self.fhdhr.logger.info(
"Passing Through Chunk: %s" % chunkurl)
yield chunk
if playlist.target_duration:
time.sleep(int(playlist.target_duration))
self.fhdhr.logger.info(
"Connection Closed: Tuner Lock Removed")
except GeneratorExit:
self.fhdhr.logger.info("Connection Closed.")
except Exception as e:
self.fhdhr.logger.info("Connection Closed: " + str(e))
finally:
self.tuner.close()
print "Got flight %s(%s)" % (flightObj.name, flightObj.uuid)
print "Segment path: %s/%s/Video/Recordings/Segment*" % (FLIGHT_BASE,
opts.flight)
segmentDirs = glob("%s/%s/Video/Recordings/Segment*" %
(FLIGHT_BASE, opts.flight))
segmentDirs = sorted(segmentDirs)
print "Found %d Segment directories" % len(segmentDirs)
for i, dir in enumerate(segmentDirs):
videoChunks = glob("%s/*.ts" % dir)
videoChunks = sorted(
videoChunks,
key=lambda chunk: int(re.sub(".+prog_index-(\d+).ts", "\\1", chunk)))
if len(videoChunks) > 0:
(mode, ino, dev, nlink, uid, gid, size, atime, mtime,
ctime) = os.stat(videoChunks[0])
index = m3u8.load('%s/%s' % (dir, 'prog_index.m3u8'))
m3u8segment = index.segments[0]
duration = m3u8segment.duration
# print "duration %f " % duration
# print "mtime %d " % mtime
# print "mdt %s " % datetime.fromtimestamp(mtime, pytz.utc)
startTime = float(mtime) - duration
startDT = datetime.fromtimestamp(startTime, pytz.utc)
(mode, ino, dev, nlink, uid, gid, size, atime, mtime,
ctime) = os.stat(videoChunks[-1])
endTime = mtime
endDT = datetime.fromtimestamp(endTime, pytz.utc)
print "Segment%03d: Start: %s End: %s" % (i, startDT, endDT)
def find_m3u8_playlist(itunes_url: str, user_agent: str = None) -> MovieData:
"""
Scrape an iTunes page to find the URL of the M3U8 playlist.
Args:
itunes_url (str): URL of an iTunes movie page to scrape.
user_agent (str, optional): User-Agent string to use for scraping. Defaults to None.
Raises:
InvalidURL: An inavlid iTunes URL was provided.
ConnectionError: A connection error occurred while trying to request the page.
HTTPError: An error while trying to download m3u8 playlist data.
PageLoadError: The page did not load properly.
Returns:
MovieData: A MovieData (NamedTuple) object with movie's name, and an M3U8 object of the playlist
if the playlist is found. None otherwise.
"""
# Check whether URL is valid
if re.match(ITUNES_STORE_REGEX, itunes_url) is None:
raise InvalidURL(f"{itunes_url} is not a valid iTunes movie URL.")
site_page: BeautifulSoup = BeautifulSoup(session().get(itunes_url, headers={"User-Agent": user_agent}).text, "lxml")
movie_metadata: Union[Tag, NavigableString, None] = site_page.find("script", attrs={"name": "schema:movie", "type": 'application/ld+json'})
if not isinstance(movie_metadata, Tag):
raise PageLoadError("The page did not load properly.")
# Convert to dictionary structure
movie_metadata_dict: dict = json.loads(str(movie_metadata.contents[0]).strip())
media_type: str = movie_metadata_dict['@type']
movie_title: str = html.unescape(movie_metadata_dict['name'])
if media_type != "Movie":
raise InvalidURL("The provided iTunes URL is not for a movie.")
# Scrape a dictionary on the webpage for playlists data
playlists_data_tag: Union[Tag, NavigableString, None] = site_page.find("script", attrs={"id": "shoebox-ember-data-store", "type": "fastboot/shoebox"})
# fastboot/shoebox data could not be found
if not isinstance(playlists_data_tag, Tag):
raise PageLoadError("fastboot/shoebox data could not be found.")
# Convert to dictionary structure
playlists_data: dict[str, dict] = json.loads(str(playlists_data_tag.contents[0]).strip())
# Loop safely over different structures to find a matching playlist
for key in playlists_data.keys():
if isinstance(playlists_data[key].get("included"), list):
for item in playlists_data[key]["included"]:
if (isinstance(item.get("type"), str) and item["type"] == "offer" and
isinstance(item.get("attributes"), dict) and
isinstance(item["attributes"].get("assets"), list) and
len(item["attributes"]["assets"]) > 0 and
isinstance(item["attributes"]["assets"][0], dict) and
isinstance(item["attributes"]["assets"][0].get("hlsUrl"), str)):
m3u8_url: str = item["attributes"]["assets"][0]["hlsUrl"]
try:
playlist: M3U8 = m3u8.load(m3u8_url)
# If m3u8 playlist is invalid, skip it
except ValueError:
continue
except HTTPError:
continue
# Assure playlist is for the correct movie
if iSubRip.is_playlist_valid(playlist, movie_title):
return MovieData(movie_title, playlist)
return MovieData(movie_title, None)
def get(self, path):
user_id = self.request.get('user_id')
if not user_id:
raise ValueError("Request param 'user_id' is required.")
# For ease of testing we let the world access it
self.response.headers[_CORS_ORIGIN_HEADER] = '*'
media_url = urlparse.urljoin(_HLS_MEDIA_BASE_URL, path)
if path.endswith(_HLS_PLAYLIST_EXTENSION):
playlist_m3u8 = m3u8.load(media_url)
if playlist_m3u8.is_variant:
self.response.out.write(playlist_m3u8.dumps())
else:
# Process lookahead segments in the consumed playlist and return
# the ad info if one was choosen for the current user in session.
ad_info = self._process_lookahead_segments(
media_url, user_id, playlist=playlist_m3u8)
logging.debug("Processed lookahead segments in %s." %
media_url)
# If an ad is returned for the consumed playlist, we also
# process lookahead segments in the alternative playlists.
if ad_info is not None:
master_playlist = m3u8.load(
self._get_master_playlist_url(media_url))
for playlist in master_playlist.playlists:
# Current playlist is already processed
if playlist.absolute_uri != media_url:
self._process_lookahead_segments(
playlist.absolute_uri,
user_id,
ad_info=ad_info)
logging.debug(
"Processed lookahead segments in %s." %
playlist.absolute_uri)
# We pass user ID down to the segments
for segment in playlist_m3u8.segments:
segment.uri = self._update_query_string(segment.uri,
user_id=user_id)
# After we update all the segment URIs
playlist_content = playlist_m3u8.dumps()
for segment in playlist_m3u8.segments:
if segment.cue_out:
segment_path = self._get_segment_path(segment)
# Get the task name from a previous HlsHandler request
task_name = memcache.get(segment_path,
namespace=user_id)
if task_name is not None:
hls_cue_out_start = playlist_content.find(
_HLS_EXT_CUE_OUT + "\n")
hls_cue_out_end = hls_cue_out_start + len(
_HLS_EXT_CUE_OUT + "\n")
vast_url = self.uri_for('vast',
task_name=task_name,
_full=True)
# VAST URL tag with the format of '#EXT-X-VAST-URL:<vast_url>'
hls_ext_vast_url_tag = "%s:%s" % (
_HLS_EXT_VAST_URL, vast_url)
# M3U8 Python library has no support for our custom tag,
# and it also doesn't let us add comments. So we instead
# do a string manipulation to populate the VAST URL tag.
playlist_content = (
playlist_content[:hls_cue_out_end] +
hls_ext_vast_url_tag + "\n" +
playlist_content[hls_cue_out_end:])
self.response.out.write(playlist_content)
else:
# Get the task name from a previous HlsHandler request
task_name = memcache.get(path, namespace=user_id)
if task_name is not None:
# Get the injected ad segment URL from InjectionWorker if
# available, otherwise redirect to the original media URL
redirect_dict = memcache.get(task_name) or {}
redirect_url = redirect_dict.get(path, media_url)
else:
# We redirect to the original media URL (no ad injection)
redirect_url = media_url
self.redirect(redirect_url)
def _process_lookahead_segments(self,
playlist_url,
user_id,
playlist=None,
ad_info=None):
if playlist is None:
playlist = m3u8.load(playlist_url)
# Index of the first and the oldest lookahead segment
lookahead_start = self._find_lookahead_start(playlist)
# Iterate until there are no lookahead segments
while lookahead_start < len(playlist.segments):
# Process the lookahead segments backwards
segment_index = len(playlist.segments) - 1
segment = playlist.segments[segment_index]
if segment.cue_out:
ad_start_path = self._get_segment_path(segment)
# Get the task name from a previous HlsHandler request
task_name = memcache.get(ad_start_path, namespace=user_id)
# Skip if the task is already added to the queue
if task_name is None:
if ad_info is None:
# Get the ad source and the ad duration pair from the Studio UI
ad_info_args = memcache.get(segment.absolute_uri,
namespace=user_id)
if ad_info_args is not None:
ad_info = AdInfo(*ad_info_args)
# Request ad from DFP if one is not specified
if ad_info is None:
dfp_response = self._request_ad_from_dfp(
self._get_master_playlist_path(playlist_url))
self._copy_cookies(dfp_response, self.response)
ad_elem = etree.fromstring(dfp_response.content)
if ad_elem is not None:
ad_info = self._get_ad_info(ad_elem)
# DFP may also return no ads, so we have to check for it
if ad_info is not None:
task_name = self._generate_task_name(segment, ad_info)
ad_start_seq = playlist.media_sequence + segment_index
ad_segment_path = ad_start_path
ad_segment_seq = ad_start_seq
while ad_info.duration > timedelta(seconds=0):
# Pass the task name to the subsequent HlsHandler requests
memcache.set(ad_segment_path,
task_name,
time=_TTL_1_HOUR,
namespace=user_id)
ad_segment_path = ad_segment_path.replace(
str(ad_segment_seq), str(ad_segment_seq + 1))
ad_segment_seq += 1
ad_info.duration -= timedelta(
seconds=playlist.target_duration)
try:
taskqueue.add(queue_name=_TASKQUEUE_NAME,
name=task_name,
url=_TASKQUEUE_URL,
params={
'playlist_url': playlist_url,
'ad_source': ad_info.source,
'ad_start_seq': ad_start_seq,
'ad_start_time': _AD_START_TIME
})
except (taskqueue.TaskAlreadyExistsError,
taskqueue.TombstonedTaskError) as e:
logging.debug("Task '%s' already queued." %
task_name)
finally:
if ad_info.vast is not None:
vast_cache_key = _VAST_CACHE_KEY % task_name
vast_content = etree.tostring(ad_info.vast)
memcache.set(vast_cache_key,
vast_content,
time=_TTL_1_HOUR)
# Drop the last segment
playlist.segments.pop()
return ad_info
def download_news_video_and_content(news_id,
base_dir,
chunk_size=32 * 1024,
video_dir="video",
asset_dir="assets",
audio_dir="audio"):
video_dir = os.path.join(base_dir, video_dir)
asset_dir = os.path.join(base_dir, asset_dir)
audio_dir = os.path.join(base_dir, audio_dir)
makedirs(video_dir)
makedirs(asset_dir)
makedirs(audio_dir)
text_path = os.path.join(asset_dir, "{}.txt".format(news_id))
original_text_path = os.path.join(asset_dir,
"original-{}.txt".format(news_id))
video_path = os.path.join(video_dir, "{}.ts".format(news_id))
audio_path = os.path.join(audio_dir, "{}.wav".format(news_id))
params = {
'NJC': 'NJC400',
'NID': news_id, # NB11515152
'CD': 'A0100',
}
response = requests.request(
method='GET',
url=BASE_URL,
params=params,
)
soup = soupify(response.text)
article_contents = soup.find_all('article_contents')
assert len(article_contents) == 1, \
"# of <article_contents> of {} should be 1: {}".format(news_id, response.text)
text = soupify(article_contents[0].text).get_text() # remove <div>
with open(original_text_path, "w", -1, "utf-8") as f:
f.write(text)
with open(text_path, "w", -1, "utf-8") as f:
from nltk import sent_tokenize
text = re.sub(r'\[.{0,80} :\s.+]', '', text) # remove quote
text = re.sub(r'☞.+http.+\)', '', text) # remove quote
text = re.sub(r'\(https?:\/\/.*[\r\n]*\)', '', text) # remove url
sentences = sent_tokenize(text)
sentences = [
sent for sentence in sentences for sent in sentence.split('\n')
if sent
]
new_texts = []
for sent in sentences:
sent = sent.strip()
sent = re.sub(r'\([^)]*\)', '', sent)
#sent = re.sub(r'\<.{0,80}\>', '', sent)
sent = sent.replace('…', '.')
new_texts.append(sent)
f.write("\n".join([sent for sent in new_texts if sent]))
vod_paths = soup.find_all('vod_path')
assert len(vod_paths) == 1, \
"# of <vod_path> of {} should be 1: {}".format(news_id, response.text)
if not os.path.exists(video_path):
redirect_url = soup.find_all('vod_path')[0].text
list_url = m3u8.load(redirect_url).playlists[0].absolute_uri
video_urls = [
segment.absolute_uri for segment in m3u8.load(list_url).segments
]
with open(video_path, "wb") as f:
for url in video_urls:
response = requests.get(url, stream=True)
total_size = int(response.headers.get('content-length', 0))
for chunk in response.iter_content(chunk_size):
if chunk: # filter out keep-alive new chunks
f.write(chunk)
if not os.path.exists(audio_path):
encoder = get_encoder_name()
command = "{} -y -loglevel panic -i {} -ab 160k -ac 2 -ar 44100 -vn {}".\
format(encoder, video_path, audio_path)
subprocess.call(command, shell=True)
return True
def get_next_clip(self, current_clip_name=None):
"""
"""
# Get current folder
current_folder = int(self.valid_folders[self.current_folder_index])
clipname, clip_start_time = datetime_utils.get_clip_name_from_unix_time(
self.folder_name.replace("_", "-"), self.current_clip_start_time)
# if real_time execution mode is specified
if self.real_time:
# sleep till enough time has elapsed
now = datetime.utcnow()
time_to_sleep = (current_clip_name - now).total_seconds()
if time_to_sleep < 0:
print("Issue with timing")
if time_to_sleep > 0:
time.sleep(time_to_sleep)
# read in current m3u8 file
# stream_url for the current AWS folder
stream_url = "{}/hls/{}/live.m3u8".format((self.stream_base),
(current_folder))
stream_obj = m3u8.load(stream_url)
num_total_segments = len(stream_obj.segments)
num_segments_in_wav_duration = math.ceil(
self.polling_interval_in_seconds / stream_obj.target_duration)
# calculate the start index by computing the current time - start of current folder
segment_start_index = math.ceil(
datetime_utils.get_difference_between_times_in_seconds(
self.current_clip_start_time, current_folder) /
stream_obj.target_duration)
segment_end_index = segment_start_index + num_segments_in_wav_duration
if segment_end_index > num_total_segments:
# move to the next folder and increment the current_clip_start_time to the new
self.current_folder_index += 1
self.current_clip_start_time = self.valid_folders[
self.current_folder_index]
return None, None, None
# Can get the whole segment so update the clip_start_time for the next clip
# We do this before we actually do the pulling in case there is a problem with this clip
self.current_clip_start_time = datetime_utils.add_interval_to_unix_time(
self.current_clip_start_time, self.polling_interval_in_seconds)
# Create tmp path to hold .ts segments
tmp_path = "tmp_path"
os.makedirs(tmp_path, exist_ok=True)
file_names = []
for i in range(segment_start_index, segment_end_index):
audio_segment = stream_obj.segments[i]
base_path = audio_segment.base_uri
file_name = audio_segment.uri
audio_url = base_path + file_name
try:
scraper.download_from_url(audio_url, tmp_path)
file_names.append(file_name)
except Exception:
print("Skipping", audio_url, ": error.")
# concatentate all .ts files with ffmpeg
hls_file = (clipname + ".ts")
audio_file = (clipname + ".wav")
wav_file_path = os.path.join(self.wav_dir, audio_file)
filenames_str = " ".join(file_names)
concat_ts_cmd = "cd {tp} && cat {fstr} > {hls_file}".format(
tp=tmp_path, fstr=filenames_str, hls_file=hls_file)
os.system(concat_ts_cmd)
# read the concatenated .ts and write to wav
stream = ffmpeg.input(os.path.join(tmp_path, Path(hls_file)))
stream = ffmpeg.output(stream, wav_file_path)
ffmpeg.run(stream, quiet=False)
# clear the tmp_path
os.system(f'rm -rf {tmp_path}')
# If we're in demo mode, we need to fake timestamps to make it seem like the date range is real-time
if current_clip_name:
clipname, _ = get_readable_clipname(
self.folder_name.replace("_", "-"), current_clip_name)
# rename wav file
full_new_clip_path = os.path.join(self.wav_dir, clipname + ".wav")
os.rename(wav_file_path, full_new_clip_path)
wav_file_path = full_new_clip_path
# change clip_start_time - this has to be in UTC so that the email can be in PDT
clip_start_time = current_clip_name.isoformat() + "Z"
# Get new index
return wav_file_path, clip_start_time, current_clip_name
def verify_playlist_link(url, timeout, indent=1, check_first_N_only=5):
'''
'''
nice_print('Loading playlist: {0}'.format(url), indent=indent, debug=True)
if indent > 6:
nice_print('ERROR nested playlist too deep', indent=indent)
return False
# one nasty server 301-redirected the .m3u8 file to a .mp3 file, ouch. This catches that:
try:
m3u8_head = requests.head(url,
timeout=(timeout, timeout),
allow_redirects=False)
if 300 <= m3u8_head.status_code < 310:
m3u8_head2 = requests.head(url,
timeout=(timeout, timeout),
allow_redirects=True)
url_redirected = m3u8_head2.history[0].headers['Location']
extension = urlparse(url_redirected).path.split(".")[-1]
if extension not in ("m3u8", "m3u"):
nice_print(
'ERROR m3u8-playlist 30x-redirected to "{0}"-filetype. Skipping this.'
.format(extension),
indent=indent,
debug=False)
return False
except Exception as e:
nice_print('ERROR loading redirected playlist: {0}'.format(
str(e)[:100]),
indent=indent,
debug=True)
return False
try:
m3u8_obj = m3u8.load(url, timeout=timeout)
except Exception as e:
nice_print('ERROR loading playlist: {0}'.format(str(e)[:100]),
indent=indent,
debug=True)
return False
if 0 == len(m3u8_obj.data['playlists']) + len(m3u8_obj.data['segments']):
nice_print('ERROR: playlist is empty.', indent=indent, debug=True)
return False
for nested_playlist in m3u8_obj.data['playlists']:
if nested_playlist['uri'].startswith(('https://', 'http://')):
nested_url = nested_playlist['uri']
else:
nested_url = '{0}{1}'.format(m3u8_obj.base_uri,
nested_playlist['uri'])
return verify_playlist_link(nested_url,
timeout=timeout,
indent=indent + 1)
counter = 0
for segment in m3u8_obj.data['segments']:
if segment['uri'].startswith(('https://', 'http://')):
url = segment['uri']
else:
url = '{0}{1}'.format(m3u8_obj.base_uri, segment['uri'])
if not verify_video_link(url, timeout=timeout):
return False # first occurring bad one declares this whole list bad. Is that a good choice?
counter += 1
if counter >= check_first_N_only:
msg = 'OK: skipping tests of remaining {0} entries because we have {1} good files already in this playlist'
nice_print(msg.format(
len(m3u8_obj.data['segments']) - counter, counter),
indent=indent,
debug=True)
return True
return True
def Get_ts_Playlist(m3u8_URL):
m3u8_obj = m3u8.load(m3u8_URL)
playlist = [el['uri'] for el in m3u8_obj.data['segments']]
return playlist
def loadM3u8(url):
playlist = m3u8.load(url)
for i in range(len(playlist.segments)):
playlist.segments[i].uri=playlist.segments[i].absolute_uri
return playlist.dumps()
def test_load_should_remember_redirect():
obj = m3u8.load(playlists.REDIRECT_PLAYLIST_URI)
urlparsed = url_parser.urlparse(playlists.SIMPLE_PLAYLIST_URI)
assert urlparsed.scheme + '://' + urlparsed.netloc + "/" == obj.base_uri
import m3u8
_path = input('Enter m3u8 path:')
playlist = m3u8.load(_path)
# playlist = m3u8.load('https://dplusindia-google.prod-vod.h264.io/d6e4176a-8384-4b8f-98f1-9688721421c4/x-goog-token=Expires=1590180862&KeyName=prod-sign-url-key&Signature=E_6EtuTzw6abggGHrOfxUkKaWcs/Master.m3u8')
# playlist = m3u8.load('https://test-streams.mux.dev/x36xhzz/x36xhzz.m3u8')
# playlist = m3u8.load('https://bitdash-a.akamaihd.net/content/sintel/hls/playlist.m3u8')
# playlist = m3u8.load('https://test-streams.mux.dev/x36xhzz/url_2/193039199_mp4_h264_aac_ld_7.m3u8')
# this is an absolute filename
# playlist = m3u8.load('../sample-stream-playlists/sintel/playlist.m3u8')
# print(playlist.segments)
# print(playlist.target_duration)
# print(playlist.media)
# print(playlist.playlists)
# print(playlist.playlists[0].stream_info.resolution)
# print(playlist.playlists[0].media[0].type)
# print(playlist.files)
# if you want to write a file from its content
# playlist.dump('playlist.m3u8')
if playlist.playlists:
print('%s playlists found...' % len(playlist.playlists))
count = 0
for pl in playlist.playlists:
count += 1
print('%d - Resolution: %s Bandwidth: %s' %
def test_load():
m3u8_obj = m3u8.load(
'./index-v1-a1.m3u8') # this could also be an absolute filename
print(m3u8_obj.segments)
print(m3u8_obj.target_duration)
print(m3u8_obj.keys)
def get_uri_from_m3u8(self, file_name):
print("正在解析真实下载地址..." + str(file_name))
m3u8Obj = m3u8.load(file_name)
print("解析完成.")
return m3u8Obj.segments
def load_m3u8(url):
urls = []
m = m3u8.load(url)
for seg in m.segments:
urls.append(seg.absolute_uri)
return urls
def _collectSegments(self):
logger.info('Downloading and parsing HLS manifest from %s' % self.manifesturi)
m3u8_obj = m3u8.load(self.manifesturi)
if not m3u8_obj.is_variant:
raise Exception('%s is not a master manifest' % self.manifesturi)
listlengths = []
for mediaplaylist in m3u8_obj.playlists:
url = urlparse(self.manifesturi)
mediauri = mediaplaylist.uri
if mediaplaylist.uri[0] == "/":
mediauri = url.scheme + "://" + url.hostname + mediaplaylist.uri
debug.log('Building segment list from %s' % mediauri)
try:
logger.info('Downloading segment playlist from %s' % mediauri)
bw = mediaplaylist.stream_info.average_bandwidth
if not bw:
bw = mediaplaylist.stream_info.bandwidth
segmentlist = SegmentList(mediauri, str(bw), self.tmpdir)
except Exception as e:
logger.error('Failed to download: %s' % str(e))
else:
logger.info('Segment playlist from %s downloaded and parsed' % mediauri)
self.bitrates.append(segmentlist)
listlengths.append(segmentlist.getLength())
if len(self.bitrates) == 0:
raise Exception('No segment playlists that could be downloaded was found')
# This is to handle the edge case where the segmentlists differs in length and start segment
# A special case that actually should not happened
debug.log('Shortest list length %d' % min(listlengths))
debug.log('Longest list length %d' % max(listlengths))
headsegments = {}
for segmentlist in self.bitrates:
if segmentlist.getFirstSegment() in headsegments:
headsegments[segmentlist.getFirstSegment()] += 1
else:
headsegments[segmentlist.getFirstSegment()] = 1
debug.log(headsegments)
# Find start segment winner
winner = sorted(headsegments.items(), key=operator.itemgetter(1), reverse=True)[0][0]
# Make sure all bitrates starts with the same segment
if len(headsegments.keys()) > 1:
debug.log('First segment differs and we have chosen %s as winner' % winner)
for segmentlist in self.bitrates:
if segmentlist.getFirstSegment() != winner:
segmentlist.removeFirstSegment()
# Make sure that we have the same length on all bitrates
segmentlengths = {}
for segmentlist in self.bitrates:
length = segmentlist.getLength()
if length in segmentlengths:
segmentlengths[length] += 1
else:
segmentlengths[length] = 1
shortestlength = sorted(segmentlengths.items(), key=operator.itemgetter(0))[0][0]
debug.log(shortestlength)
for segmentlist in self.bitrates:
length = segmentlist.getLength()
if length > shortestlength:
segmentlist.removeLastSegment()
# Sanity check
firstsegments = {}
for segmentlist in self.bitrates:
debug.log('First segment: %s of (%d)' % (segmentlist.getFirstSegment(), segmentlist.getLength()))
if segmentlist.getFirstSegment() in firstsegments:
firstsegments[segmentlist.getFirstSegment()] += 1
else:
firstsegments[segmentlist.getFirstSegment()] = 1
debug.log('Keys %d' % len(firstsegments.keys()))
if len(firstsegments.keys()) > 1:
debug.log(firstsegments)
logger.warning("First segment in segment lists differs")
#!/usr/bin/env python3
"""
Use a playlist to stream files from m3u8: https://github.com/globocom/m3u8
pip install m3u8
Example from Akamai:
https://learn.akamai.com/en-us/webhelp/media-services-live/media-services-live-4-user-guide/GUID-0A50253F-0B1B-406B-A8C9-3788CB42F950.html
"""
from pathlib import Path
import pylivestream.api as pls
import m3u8
playlist_m3u8 = Path(__file__).parent / "local.m3u8"
playlist = m3u8.load(str(playlist_m3u8))
files = playlist.files
for file in files:
pls.stream_file(
ini_file=None,
websites="localhost",
assume_yes=True,
video_file=file,
)
from kivy.uix.videoplayer import VideoPlayer
import m3u8
import requests
#http://bsn246u6-i.akamaihd.net/hls/live/237201/Live2N24CMS/03.m3u8
m3u8_obj = m3u8.load(
'http://bsn246u6-i.akamaihd.net/hls/live/237201/Live2N24CMS/03.m3u8')
print str(m3u8_obj)
print m3u8_obj.segments
print m3u8_obj.target_duration
t = requests.get(
'http://bsn246u6-i.akamaihd.net/hls/live/237201/Live2N24CMS/20161004T125034-03-227477.ts'
)
print t
player = VideoPlayer(
source=
'http://bsn246u6-i.akamaihd.net/hls/live/237201/Live2N24CMS/20161004T125034-03-227630.ts'
)
player.state = 'play'
def get(self):
if not self.stream_args["duration"] == 0:
self.stream_args[
"time_end"] = self.stream_args["duration"] + time.time()
if not self.stream_args["true_content_type"].startswith(
tuple(["application/", "text/"])):
self.fhdhr.logger.info("Direct Stream of %s URL: %s" %
(self.stream_args["true_content_type"],
self.stream_args["channelUri"]))
req = self.fhdhr.web.session.get(self.stream_args["channelUri"],
stream=True)
def generate():
try:
chunk_counter = 1
while self.tuner.tuner_lock.locked():
for chunk in req.iter_content(
chunk_size=self.chunksize):
if (not self.stream_args["duration"] == 0
and not time.time() <
self.stream_args["time_end"]):
req.close()
self.fhdhr.logger.info(
"Requested Duration Expired.")
self.tuner.close()
if not chunk:
break
# raise TunerError("807 - No Video Data")
self.fhdhr.logger.info(
"Passing Through Chunk #%s with size %s" %
(chunk_counter, self.chunksize))
yield chunk
chunk_counter += 1
self.fhdhr.logger.info(
"Connection Closed: Tuner Lock Removed")
except GeneratorExit:
self.fhdhr.logger.info("Connection Closed.")
except Exception as e:
self.fhdhr.logger.info("Connection Closed: " + str(e))
finally:
req.close()
self.tuner.close()
# raise TunerError("806 - Tune Failed")
else:
self.fhdhr.logger.info("Detected stream URL is m3u8: %s" %
self.stream_args["true_content_type"])
channelUri = self.stream_args["channelUri"]
while True:
videoUrlM3u = m3u8.load(channelUri)
if len(videoUrlM3u.playlists):
channelUri = videoUrlM3u.playlists[0].absolute_uri
else:
break
def generate():
try:
played_chunk_urls = []
while self.tuner.tuner_lock.locked():
playlist = m3u8.load(channelUri)
segments = playlist.segments
if len(played_chunk_urls):
newsegments = 0
for segment in segments:
if segment.absolute_uri not in played_chunk_urls:
newsegments += 1
self.fhdhr.logger.info(
"Refreshing m3u8, Loaded %s new segments." %
str(newsegments))
else:
self.fhdhr.logger.info("Loaded %s segments." %
str(len(segments)))
if playlist.keys != [None]:
keys = [{
"url": key.uri,
"method": key.method,
"iv": key.iv
} for key in playlist.keys if key]
else:
keys = [None for i in range(0, len(segments))]
for segment, key in zip(segments, keys):
chunkurl = segment.absolute_uri
if chunkurl not in played_chunk_urls:
played_chunk_urls.append(chunkurl)
if (not self.stream_args["duration"] == 0
and not time.time() <
self.stream_args["time_end"]):
self.fhdhr.logger.info(
"Requested Duration Expired.")
self.tuner.close()
chunk = self.fhdhr.web.session.get(
chunkurl).content
if not chunk:
break
# raise TunerError("807 - No Video Data")
if key:
keyfile = self.fhdhr.web.session.get(
key["url"]).content
cryptor = AES.new(keyfile, AES.MODE_CBC,
keyfile)
chunk = cryptor.decrypt(chunk)
self.fhdhr.logger.info(
"Passing Through Chunk: %s" % chunkurl)
yield chunk
if playlist.target_duration:
time.sleep(int(playlist.target_duration))
self.fhdhr.logger.info(
"Connection Closed: Tuner Lock Removed")
except GeneratorExit:
self.fhdhr.logger.info("Connection Closed.")
except Exception as e:
self.fhdhr.logger.info("Connection Closed: " + str(e))
finally:
self.tuner.close()
# raise TunerError("806 - Tune Failed")
return generate()
def test_load_non_exist_file():
m3u8_obj = m3u8.load(
'./non_exist.m3u8') # this could also be an absolute filename
print(m3u8_obj.segments)
print(m3u8_obj.target_duration)
print(m3u8_obj.keys)
def downts_thread(url=None,save_dir=None,processInfo=None):
# 获取频道名称
channel_name = url.split('/')[-2]
listfile_name = url.split('/')[-1]
# 初始化提取信息
processInfo = {channel_name: {'latest': '-', 'action': 'init process', 'info': 'init process'}}
showProcessInfo(processInfo)
# 新建文件夹
save_dir = os.path.join(save_dir, channel_name)
if not os.path.exists(save_dir):
os.makedirs(save_dir)
# list_file_name
list_file_name = channel_name + "_" + listfile_name
# list_file_full_path
list_file_full_path = os.path.join(save_dir, list_file_name)
pre_md5 = "" # 文件校验码
while 1:
djv = Dejavu(config)
# 保存ts流列表文件（cctv15_01.m3u8）
processInfo[channel_name]['action'] = 'downloading'
processInfo[channel_name]['info'] = 'downloading file:' + list_file_name
showProcessInfo(processInfo)
# save(url,list_file_full_path,True)
download_to_file(url,list_file_full_path,True)
#并计算校验码
current_md5 = getFileMD5(list_file_full_path) # 当前文件校验码
# 对比是否有变化
if current_md5==pre_md5:
processInfo[channel_name]['action'] = 'thread sleeping'
processInfo[channel_name]['info'] = 'there is no new media ,'+ channel_name+'-thread is sleeping'
showProcessInfo(processInfo)
#休息一下继续开始
sleep(Config.TS_SLEEP);
continue
# 提取文件列表
m3u8_obj = m3u8.load(url)
# 间隔时间
gap_time = m3u8_obj.target_duration
base_URL = url[:url.rfind('/')+1]
for index,file in enumerate(m3u8_obj.files):
# 文件的网络路径
file_url = base_URL + file
# 获取开始时间
ts_start_time = getTS_Format_Time(m3u8_obj.program_date_time, int(index * int(gap_time)))
# 得到完整文件名
file_temp_name = channel_name+"_"+ts_start_time+'.ts'
file_full_path = os.path.join(save_dir,file_temp_name)
song_name = os.path.splitext(os.path.basename(file_temp_name))[0]
if not djv.isSongFingerPrinted(song_name):
#下载该文件
processInfo[channel_name]['action'] = 'downloading'
processInfo[channel_name]['info'] = 'downloading file:' + file_temp_name
showProcessInfo(processInfo)
download_to_file(file_url,file_full_path,False)
# 调用指纹提取程序提取指纹
processInfo[channel_name]['action'] = 'fingerprinting'
processInfo[channel_name]['info'] = 'fingerpinting file:' + file_temp_name
showProcessInfo(processInfo)
djv.fingerprint_file(file_full_path)
else:
processInfo[channel_name]['action'] = 'skip fingerprinted'
processInfo[channel_name]['info'] = file_temp_name + 'has fingerprinted'
showProcessInfo(processInfo)
# 更新信息
processInfo[channel_name]['action'] = 'update-info'
processInfo[channel_name]['info'] = channel_name+' has updated to ' + ts_start_time
processInfo[channel_name]['latest'] = ts_start_time
showProcessInfo(processInfo)
# 清理文件
processInfo[channel_name]['action'] = 'delete ts file'
processInfo[channel_name]['info'] = 'deleting file:' + file_temp_name
showProcessInfo(processInfo)
if os.path.exists(file_full_path):
os.remove(file_full_path)
# 提取完成更新文件校验码
pre_md5 = current_md5
# 是否需要清理历史数据
# 休息一下继续开始
processInfo[channel_name]['action'] = 'thread sleeping'
processInfo[channel_name]['info'] = 'playlist has been stored,' + channel_name + '-thread is sleeping'
showProcessInfo(processInfo)
if os.path.exists(list_file_full_path):
os.remove(list_file_full_path)
sleep(Config.TS_SLEEP)
htmlfile = cloudscraper.create_scraper(browser='firefox', delay=10).get(url)
result = re.search("https://.+m3u8", htmlfile.text)
m3u8url = result[0]
m3u8urlList = m3u8url.split('/')
m3u8urlList.pop(-1)
downloadurl = '/'.join(m3u8urlList)
# 儲存 m3u8 file 至資料夾
m3u8file = os.path.join(folderPath, dirName + '.m3u8')
urllib.request.urlretrieve(m3u8url, m3u8file)
# In[5]:
# 得到 m3u8 file裡的 URI和 IV
m3u8obj = m3u8.load(m3u8file)
m3u8uri = ''
m3u8iv = ''
for key in m3u8obj.keys:
if key:
m3u8uri = key.uri
m3u8iv = key.iv
# 儲存 ts網址 in tsList
tsList = []
for seg in m3u8obj.segments:
tsUrl = downloadurl + '/' + seg.uri
tsList.append(tsUrl)
# In[6]:
for url in videoUrls:
sleep(randint(1, 2))
r = requests.get(baseUrl + url)
soup = BeautifulSoup(r.text, 'lxml')
videoUrl = '/'.join(soup.find('video')['src'].split('/')[:-1])
baseName = videoUrl.split('/')[-1]
if not isdir(baseName):
if exists(baseName):
remove(baseName)
mkdir(baseName)
# Get list of audio clips and save them
if not exists(baseName + '.wav'):
clips = []
audioIndex = m3u8.load(videoUrl + '/audio/zho/zho.m3u8')
if audioIndex.files:
for fileAudio in audioIndex.files:
if not exists(join(baseName, fileAudio)):
sleep(randint(3, 5))
print(videoUrl + '/audio/zho/' + fileAudio)
r = requests.get(videoUrl + '/audio/zho/' + fileAudio)
with open(join(baseName, fileAudio), 'wb') as f:
f.write(r.content)
FFmpeg(global_options='-y',
inputs={
join(baseName, fileAudio): None
},
outputs={
join(baseName, fileAudio + '.wav'): None
}).run()
def get_station_stream_uri(self, station_id):
print("Getting station info for " + station_id + "...")
try:
videoUrlReq = urllib.request.Request('https://api.locastnet.org/api/watch/station/' +
str(station_id) + '/' +
self.current_location['latitude'] + '/' +
self.current_location['longitude'],
headers={'Content-Type': 'application/json',
'authorization': 'Bearer ' + self.current_token,
'User-agent': 'Mozilla/5.0'})
videoUrlOpn = urllib.request.urlopen(videoUrlReq)
videoUrlRes = json.load(videoUrlOpn)
videoUrlOpn.close()
except urllib.error.URLError as urlError:
print("Error when getting the video URL: " + str(urlError.reason))
return False
except urllib.error.HTTPError as httpError:
print("Error when getting the video URL: " + str(httpError.reason))
return False
except:
videoUrlReqErr = sys.exc_info()[0]
print("Error when getting the video URL: " + videoUrlReqErr.message)
return False
print("Determining best video stream for " + station_id + "...")
bestStream = None
# find the heighest stream url resolution and save it to the list
videoUrlM3u = m3u8.load(videoUrlRes['streamUrl'], headers={'authorization': 'Bearer ' + self.current_token,
'User-agent': 'Mozilla/5.0'})
print("Found " + str(len(videoUrlM3u.playlists)) + " Playlists")
if len(videoUrlM3u.playlists) > 0:
for videoStream in videoUrlM3u.playlists:
if bestStream is None:
bestStream = videoStream
elif ((videoStream.stream_info.resolution[0] > bestStream.stream_info.resolution[0]) and
(videoStream.stream_info.resolution[1] > bestStream.stream_info.resolution[1])):
bestStream = videoStream
elif ((videoStream.stream_info.resolution[0] == bestStream.stream_info.resolution[0]) and
(videoStream.stream_info.resolution[1] == bestStream.stream_info.resolution[1]) and
(videoStream.stream_info.bandwidth > bestStream.stream_info.bandwidth)):
bestStream = videoStream
if bestStream is not None:
print(station_id + " will use " +
str(bestStream.stream_info.resolution[0]) + "x" + str(bestStream.stream_info.resolution[1]) +
" resolution at " + str(bestStream.stream_info.bandwidth) + "bps")
return bestStream.absolute_uri
else:
print("No variant streams found for this station. Assuming single stream only.")
return videoUrlRes['streamUrl']
def process_playback_url(playback_url, auth_string):
logging.debug('Playback url %s' % playback_url)
stream_quality = str(get_setting('StreamQuality'))
bitrate_limit = int(get_setting('BitrateLimit'))
logging.debug('Stream Quality %s' % stream_quality)
try:
m3u8_obj = m3u8.load(playback_url)
except Exception as e:
logging.error('Unable to load m3u8 %s' % e)
playback_url += '|' + auth_string
item = xbmcgui.ListItem(path=playback_url)
return setResolvedUrl(plugin.handle, True, item)
success = True
use_inputstream_addon = not get_setting_as_bool('DisableInputStream')
if not use_inputstream_addon:
if m3u8_obj.is_variant:
stream_options = list()
bandwidth_key = 'bandwidth'
m3u8_obj.playlists.sort(
key=lambda playlist: playlist.stream_info.bandwidth,
reverse=True)
m3u8_obj.data['playlists'].sort(key=lambda playlist: int(playlist[
'stream_info'][bandwidth_key]),
reverse=True)
stream_quality_index = str(get_setting('StreamQualityIndex'))
stream_index = None
should_ask = False
try:
stream_index = int(stream_quality_index)
if stream_index < 0 or stream_index >= len(m3u8_obj.playlists):
should_ask = True
except ValueError:
should_ask = True
if '0' == stream_quality: # Best
stream_index = 0
should_ask = False
for playlist in m3u8_obj.data['playlists']:
stream_info = playlist['stream_info']
bandwidth = int(stream_info[bandwidth_key]) / 1024
if bandwidth <= bitrate_limit:
break
stream_index += 1
elif '2' == stream_quality: # Ask everytime
should_ask = True
if should_ask:
for playlist in m3u8_obj.data['playlists']:
stream_info = playlist['stream_info']
resolution = stream_info['resolution']
frame_rate = stream_info[
'frame_rate'] if 'frame_rate' in stream_info else 30.0
bandwidth = int(int(stream_info[bandwidth_key]) / 1024)
if 'average_bandwidth' in stream_info:
logging.debug('bandwidth: %s average bandwidth: %s' %
(stream_info['bandwidth'],
stream_info['average_bandwidth']))
stream_options.append(
get_string(30450) %
(resolution, frame_rate, bandwidth))
dialog = xbmcgui.Dialog()
stream_index = dialog.select(get_string(30440), stream_options)
if stream_index < 0:
success = False
else:
set_setting('StreamQualityIndex', value=str(stream_index))
uri = m3u8_obj.playlists[stream_index].uri
logging.debug('Chose stream %d; %s' % (stream_index, uri))
if 'http' not in uri[0:4]:
index_of_last_slash = playback_url.rfind('/')
uri = playback_url[0:index_of_last_slash] + '/' + uri
item = xbmcgui.ListItem(path=uri + '|' + auth_string)
setResolvedUrl(plugin.handle, success, item)
else:
item = xbmcgui.ListItem(path=playback_url)
setResolvedUrl(plugin.handle, success, item)
else:
logging.debug('Using inputstream.adaptive addon')
item = xbmcgui.ListItem(path=playback_url)
item.setProperty('inputstreamaddon', 'inputstream.adaptive')
item.setProperty('inputstream.adaptive.manifest_type', 'hls')
setResolvedUrl(plugin.handle, success, item)
def handle(self, *args, **options):
m3u8_obj = m3u8.load(settings.BASE_DIR +
'/importar/{}'.format(options['filename']))
x = 1
for channel in m3u8_obj.segments:
name = channel.title.split(',')[-1]
name = name[:255]
try:
group_title = re.findall(r'group-title=\"(.*)\" ',
channel.title)[0]
group_title = group_title[:255]
except IndexError as e:
group_title = ''
try:
logo = re.findall(r'tvg-logo=\"(.*)\" ', channel.title)[0]
logo = logo[:255]
except IndexError as e:
logo = ''
print('sem logo e:', e)
uri = channel.uri
uri = uri[:255]
print('* {}'.format(x))
try:
canal = Canal.objects.get(uri=uri)
if options['test_stream']:
if canal.status == 200:
response = requests.get(uri, stream=True)
size = 0
status = 0
for chunk in response.iter_content(256):
size += len(chunk)
if size > 512:
print('size: ', size)
status = 200
break
if status == 0:
status = response.status_code
canal.status = status
canal.save()
print('ID: {} - status: {}'.format(canal.id, status))
print('ID: {}'.format(canal.id))
except Canal.DoesNotExist:
try:
print('testing: {} ({})'.format(uri, name))
response = requests.get(uri, stream=True, timeout=60)
headers = requests.head(uri, stream=True,
timeout=60).headers
content_type = headers.get('content-type')
print('Content-type: ', content_type)
if content_type == 'application/vnd.apple.mpegurl':
status = 200
elif content_type == 'application/vnd.apple.mpegURL':
status = 200
elif content_type == 'video/mp4':
status = 200
elif content_type == 'application/x-mpegURL':
status = 200
elif content_type == 'application/vnd.apple.mpegurl, application/vnd.apple.mpegurl':
status = 200
elif content_type == 'video/mp2t':
status = 200
elif content_type == 'audio/aacp': # RADIO
status = 1
elif content_type == 'audio/mpeg': # RADIO
status = 1
elif content_type == 'audio/x-mpegurl': # RADIO
status = 1
elif content_type == 'text/html; charset=UTF-8':
status = 404
elif content_type == 'text/html; charset=utf-8':
status = 404
elif content_type == 'text/html;charset=UTF-8':
status = 404
elif content_type == 'image/png':
status = 404
elif content_type == 'image/jpeg':
status = 404
elif content_type == 'application/xml':
status = 404
elif content_type == 'text/plain':
status = 404
elif content_type == 'text/html':
status = 404
elif content_type == 'application/octet-stream':
status = 2
elif content_type is None:
status = 0
else:
status = 404
exit(0)
if status == 200:
size = 0
status = 0
for chunk in response.iter_content(256):
size += len(chunk)
if size > 4096:
print('size: ', size)
status = 200
break
if status == 0:
status = response.status_code
canal = Canal(name=name,
logo=logo,
uri=uri,
group_title=group_title,
status=status)
canal.save()
print(
'#EXTINF:{}, tvg-id="{}" tvg-name="{}" tvg-logo="{}" group-title="{}",{}\n{}'
.format(canal.id, name, name, logo, group_title,
name, uri))
else:
canal = Canal(name=name,
logo=logo,
uri=uri,
group_title=group_title,
status=status)
canal.save()
print('status: ', status)
except requests.exceptions.RequestException as e:
print(e)
canal = Canal(name=name,
logo=logo,
uri=uri,
group_title=group_title,
status=404)
canal.save()
except DataError as e:
print(e)
canal = Canal(name=name,
logo=logo,
uri=uri,
group_title=group_title,
status=404)
canal.save()
except Exception as e:
print(e)
canal = Canal(name=name,
logo=logo,
uri=uri,
group_title=group_title,
status=status)
canal.save()
x += 1
"""
pip install m3u8 first, please
"""
import m3u8
import time
import urllib2
url = "http://192.168.7.40:20119/live/cctvnews/encoder/0/playlist.m3u8"
media_sequence = 0
playlist = m3u8.load(url)
while playlist.is_variant:
url = playlist.base_uri + "/" + playlist.playlists[0].uri
playlist = m3u8.load(url)
current_sequence = media_sequence = playlist.media_sequence
for segment in playlist.segments:
print current_sequence, segment.uri
current_sequence += 1
while True:
playlist = m3u8.load(url)
if playlist.media_sequence == media_sequence:
time.sleep(1)
continue
media_sequence = playlist.media_sequence
target_duration = playlist.target_duration
if current_sequence < media_sequence:
print "ERROR : missing segment"