def getDirectM3U8Playlist(M3U8Url, checkExt=True, variantCheck=True, cookieParams={}): if checkExt and not M3U8Url.split("?")[0].endswith(".m3u8"): return [] cm = common() meta = strwithmeta(M3U8Url).meta params, postData = cm.getParamsFromUrlWithMeta(M3U8Url) params.update(cookieParams) retPlaylists = [] try: finallM3U8Url = meta.get("iptv_m3u8_custom_base_link", "") if "" == finallM3U8Url: params["return_data"] = False sts, response = cm.getPage(M3U8Url, params, postData) finallM3U8Url = response.geturl() data = response.read().strip() response.close() else: sts, data = cm.getPage(M3U8Url, params, postData) data = data.strip() m3u8Obj = m3u8.inits(data, finallM3U8Url) if m3u8Obj.is_variant: for playlist in m3u8Obj.playlists: item = {} if not variantCheck or playlist.absolute_uri.split("?")[-1].endswith(".m3u8"): meta.update({"iptv_proto": "m3u8", "iptv_bitrate": playlist.stream_info.bandwidth}) item["url"] = strwithmeta(playlist.absolute_uri, meta) else: meta.pop("iptv_proto", None) item["url"] = decorateUrl(playlist.absolute_uri, meta) item["bitrate"] = playlist.stream_info.bandwidth if None != playlist.stream_info.resolution: item["with"] = playlist.stream_info.resolution[0] item["heigth"] = playlist.stream_info.resolution[1] else: item["with"] = 0 item["heigth"] = 0 item["codec"] = playlist.stream_info.codecs item["name"] = "bitrate: %s res: %dx%d kodek: %s" % ( item["bitrate"], item["with"], item["heigth"], item["codec"], ) retPlaylists.append(item) else: item = {"name": "m3u8", "url": M3U8Url, "codec": "unknown", "with": 0, "heigth": 0, "bitrate": "unknown"} retPlaylists.append(item) except: printExc() return retPlaylists
def getDirectM3U8Playlist(M3U8Url, checkExt=True, variantCheck=True, cookieParams={}): if checkExt and not M3U8Url.split('?')[0].endswith('.m3u8'): return [] cm = common() meta = strwithmeta(M3U8Url).meta params, postData = cm.getParamsFromUrlWithMeta(M3U8Url) params.update(cookieParams) retPlaylists = [] try: finallM3U8Url = meta.get('iptv_m3u8_custom_base_link', '') if '' == finallM3U8Url: params['return_data'] = False sts, response = cm.getPage(M3U8Url, params, postData) finallM3U8Url = response.geturl() data = response.read().strip() response.close() else: sts, data = cm.getPage(M3U8Url, params, postData) data = data.strip() m3u8Obj = m3u8.inits(data, finallM3U8Url) if m3u8Obj.is_variant: for playlist in m3u8Obj.playlists: item = {} if not variantCheck or playlist.absolute_uri.split('?')[-1].endswith('.m3u8'): meta.update({'iptv_proto':'m3u8', 'iptv_bitrate':playlist.stream_info.bandwidth}) item['url'] = strwithmeta(playlist.absolute_uri, meta) else: meta.pop('iptv_proto', None) item['url'] = decorateUrl(playlist.absolute_uri, meta) item['bitrate'] = playlist.stream_info.bandwidth if None != playlist.stream_info.resolution: item['with'] = playlist.stream_info.resolution[0] item['heigth'] = playlist.stream_info.resolution[1] else: item['with'] = 0 item['heigth'] = 0 item['codec'] = playlist.stream_info.codecs item['name'] = "bitrate: %s res: %dx%d kodek: %s" % ( item['bitrate'], \ item['with'], \ item['heigth'], \ item['codec'] ) retPlaylists.append(item) else: item = {'name':'m3u8', 'url':M3U8Url, 'codec':'unknown', 'with':0, 'heigth':0, 'bitrate':'unknown'} retPlaylists.append(item) except: printExc() return retPlaylists
def _updateM3U8Finished(self, code=0): printDBG('m3u8 _updateM3U8Finished update code[%d]--- ' % (code)) if self.liveStream and self.M3U8Updater: if 0 < len(self.M3U8ListData) and 0 == code: try: m3u8Obj = m3u8.inits(self.M3U8ListData, self.m3u8Url) if self.liveStream and not m3u8Obj.is_variant: self.refreshDelay = int(m3u8Obj.target_duration) if self.refreshDelay < 5: self.refreshDelay = 5 if 0 < len(m3u8Obj.segments): newFragments = [ self._segUri(seg.absolute_uri) for seg in m3u8Obj.segments ] #self.mergeFragmentsList(newFragments) self.mergeFragmentsListWithChecking( newFragments, m3u8Obj.media_sequence + len(m3u8Obj.segments)) printDBG( 'm3u8 _updateM3U8Finished list updated ---') except Exception: printDBG( "m3u8 _updateM3U8Finished exception url[%s] data[%s]" % (self.m3u8Url, self.M3U8ListData)) else: printDBG('m3u8 _updateM3U8Finished no data ---') # hardcode self.M3U8UpdaterRefreshDelay += 1 if self.refreshDelay < self.M3U8UpdaterRefreshDelay or 0 != code: self.M3U8UpdaterRefreshDelay = 0 self.M3U8ListData = '' m3u8Url = self._addTimeStampToUrl(self.m3u8Url) printDBG( ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> [%s]" % m3u8Url) cmd = DMHelper.getBaseWgetCmd(self.downloaderParams) + ( ' --tries=0 --timeout=%d ' % self._getTimeout()) + '"' + m3u8Url + '" -O - 2> /dev/null' printDBG("m3u8 _updateM3U8Finished download cmd[%s]" % cmd) self.M3U8Updater.execute(E2PrioFix(cmd)) return else: self.M3U8Updater.execute(E2PrioFix("sleep 1")) return printDBG( "|||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||") printDBG( "||||||||||||| m3u8 _updateM3U8Finished FINISHED |||||||||||||") printDBG( "|||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||")
def getDirectM3U8Playlist(M3U8Url, checkExt=True): if checkExt and not M3U8Url.split('?')[0].endswith('.m3u8'): return [] cm = common() headerParams, postData = cm.getParamsFromUrlWithMeta(M3U8Url) headerParams['return_data'] = False retPlaylists = [] try: sts, response = cm.getPage(M3U8Url, headerParams, postData) finallM3U8Url = response.geturl() data = response.read().strip() response.close() m3u8Obj = m3u8.inits(data, finallM3U8Url) if m3u8Obj.is_variant: for playlist in m3u8Obj.playlists: item = {} item['url'] = strwithmeta( playlist.absolute_uri, { 'iptv_proto': 'm3u8', 'iptv_bitrate': playlist.stream_info.bandwidth }) item['bitrate'] = playlist.stream_info.bandwidth if None != playlist.stream_info.resolution: item['with'] = playlist.stream_info.resolution[0] item['heigth'] = playlist.stream_info.resolution[1] else: item['with'] = 0 item['heigth'] = 0 item['codec'] = playlist.stream_info.codecs item['name'] = "bitrate: %s res: %dx%d kodek: %s" % ( item['bitrate'], \ item['with'], \ item['heigth'], \ item['codec'] ) retPlaylists.append(item) else: item = { 'name': 'm3u8', 'url': M3U8Url, 'codec': 'unknown', 'with': 0, 'heigth': 0, 'bitrate': 'unknown' } retPlaylists.append(item) except: printExc() return retPlaylists
def _updateM3U8Finished(self, code=0): printDBG('m3u8 _updateM3U8Finished update code[%d]--- ' % (code)) if self.liveStream and self.M3U8Updater: if 0 < len(self.M3U8ListData) and 0 == code: try: m3u8Obj = m3u8.inits(self.M3U8ListData, self.m3u8Url) if self.liveStream and not m3u8Obj.is_variant: self.refreshDelay = int(m3u8Obj.target_duration) if self.refreshDelay < 5: self.refreshDelay = 5 if 0 < len(m3u8Obj.segments): newFragments = [self._segUri(seg.absolute_uri) for seg in m3u8Obj.segments] #self.mergeFragmentsList(newFragments) self.mergeFragmentsListWithChecking(newFragments) printDBG('m3u8 _updateM3U8Finished list updated ---') except: printDBG("m3u8 _updateM3U8Finished exception url[%s] data[%s]" % (self.m3u8Url, self.M3U8ListData)) else: printDBG('m3u8 _updateM3U8Finished no data ---') # hardcode self.M3U8UpdaterRefreshDelay += 1 if self.refreshDelay < self.M3U8UpdaterRefreshDelay or 0 != code: self.M3U8UpdaterRefreshDelay = 0 self.M3U8ListData = '' m3u8Url = self.m3u8Url if '?' in m3u8Url: m3u8Url += '&iptv_stamp=' else: m3u8Url += '?iptv_stamp=' m3u8Url += ('%s' % time()) printDBG(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> [%s]" % m3u8Url) cmd = DMHelper.getBaseWgetCmd(self.downloaderParams) + (' --tries=0 --timeout=%d ' % self._getTimeout()) + '"' + m3u8Url + '" -O - 2> /dev/null' printDBG("m3u8 _updateM3U8Finished download cmd[%s]" % cmd) self.M3U8Updater.execute(cmd) return else: self.M3U8Updater.execute("sleep 1") return printDBG("|||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||") printDBG("||||||||||||| m3u8 _updateM3U8Finished FINISHED |||||||||||||") printDBG("|||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||")
def _cmdFinished(self, code, terminated=False): printDBG( "M3U8Downloader._cmdFinished code[%r] terminated[%r] downloadType[%s]" % (code, terminated, self.downloadType)) localStatus = DMHelper.STS.ERROR if terminated: BaseDownloader.updateStatistic(self) localStatus = DMHelper.STS.INTERRUPTED elif self.DOWNLOAD_TYPE.M3U8 == self.downloadType: self.console_appClosed_conn = None self.console_stdoutAvail_conn = None if 0 < len(self.outData): try: m3u8Obj = m3u8.inits(self.outData, self.m3u8Url) # uri given to m3u8 downloader should not be variant, # format should be selected before starting downloader # however if this was not done the firs one will be selected if m3u8Obj.is_variant: if 0 < len(m3u8Obj.playlists): self.m3u8Url = self._segUri( m3u8Obj.playlists[-1].absolute_uri) self._startM3U8() localStatus = DMHelper.STS.DOWNLOADING else: if 0 < len(m3u8Obj.segments): if not m3u8Obj.is_endlist: self.liveStream = True if -1 == self.startLiveDuration: self.fragmentList = [ self._segUri(seg.absolute_uri) for seg in m3u8Obj.segments ] else: # some live streams only add new fragments not removing old, # in this case most probably we not want to download old fragments # but start from last N fragments/seconds # self.startLiveDuration self.fragmentList = [] currentDuration = 0 maxFragDuration = m3u8Obj.target_duration for seg in reversed(m3u8Obj.segments): if None != seg.duration: currentDuration += seg.duration else: currentDuration += maxFragDuration self.fragmentList.append( self._segUri(seg.absolute_uri)) if currentDuration >= self.startLiveDuration: break self.fragmentList.reverse() if len(m3u8Obj.segments) == len( self.fragmentList) and len( self.fragmentList ) > self.skipFirstSegFromList: self.fragmentList = self.fragmentList[ self.skipFirstSegFromList:] self.lastMediaSequence = m3u8Obj.media_sequence + len( m3u8Obj.segments) # start update fragment list loop #self.fragmentList = self.fixFragmentsList(self.fragmentList) self._updateM3U8Finished(-1) else: self.fragmentList = [ self._segUri(seg.absolute_uri) for seg in m3u8Obj.segments ] try: self.totalDuration = 0 self.fragmentDurationList = [] for seg in m3u8Obj.segments: self.totalDuration += seg.duration self.fragmentDurationList.append( seg.duration) except Exception: printExc() self.totalDuration = -1 self.fragmentDurationList = [] localStatus = self._startFragment() except Exception: pass printDBG(">>>>>>>>>>>>>>>>>> localStatus [%s] tries[%d]" % (localStatus, self.tries)) if localStatus == DMHelper.STS.ERROR and self.tries < self.maxTriesAtStart: self.console_appClosed_conn = None self.console_stdoutAvail_conn = None self.tries += 1 self._startM3U8(self.MIN_REFRESH_DELAY) return else: self.tries = 0 elif self.liveStream and self.DOWNLOAD_TYPE.WAITTING == self.downloadType: printDBG( "m3u8 liveStream waitting finished--------------------------------" ) localStatus = self._startFragment() else: BaseDownloader.updateStatistic(self) printDBG( "m3u8 nextFragment finished: live[%r]: r[%d], l[%d], p[%d]" % (self.liveStream, self.remoteFragmentSize, self.localFileSize, self.m3u8_prevLocalFileSize)) if 0 >= self.localFileSize: if not self.liveStream: localStatus = DMHelper.STS.ERROR else: localStatus = self._startFragment() #elif not self.liveStream and self.remoteFragmentSize > 0 and self.remoteFragmentSize > (self.localFileSize - self.m3u8_prevLocalFileSize): # localStatus = DMHelper.STS.INTERRUPTED elif 0 < (self.localFileSize - self.m3u8_prevLocalFileSize): if self.totalDuration > 0: try: self.downloadDuration += self.fragmentDurationList[ self.currentFragment] except Exception: printExc() localStatus = self._startFragment() elif 0 == (self.localFileSize - self.m3u8_prevLocalFileSize): localStatus = self._startFragment(True) # retry else: localStatus = DMHelper.STS.INTERRUPTED self.status = localStatus if DMHelper.STS.DOWNLOADING == self.status: return # clean up at finish if self.M3U8Updater: self.M3U8Updater_appClosed_conn = None self.M3U8Updater_stdoutAvail_conn = None self.M3U8Updater = None self.liveStream = False if self.console: self.console_appClosed_conn = None self.console_stdoutAvail_conn = None self.console.sendCtrlC() # kill # produce zombies self.console = None ''' if None != self.updateThread: if self.updateThread.Thread.isAlive(): # give some time for update thread to finish sleep(self.MIN_REFRESH_DELAY) printDBG('m3u8 downloader killing update thread') ''' if not terminated: self.onFinish()
def getDirectM3U8Playlist(M3U8Url, checkExt=True, variantCheck=True, cookieParams={}, checkContent=False, sortWithMaxBitrate=-1, mergeAltAudio=True): if checkExt and not M3U8Url.split('?', 1)[0].endswith('.m3u8'): return [] cm = common() meta = strwithmeta(M3U8Url).meta params, postData = cm.getParamsFromUrlWithMeta(M3U8Url) params.update(cookieParams) retPlaylists = [] try: finallM3U8Url = meta.get('iptv_m3u8_custom_base_link', '') if '' == finallM3U8Url: params['with_metadata'] = True sts, data = cm.getPage(M3U8Url, params, postData) finallM3U8Url = data.meta['url'] else: sts, data = cm.getPage(M3U8Url, params, postData) data = data.strip() m3u8Obj = m3u8.inits(data, finallM3U8Url) if m3u8Obj.is_variant: for playlist in m3u8Obj.playlists: item = {} if not variantCheck or playlist.absolute_uri.split( '?')[-1].endswith('.m3u8'): meta.update({ 'iptv_proto': 'm3u8', 'iptv_bitrate': playlist.stream_info.bandwidth }) item['url'] = strwithmeta(playlist.absolute_uri, meta) else: meta.pop('iptv_proto', None) item['url'] = decorateUrl(playlist.absolute_uri, meta) item['bitrate'] = playlist.stream_info.bandwidth if None != playlist.stream_info.resolution: item['with'] = playlist.stream_info.resolution[0] item['heigth'] = playlist.stream_info.resolution[1] else: item['with'] = 0 item['heigth'] = 0 item['width'] = item['with'] item['height'] = item['heigth'] try: tmpCodecs = playlist.stream_info.codecs.split(',') codecs = [] for c in tmpCodecs[::-1]: codecs.append(c.split('.')[0].strip()) item['codecs'] = ','.join(codecs) except Exception: item['codecs'] = None item['name'] = "bitrate: %s res: %dx%d %s" % (item['bitrate'], \ item['width'], \ item['height'], \ item['codecs'] ) if mergeAltAudio and playlist.alt_audio_streams and item[ 'url'].meta.get('iptv_proto') == 'm3u8': for audio_stream in playlist.alt_audio_streams: audioUrl = strwithmeta(audio_stream.absolute_uri, item['url'].meta) altItem = dict(item) altItem['name'] = '[%s] %s' % (audio_stream.name, altItem['name']) altItem['url'] = decorateUrl( "merge://audio_url|video_url", { 'audio_url': audioUrl, 'video_url': altItem['url'], 'ff_out_container': 'mpegts', 'prefered_merger': 'hlsdl' }) retPlaylists.append(altItem) else: item['alt_audio_streams'] = playlist.alt_audio_streams retPlaylists.append(item) if sortWithMaxBitrate > -1: def __getLinkQuality(itemLink): try: return int(itemLink['bitrate']) except Exception: printExc() return 0 retPlaylists = CSelOneLink( retPlaylists, __getLinkQuality, sortWithMaxBitrate).getSortedLinks() else: if checkContent and 0 == len(m3u8Obj.segments): return [] item = { 'name': 'm3u8', 'url': M3U8Url, 'codec': 'unknown', 'with': 0, 'heigth': 0, 'width': 0, 'height': 0, 'bitrate': 'unknown' } retPlaylists.append(item) except Exception: printExc() return retPlaylists
def getDirectM3U8Playlist(M3U8Url, checkExt=True, variantCheck=True, cookieParams={}, checkContent=False): if checkExt and not M3U8Url.split('?')[0].endswith('.m3u8'): return [] cm = common() meta = strwithmeta(M3U8Url).meta params, postData = cm.getParamsFromUrlWithMeta(M3U8Url) params.update(cookieParams) retPlaylists = [] try: finallM3U8Url = meta.get('iptv_m3u8_custom_base_link', '') if '' == finallM3U8Url: params['return_data'] = False sts, response = cm.getPage(M3U8Url, params, postData) finallM3U8Url = response.geturl() data = response.read().strip() response.close() else: sts, data = cm.getPage(M3U8Url, params, postData) data = data.strip() m3u8Obj = m3u8.inits(data, finallM3U8Url) if m3u8Obj.is_variant: for playlist in m3u8Obj.playlists: item = {} if not variantCheck or playlist.absolute_uri.split( '?')[-1].endswith('.m3u8'): meta.update({ 'iptv_proto': 'm3u8', 'iptv_bitrate': playlist.stream_info.bandwidth }) item['url'] = strwithmeta(playlist.absolute_uri, meta) else: meta.pop('iptv_proto', None) item['url'] = decorateUrl(playlist.absolute_uri, meta) item['bitrate'] = playlist.stream_info.bandwidth if None != playlist.stream_info.resolution: item['with'] = playlist.stream_info.resolution[0] item['heigth'] = playlist.stream_info.resolution[1] else: item['with'] = 0 item['heigth'] = 0 item['width'] = item['with'] item['height'] = item['heigth'] try: tmpCodecs = playlist.stream_info.codecs.split(',') codecs = [] for c in tmpCodecs[::-1]: codecs.append(c.split('.')[0].strip()) item['codecs'] = ','.join(codecs) except Exception: printExc() item['codecs'] = None pass item['name'] = "bitrate: %s res: %dx%d %s" % ( item['bitrate'], \ item['width'], \ item['height'], \ item['codecs'] ) retPlaylists.append(item) else: if checkContent and 0 == len(m3u8Obj.segments): return [] item = { 'name': 'm3u8', 'url': M3U8Url, 'codec': 'unknown', 'with': 0, 'heigth': 0, 'width': 0, 'height': 0, 'bitrate': 'unknown' } retPlaylists.append(item) except Exception: printExc() return retPlaylists
def _cmdFinished(self, code, terminated=False): printDBG("M3U8Downloader._cmdFinished code[%r] terminated[%r] downloadType[%s]" % (code, terminated, self.downloadType)) localStatus = DMHelper.STS.ERROR if terminated: BaseDownloader.updateStatistic(self) localStatus = DMHelper.STS.INTERRUPTED elif self.DOWNLOAD_TYPE.M3U8 == self.downloadType: self.console_appClosed_conn = None self.console_stdoutAvail_conn = None if 0 < len(self.outData): try: m3u8Obj = m3u8.inits(self.outData, self.m3u8Url) # uri given to m3u8 downloader should not be variant, # format should be selected before starting downloader # however if this was not done the firs one will be selected if m3u8Obj.is_variant: if 0 < len(m3u8Obj.playlists): self._startM3U8(self._segUri(m3u8Obj.playlists[-1].absolute_uri)) localStatus = DMHelper.STS.DOWNLOADING else: if 0 < len(m3u8Obj.segments): if not m3u8Obj.is_endlist: self.liveStream = True if -1 == self.startLiveDuration: self.fragmentList = [self._segUri(seg.absolute_uri) for seg in m3u8Obj.segments] else: # some live streams only add new fragments not removing old, # in this case most probably we not want to download old fragments # but start from last N fragments/seconds # self.startLiveDuration self.fragmentList = [] currentDuration = 0 maxFragDuration = m3u8Obj.target_duration for seg in reversed(m3u8Obj.segments): if None != seg.duration: currentDuration += seg.duration else: currentDuration += maxFragDuration self.fragmentList.append(self._segUri(seg.absolute_uri)) if currentDuration >= self.startLiveDuration: break self.fragmentList.reverse() # start update fragment list loop #self.fragmentList = self.fixFragmentsList(self.fragmentList) self._updateM3U8Finished(-1) else: self.fragmentList = [self._segUri(seg.absolute_uri) for seg in m3u8Obj.segments] localStatus = self._startFragment() except: pass elif self.liveStream and self.DOWNLOAD_TYPE.WAITTING == self.downloadType: printDBG("m3u8 liveStream waitting finished--------------------------------") localStatus = self._startFragment() else: BaseDownloader.updateStatistic(self) printDBG("m3u8 nextFragment finished: live[%r]: r[%d], l[%d], p[%d]" % (self.liveStream, self.remoteFragmentSize, self.localFileSize, self.m3u8_prevLocalFileSize )) if 0 >= self.localFileSize: if not self.liveStream: localStatus = DMHelper.STS.ERROR else: localStatus = self._startFragment() #elif not self.liveStream and self.remoteFragmentSize > 0 and self.remoteFragmentSize > (self.localFileSize - self.m3u8_prevLocalFileSize): # localStatus = DMHelper.STS.INTERRUPTED elif 0 < (self.localFileSize - self.m3u8_prevLocalFileSize): localStatus = self._startFragment() elif 0 == (self.localFileSize - self.m3u8_prevLocalFileSize): localStatus = self._startFragment(True) # retry else: localStatus = DMHelper.STS.INTERRUPTED self.status = localStatus if DMHelper.STS.DOWNLOADING == self.status: return # clean up at finish if self.M3U8Updater: self.M3U8Updater_appClosed_conn = None self.M3U8Updater_stdoutAvail_conn = None self.M3U8Updater = None self.liveStream = False if self.console: self.console_appClosed_conn = None self.console_stdoutAvail_conn = None self.console.sendCtrlC() # kill # produce zombies self.console = None ''' if None != self.updateThread: if self.updateThread.Thread.isAlive(): # give some time for update thread to finish sleep(self.MIN_REFRESH_DELAY) printDBG('m3u8 downloader killing update thread') ''' if not terminated: self.onFinish()