class M3U8Downloader(BaseDownloader): MIN_REFRESH_DELAY = 1 MAX_RETRIES = 3 WGET_TIMEOUT = 10 LIVE_START_OFFSET = 120 # wget status WGET_STS = enum(NONE='WGET_NONE', CONNECTING='WGET_CONNECTING', DOWNLOADING='WGET_DOWNLOADING', ENDED='WGET_ENDED') # local status DOWNLOAD_TYPE = enum(M3U8='TYPE_M3U8', SEGMENT='TYPE_SEGMENT', WAITTING='TYPE_WAITTING') def __init__(self): printDBG('M3U8Downloader.__init__ ----------------------------------') BaseDownloader.__init__(self) self.wgetStatus = self.WGET_STS.NONE # instance of E2 console self.console = eConsoleAppContainer() self.iptv_sys = None # M3U8 list updater self.M3U8Updater = eConsoleAppContainer() self.M3U8Updater_appClosed_conn = eConnectCallback( self.M3U8Updater.appClosed, self._updateM3U8Finished) self.M3U8Updater_stdoutAvail_conn = eConnectCallback( self.M3U8Updater.stdoutAvail, self._updateM3U8DataAvail) self.M3U8ListData = '' self.M3U8UpdaterRefreshDelay = 0 self.refreshDelay = M3U8Downloader.MIN_REFRESH_DELAY # get only last fragments from first list, to satisfy specified duration in seconds # -1 means, starts from beginning self.startLiveDuration = M3U8Downloader.LIVE_START_OFFSET # 0 means, starts from beginning self.skipFirstSegFromList = 0 self.addStampToUrl = False self.totalDuration = -1 self.downloadDuration = 0 self.fragmentDurationList = [] self.maxTriesAtStart = 0 def __del__(self): printDBG("M3U8Downloader.__del__ ----------------------------------") def getName(self): return "wget m3u8" def isWorkingCorrectly(self, callBackFun): self.iptv_sys = iptv_system( DMHelper.GET_WGET_PATH() + " -V 2>&1 ", boundFunction(self._checkWorkingCallBack, callBackFun)) def isLiveStream(self): return self.liveStream def _checkWorkingCallBack(self, callBackFun, code, data): reason = '' sts = True if code != 0: sts = False reason = data self.iptv_sys = None callBackFun(sts, reason) def start(self, url, filePath, params={}): ''' Owervrite start from BaseDownloader ''' self.filePath = filePath self.downloaderParams = params self.fileExtension = '' # should be implemented in future self.status = DMHelper.STS.DOWNLOADING self.updateThread = None self.fragmentList = [] self.lastMediaSequence = -1 self.currentFragment = -1 self.tries = 0 self.liveStream = False self.skipFirstSegFromList = strwithmeta(url).meta.get( 'iptv_m3u8_skip_seg', 0) self.m3u8Url = url self._startM3U8() self.onStart() return BaseDownloader.CODE_OK def _getTimeout(self): if self.liveStream: return self.WGET_TIMEOUT else: return 2 * self.WGET_TIMEOUT def _addTimeStampToUrl(self, m3u8Url): if self.addStampToUrl: if '?' in m3u8Url: m3u8Url += '&iptv_stamp=' else: m3u8Url += '?iptv_stamp=' m3u8Url += ('%s' % time()) return m3u8Url def _updateM3U8Finished(self, code=0): printDBG('m3u8 _updateM3U8Finished update code[%d]--- ' % (code)) if self.liveStream and self.M3U8Updater: if 0 < len(self.M3U8ListData) and 0 == code: try: m3u8Obj = m3u8.inits(self.M3U8ListData, self.m3u8Url) if self.liveStream and not m3u8Obj.is_variant: self.refreshDelay = int(m3u8Obj.target_duration) if self.refreshDelay < 5: self.refreshDelay = 5 if 0 < len(m3u8Obj.segments): newFragments = [ self._segUri(seg.absolute_uri) for seg in m3u8Obj.segments ] #self.mergeFragmentsList(newFragments) self.mergeFragmentsListWithChecking( newFragments, m3u8Obj.media_sequence + len(m3u8Obj.segments)) printDBG( 'm3u8 _updateM3U8Finished list updated ---') except Exception: printDBG( "m3u8 _updateM3U8Finished exception url[%s] data[%s]" % (self.m3u8Url, self.M3U8ListData)) else: printDBG('m3u8 _updateM3U8Finished no data ---') # hardcode self.M3U8UpdaterRefreshDelay += 1 if self.refreshDelay < self.M3U8UpdaterRefreshDelay or 0 != code: self.M3U8UpdaterRefreshDelay = 0 self.M3U8ListData = '' m3u8Url = self._addTimeStampToUrl(self.m3u8Url) printDBG( ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> [%s]" % m3u8Url) cmd = DMHelper.getBaseWgetCmd(self.downloaderParams) + ( ' --tries=0 --timeout=%d ' % self._getTimeout()) + '"' + m3u8Url + '" -O - 2> /dev/null' printDBG("m3u8 _updateM3U8Finished download cmd[%s]" % cmd) self.M3U8Updater.execute(E2PrioFix(cmd)) return else: self.M3U8Updater.execute(E2PrioFix("sleep 1")) return printDBG( "|||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||") printDBG( "||||||||||||| m3u8 _updateM3U8Finished FINISHED |||||||||||||") printDBG( "|||||||||||||||||||||||||||||||||||||||||||||||||||||||||||||") def _updateM3U8DataAvail(self, data): if None != data and 0 < len(data): self.M3U8ListData += data def mergeFragmentsListWithChecking_OLD(self, newFragments, media_sequence=-1): #newFragments = self.fixFragmentsList(newFragments) try: idx = newFragments.index(self.fragmentList[-1]) newFragments = newFragments[idx + 1:] except Exception: printDBG( 'm3u8 update thread - last fragment from last list not available in new list!' ) tmpList = [] for item in reversed(newFragments): if item in self.fragmentList: break tmpList.insert(0, item) if 0 < len(tmpList): printDBG( ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> DODANO[%d]" % len(tmpList)) if 21 < self.currentFragment: idx = self.currentFragment - 20 self.fragmentList = self.fragmentList[idx:] self.currentFragment = 20 self.fragmentList.extend(tmpList) def mergeFragmentsListWithChecking(self, newFragments, media_sequence=-1): tmpList = [] #DebugToFile('last[%s] new[%s] = %s' % (self.lastMediaSequence, media_sequence, newFragments)) if self.lastMediaSequence > 0 and media_sequence > 0: if media_sequence > self.lastMediaSequence: toAdd = media_sequence - self.lastMediaSequence if toAdd > len(newFragments): toAdd = len(newFragments) tmpList = newFragments[-toAdd:] self.lastMediaSequence = media_sequence else: try: tmpCurrFragmentList = [ seg[seg.rfind('/') + 1:] for seg in self.fragmentList ] tmpNewFragments = [ seg[seg.rfind('/') + 1:] for seg in newFragments ] idx = tmpNewFragments.index(tmpCurrFragmentList[-1]) newFragments = newFragments[idx + 1:] except Exception: printDBG( 'm3u8 update thread - last fragment from last list not available in new list!' ) for item in reversed(newFragments): if item in self.fragmentList: break tmpList.insert(0, item) if 0 < len(tmpList): printDBG( ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> DODANO[%d]" % len(tmpList)) if 21 < self.currentFragment: idx = self.currentFragment - 20 self.fragmentList = self.fragmentList[idx:] self.currentFragment = 20 self.fragmentList.extend(tmpList) #DebugToFile(">> %s" % self.fragmentList) ''' def fixFragmentsList(self, newFragments): retList = [] for idx in range(len(newFragments)): if 0 == idx: retList.append(newFragments[0]) continue if newFragments[idx] != newFragments[idx-1]: retList.append(newFragments[idx]) return retList def mergeFragmentsList(self, newFragments): try: # merge fragments list idx = -1 if 0 < len(self.fragmentList): try: idx = newFragments.index(self.fragmentList[-1]) except Exception: printDBG('m3u8 update thread - last fragment from last list not available in new list!') if 0 <= idx: if (idx+1) < len(newFragments): self.fragmentList.extend(newFragments[idx+1:]) else: self.fragmentList.extend(newFragments) except Exception: pass #printDBG("===========================================================") #printDBG("%r" % self.fragmentList) #printDBG("===========================================================") ''' def _startM3U8(self, wait=0): self.outData = '' ############################################################################## # frist download m3u8 conntent ############################################################################## self.downloadType = self.DOWNLOAD_TYPE.M3U8 m3u8Url = self._addTimeStampToUrl(self.m3u8Url) cmd = DMHelper.getBaseWgetCmd(self.downloaderParams) + ( ' --tries=0 --timeout=%d ' % self._getTimeout()) + '"' + m3u8Url + '" -O - 2> /dev/null' if wait > 0: cmd = (' sleep %s && ' % wait) + cmd printDBG("Download cmd[%s]" % cmd) self.console_appClosed_conn = eConnectCallback(self.console.appClosed, self._cmdFinished) self.console_stdoutAvail_conn = eConnectCallback( self.console.stdoutAvail, self._dataAvail) self.console.execute(E2PrioFix(cmd)) ############################################################################## def _startFragment(self, tryAgain=False): printDBG("_startFragment tryAgain[%r]" % tryAgain) self.outData = '' self.remoteFragmentSize = -1 self.remoteFragmentType = 'unknown' if 0 > self.localFileSize: self.m3u8_prevLocalFileSize = 0 else: self.m3u8_prevLocalFileSize = self.localFileSize ############################################################################## # frist download nextFragment conntent ############################################################################## self.downloadType = self.DOWNLOAD_TYPE.SEGMENT if None != self.console: self.console_appClosed_conn = None self.console_stderrAvail_conn = None #self.console = eConsoleAppContainer() self.console_appClosed_conn = eConnectCallback(self.console.appClosed, self._cmdFinished) self.console_stderrAvail_conn = eConnectCallback( self.console.stderrAvail, self._dataAvail) if tryAgain and self.tries >= self.MAX_RETRIES: if not self.liveStream: return DMHelper.STS.INTERRUPTED else: # even if fragment is lost this is not big problem, download next one, # this is a live stream this can happen :) tryAgain = False currentFragment = None if False == tryAgain: self.tries = 0 if (self.currentFragment + 1) < len(self.fragmentList): self.currentFragment += 1 currentFragment = self.fragmentList[self.currentFragment] else: self.tries += 1 currentFragment = self.fragmentList[self.currentFragment] if None != currentFragment: self.wgetStatus = self.WGET_STS.CONNECTING cmd = DMHelper.getBaseWgetCmd(self.downloaderParams) + ( ' --tries=1 --timeout=%d ' % self._getTimeout() ) + '"' + currentFragment + '" -O - >> "' + self.filePath + '"' printDBG("Download cmd[%s]" % cmd) self.console.execute(E2PrioFix(cmd)) #DebugToFile(currentFragment) return DMHelper.STS.DOWNLOADING else: if self.liveStream: # we are in live so wait for new fragments printDBG( "m3u8 downloader - wait for new fragments ----------------------------------------------------------------" ) self.downloadType = self.DOWNLOAD_TYPE.WAITTING self.console.execute(E2PrioFix("sleep 2")) return DMHelper.STS.DOWNLOADING else: return DMHelper.STS.DOWNLOADED ############################################################################## def _dataAvail(self, data): if None != data: self.outData += data if self.DOWNLOAD_TYPE.M3U8 == self.downloadType: return if self.WGET_STS.CONNECTING == self.wgetStatus: self.outData += data lines = self.outData.replace('\r', '\n').split('\n') for idx in range(len(lines)): if lines[idx].startswith('Length:'): match = re.search( "Length: ([0-9]+?) \([^)]+?\) (\[[^]]+?\])", lines[idx]) if match: self.remoteFragmentSize = int(match.group(1)) self.remoteFragmentType = match.group(2) elif lines[idx].startswith('Saving to:'): self.console_stderrAvail_conn = None self.wgetStatus = self.WGET_STS.DOWNLOADING break def _terminate(self): printDBG("M3U8Downloader._terminate") if None != self.iptv_sys: self.iptv_sys.kill() self.iptv_sys = None if DMHelper.STS.DOWNLOADING == self.status: if self.console: self.console.sendCtrlC() # kill # produce zombies self._cmdFinished(-1, True) return BaseDownloader.CODE_OK return BaseDownloader.CODE_NOT_DOWNLOADING def _segUri(self, uri): return uri.split('iptv_stamp')[0] def _cmdFinished(self, code, terminated=False): printDBG( "M3U8Downloader._cmdFinished code[%r] terminated[%r] downloadType[%s]" % (code, terminated, self.downloadType)) localStatus = DMHelper.STS.ERROR if terminated: BaseDownloader.updateStatistic(self) localStatus = DMHelper.STS.INTERRUPTED elif self.DOWNLOAD_TYPE.M3U8 == self.downloadType: self.console_appClosed_conn = None self.console_stdoutAvail_conn = None if 0 < len(self.outData): try: m3u8Obj = m3u8.inits(self.outData, self.m3u8Url) # uri given to m3u8 downloader should not be variant, # format should be selected before starting downloader # however if this was not done the firs one will be selected if m3u8Obj.is_variant: if 0 < len(m3u8Obj.playlists): self.m3u8Url = self._segUri( m3u8Obj.playlists[-1].absolute_uri) self._startM3U8() localStatus = DMHelper.STS.DOWNLOADING else: if 0 < len(m3u8Obj.segments): if not m3u8Obj.is_endlist: self.liveStream = True if -1 == self.startLiveDuration: self.fragmentList = [ self._segUri(seg.absolute_uri) for seg in m3u8Obj.segments ] else: # some live streams only add new fragments not removing old, # in this case most probably we not want to download old fragments # but start from last N fragments/seconds # self.startLiveDuration self.fragmentList = [] currentDuration = 0 maxFragDuration = m3u8Obj.target_duration for seg in reversed(m3u8Obj.segments): if None != seg.duration: currentDuration += seg.duration else: currentDuration += maxFragDuration self.fragmentList.append( self._segUri(seg.absolute_uri)) if currentDuration >= self.startLiveDuration: break self.fragmentList.reverse() if len(m3u8Obj.segments) == len( self.fragmentList) and len( self.fragmentList ) > self.skipFirstSegFromList: self.fragmentList = self.fragmentList[ self.skipFirstSegFromList:] self.lastMediaSequence = m3u8Obj.media_sequence + len( m3u8Obj.segments) # start update fragment list loop #self.fragmentList = self.fixFragmentsList(self.fragmentList) self._updateM3U8Finished(-1) else: self.fragmentList = [ self._segUri(seg.absolute_uri) for seg in m3u8Obj.segments ] try: self.totalDuration = 0 self.fragmentDurationList = [] for seg in m3u8Obj.segments: self.totalDuration += seg.duration self.fragmentDurationList.append( seg.duration) except Exception: printExc() self.totalDuration = -1 self.fragmentDurationList = [] localStatus = self._startFragment() except Exception: pass printDBG(">>>>>>>>>>>>>>>>>> localStatus [%s] tries[%d]" % (localStatus, self.tries)) if localStatus == DMHelper.STS.ERROR and self.tries < self.maxTriesAtStart: self.console_appClosed_conn = None self.console_stdoutAvail_conn = None self.tries += 1 self._startM3U8(self.MIN_REFRESH_DELAY) return else: self.tries = 0 elif self.liveStream and self.DOWNLOAD_TYPE.WAITTING == self.downloadType: printDBG( "m3u8 liveStream waitting finished--------------------------------" ) localStatus = self._startFragment() else: BaseDownloader.updateStatistic(self) printDBG( "m3u8 nextFragment finished: live[%r]: r[%d], l[%d], p[%d]" % (self.liveStream, self.remoteFragmentSize, self.localFileSize, self.m3u8_prevLocalFileSize)) if 0 >= self.localFileSize: if not self.liveStream: localStatus = DMHelper.STS.ERROR else: localStatus = self._startFragment() #elif not self.liveStream and self.remoteFragmentSize > 0 and self.remoteFragmentSize > (self.localFileSize - self.m3u8_prevLocalFileSize): # localStatus = DMHelper.STS.INTERRUPTED elif 0 < (self.localFileSize - self.m3u8_prevLocalFileSize): if self.totalDuration > 0: try: self.downloadDuration += self.fragmentDurationList[ self.currentFragment] except Exception: printExc() localStatus = self._startFragment() elif 0 == (self.localFileSize - self.m3u8_prevLocalFileSize): localStatus = self._startFragment(True) # retry else: localStatus = DMHelper.STS.INTERRUPTED self.status = localStatus if DMHelper.STS.DOWNLOADING == self.status: return # clean up at finish if self.M3U8Updater: self.M3U8Updater_appClosed_conn = None self.M3U8Updater_stdoutAvail_conn = None self.M3U8Updater = None self.liveStream = False if self.console: self.console_appClosed_conn = None self.console_stdoutAvail_conn = None self.console.sendCtrlC() # kill # produce zombies self.console = None ''' if None != self.updateThread: if self.updateThread.Thread.isAlive(): # give some time for update thread to finish sleep(self.MIN_REFRESH_DELAY) printDBG('m3u8 downloader killing update thread') ''' if not terminated: self.onFinish() def hasDurationInfo(self): return True def getTotalFileDuration(self): # total duration in seconds return int(self.totalDuration) def getDownloadedFileDuration(self): # downloaded duration in seconds return int(self.downloadDuration)
class WgetDownloader(BaseDownloader): # wget status WGET_STS = enum(NONE='WGET_NONE', CONNECTING='WGET_CONNECTING', DOWNLOADING='WGET_DOWNLOADING', ENDED='WGET_ENDED') # wget status INFO = enum(FROM_FILE='INFO_FROM_FILE', FROM_DOTS='INFO_FROM_DOTS') def __init__(self): printDBG('WgetDownloader.__init__ ') BaseDownloader.__init__(self) self.wgetStatus = self.WGET_STS.NONE # instance of E2 console self.console = None self.iptv_sys = None self.curContinueRetry = 0 self.maxContinueRetry = 0 self.downloadCmd = '' self.remoteContentType = None self.lastErrorCode = None self.lastErrorDesc = '' def __del__(self): printDBG("WgetDownloader.__del__ ") def getName(self): return "wget" def getLastError(self): return self.lastErrorCode, self.lastErrorDesc def _setLastError(self, code): # map Exit Status to message - https://www.gnu.org/software/wget/manual/html_node/Exit-Status.html self.lastErrorCode = code if code == 0: self.lastErrorDesc = "No problems occurred." elif code == 1: self.lastErrorDesc = "Generic error code." elif code == 2: self.lastErrorDesc = "Parse error." elif code == 3: self.lastErrorDesc = "File I/O error." elif code == 4: self.lastErrorDesc = "Network failure." elif code == 5: self.lastErrorDesc = "SSL verification failure." elif code == 6: self.lastErrorDesc = "Username/password authentication failure." elif code == 7: self.lastErrorDesc = "Protocol errors." elif code == 8: self.lastErrorDesc = "Server issued an error response." else: self.lastErrorDesc = 'Unknown error code.' def isWorkingCorrectly(self, callBackFun): self.iptv_sys = iptv_system( DMHelper.GET_WGET_PATH() + " -V 2>&1 ", boundFunction(self._checkWorkingCallBack, callBackFun)) def getMimeType(self): return self.remoteContentType def _checkWorkingCallBack(self, callBackFun, code, data): reason = '' sts = True if code != 0: sts = False reason = data self.iptv_sys = None callBackFun(sts, reason) def start(self, url, filePath, params={}, info_from=None, retries=0): ''' Owervrite start from BaseDownloader ''' self.url = url self.filePath = filePath self.downloaderParams = params self.fileExtension = '' # should be implemented in future self.outData = '' self.contentType = 'unknown' if None == info_from: info_from = WgetDownloader.INFO.FROM_FILE self.infoFrom = info_from if self.infoFrom == WgetDownloader.INFO.FROM_DOTS: info = "--progress=dot:default" else: info = "" # remove file if exists if fileExists(self.filePath): rm(self.filePath) self.downloadCmd = DMHelper.getBaseWgetCmd(self.downloaderParams) + ( ' %s -t %d ' % (info, retries)) + '"' + self.url + '" -O "' + self.filePath + '"' printDBG("Download cmd[%s]" % self.downloadCmd) if self.downloaderParams.get('iptv_wget_continue', False): self.maxContinueRetry = 3 self.console = eConsoleAppContainer() self.console_appClosed_conn = eConnectCallback(self.console.appClosed, self._cmdFinished) self.console_stderrAvail_conn = eConnectCallback( self.console.stderrAvail, self._dataAvail) self.console.execute(E2PrioFix(self.downloadCmd)) self.wgetStatus = self.WGET_STS.CONNECTING self.status = DMHelper.STS.DOWNLOADING self.onStart() return BaseDownloader.CODE_OK def _dataAvail(self, data): if None != data: self.outData += data if self.infoFrom == WgetDownloader.INFO.FROM_FILE: if 'Saving to:' in self.outData: self.console_stderrAvail_conn = None lines = self.outData.replace('\r', '\n').split('\n') for idx in range(len(lines)): if 'Length:' in lines[idx]: match = re.search(" ([0-9]+?) ", lines[idx]) if match: self.remoteFileSize = int(match.group(1)) match = re.search("(\[[^]]+?\])", lines[idx]) if match: self.remoteContentType = match.group(1) self.outData = '' elif self.WGET_STS.CONNECTING == self.wgetStatus: self.outData += data lines = self.outData.replace('\r', '\n').split('\n') for idx in range(len(lines)): if lines[idx].startswith('Length:'): match = re.search( "Length: ([0-9]+?) \([^)]+?\) (\[[^]]+?\])", lines[idx]) if match: self.remoteFileSize = int(match.group(1)) self.remoteContentType = match.group(2) elif lines[idx].startswith('Saving to:'): if len(lines) > idx: self.outData = '\n'.join(lines[idx + 1:]) else: self.outData = '' self.wgetStatus = self.WGET_STS.DOWNLOADING if self.infoFrom != WgetDownloader.INFO.FROM_DOTS: self.console_stderrAvail_conn = None break def _terminate(self): printDBG("WgetDownloader._terminate") if None != self.iptv_sys: self.iptv_sys.kill() self.iptv_sys = None if DMHelper.STS.DOWNLOADING == self.status: if self.console: self.console.sendCtrlC() # kill # produce zombies self._cmdFinished(-1, True) return BaseDownloader.CODE_OK return BaseDownloader.CODE_NOT_DOWNLOADING def _cmdFinished(self, code, terminated=False): printDBG("WgetDownloader._cmdFinished code[%r] terminated[%r]" % (code, terminated)) # When finished updateStatistic based on file size on disk BaseDownloader.updateStatistic(self) printDBG( "WgetDownloader._cmdFinished remoteFileSize[%r] localFileSize[%r]" % (self.remoteFileSize, self.localFileSize)) if not terminated and self.remoteFileSize > 0 \ and self.remoteFileSize > self.localFileSize \ and self.curContinueRetry < self.maxContinueRetry: self.curContinueRetry += 1 self.console.execute(E2PrioFix(self.downloadCmd)) return self._setLastError(code) # break circular references self.console_appClosed_conn = None self.console_stderrAvail_conn = None self.console = None self.wgetStatus = self.WGET_STS.ENDED if terminated: self.status = DMHelper.STS.INTERRUPTED elif 0 >= self.localFileSize: self.status = DMHelper.STS.ERROR elif self.remoteFileSize > 0 and self.remoteFileSize > self.localFileSize: self.status = DMHelper.STS.INTERRUPTED else: self.status = DMHelper.STS.DOWNLOADED printDBG("WgetDownloader._cmdFinished status [%s]" % (self.status)) if not terminated: self.onFinish() def updateStatistic(self): if self.infoFrom == WgetDownloader.INFO.FROM_FILE: BaseDownloader.updateStatistic(self) return if self.WGET_STS.DOWNLOADING == self.wgetStatus: print(self.outData) dataLen = len(self.outData) for idx in range(dataLen): if idx + 1 < dataLen: # default style - one dot = 1K if '.' == self.outData[idx] and self.outData[idx + 1] in [ '.', ' ' ]: self.localFileSize += 1024 else: self.outData = self.outData[idx:] break BaseDownloader._updateStatistic(self)
class IPTVUpdateWindow(Screen): skin = """ <screen name="IPTVUpdateMainWindow" position="center,center" size="620,440" title="" > <widget name="sub_title" position="10,10" zPosition="2" size="600,35" valign="center" halign="left" font="Regular;22" transparent="1" foregroundColor="white" /> <widget name="list" position="10,50" zPosition="1" size="600,380" transparent="1" scrollbarMode="showOnDemand" /> <widget name="console" position="40,200" zPosition="2" size="540,80" valign="center" halign="center" font="Regular;34" transparent="0" foregroundColor="white" backgroundColor="black"/> </screen>""" ICONS = [ 'iconwait1.png', 'iconwait2.png', 'iconwait3.png', 'icondone.png', 'iconerror.png', 'iconwarning.png', 'iconcancelled.png' ] ICON = enum(WAITING=0, PROCESSING=1, PROCESSING_NOT_BREAK=2, PROCESSED=3, ERROR=4, WARNING=5, CANCELLED=6) def __init__(self, session, updateObjImpl, autoStart=True): printDBG( "IPTVUpdateMainWindow.__init__ -------------------------------") Screen.__init__(self, session) self.autoStart = autoStart self.updateObjImpl = updateObjImpl self.updateObjImpl.setStepFinishedCallBack(self.stepFinished) self.setup_title = self.updateObjImpl.getSetupTitle() self["sub_title"] = Label(_(" ")) self["console"] = Label(_("> Press OK to start <")) self["actions"] = ActionMap(["SetupActions", "ColorActions"], { "cancel": self.keyExit, "ok": self.keyOK, }, -2) self.list = [] self["list"] = IPTVUpdateList( [GetIPTVDMImgDir(x) for x in IPTVUpdateWindow.ICONS]) self.currStep = 0 self.onLayoutFinish.append(self.layoutFinished) self.onClose.append(self.__onClose) self.status = None def __del__(self): printDBG( "IPTVUpdateMainWindow.__del__ -------------------------------") def __onClose(self): printDBG( "IPTVUpdateMainWindow.__onClose -----------------------------") self.updateObjImpl.setStepFinishedCallBack(None) self.updateObjImpl.terminate() self.onClose.remove(self.__onClose) self.onLayoutFinish.remove(self.layoutFinished) self.updateObjImpl = None self.list = [] self["list"].setList([]) def layoutFinished(self): self.setTitle(self.updateObjImpl.getTitle()) self["sub_title"].setText(self.updateObjImpl.getSubTitle()) self["list"].setSelectionState(enabled=False) self.preparUpdateStepsList() if self.autoStart: self.doStart() else: self["console"].show() def doStart(self): if 0 < len(self.list): self.currStep = 0 self.status = 'working' self.stepExecute() self["console"].hide() else: self["console"].setText(_("No steps to execute.")) def reloadList(self): self["list"].hide() self["list"].setList([(x, ) for x in self.list]) self["list"].show() def keyOK(self): if not self.autoStart and None == self.status: self.doStart() else: currItem = self["list"].getCurrent() if self.status not in [None, 'working']: artItem = ArticleContent(title=currItem['title'], text=currItem['info'], images=[]) self.session.open(ArticleView, artItem) def keyExit(self): if 'working' == self.status and not self.list[self.currStep].get( 'breakable', False): self.session.open(MessageBox, _("Step [%s] cannot be aborted. Please wait."), type=MessageBox.TYPE_INFO, timeout=5) else: self.close() def preparUpdateStepsList(self): self.list = self.updateObjImpl.getStepsList() for item in self.list: item.update({'icon': self.ICON.WAITING}) self.reloadList() def stepExecute(self): self["list"].moveToIndex(self.currStep) if self.list[self.currStep].get('breakable', False): self.list[self.currStep].update({ 'info': _("During processing, please wait."), 'icon': self.ICON.PROCESSING }) else: self.list[self.currStep].update({ 'info': _("During processing, please do not interrupt."), 'icon': self.ICON.PROCESSING_NOT_BREAK }) self.reloadList() if self.updateObjImpl.isReadyToExecuteStep(self.currStep): self.list[self.currStep]['execFunction']() else: raise Exception( "IPTVUpdateMainWindow.stepExecute seems that last step has not been finished." ) def stepFinished(self, stsCode, msg): printDBG('IPTVUpdateMainWindow.stepFinished stsCode[%d], msg[%s]' % (stsCode, msg)) nextStep = True if 0 != stsCode and self.list[self.currStep].get('repeatCount', 0) > 0: self.list[self.currStep]['repeatCount'] -= 1 self.stepExecute() return if -1 == stsCode and not self.list[self.currStep]['ignoreError']: nextStep = False self.status = 'error' self.list[self.currStep].update({ 'info': msg, 'icon': self.ICON.ERROR }) # cancel other steps currStep = self.currStep + 1 while currStep < len(self.list): self.list[currStep].update({ 'info': _("Aborted"), 'icon': self.ICON.CANCELLED }) currStep += 1 elif 0 != stsCode: self.list[self.currStep].update({ 'info': msg, 'icon': self.ICON.WARNING }) else: self.list[self.currStep].update({ 'info': msg, 'icon': self.ICON.PROCESSED }) if nextStep: if self.currStep + 1 < len(self.list): self.currStep += 1 self.stepExecute() else: self.status = 'done' if self.updateObjImpl.finalize(): self.close() return else: self["list"].setSelectionState(enabled=True) else: self.status = 'error' if self.updateObjImpl.finalize(False, msg): self.close() return else: self["list"].setSelectionState(enabled=True) self.reloadList()
class RtmpDownloader(BaseDownloader): URI_TAB = ['rtmp://', 'rtmpt://', 'rtmpe://', 'rtmpte://', 'rtmps://'] # rtmp status RTMP_STS = enum(NONE='RTMP_NONE', CONNECTING='RTMP_CONNECTING', DOWNLOADING='RTMP_DOWNLOADING', ENDED='RTMP_ENDED') # rtmp status INFO = enum(FROM_FILE='INFO_FROM_FILE', FROM_DOTS='INFO_FROM_DOTS') def __init__(self): printDBG('RtmpDownloader.__init__ ----------------------------------') BaseDownloader.__init__(self) self.rtmpStatus = self.RTMP_STS.NONE # instance of E2 console self.console = None self.iptv_sys = None def __del__(self): printDBG("RtmpDownloader.__del__ ----------------------------------") def getName(self): return "rtmpdump" def isWorkingCorrectly(self, callBackFun): self.iptv_sys = iptv_system( DMHelper.GET_RTMPDUMP_PATH() + " -h 2>&1 ", boundFunction(self._checkWorkingCallBack, callBackFun)) def _checkWorkingCallBack(self, callBackFun, code, data): reason = '' sts = True if code != 0: sts = False reason = data self.iptv_sys = None callBackFun(sts, reason) def _getCMD(self, url): paramsL = [ 'help', 'url', 'rtmp', 'host', 'port', 'socks', 'protocol', 'playpath', 'playlist', 'swfUrl', 'tcUrl', 'pageUrl', 'app', 'swfhash', 'swfsize', 'swfVfy', 'swfAge', 'auth', 'conn', 'flashVer', 'live', 'subscribe', 'realtime', 'flv', 'resume', 'timeout', 'start', 'stop', 'token', 'jtv', 'weeb', 'hashes', 'buffer', 'skip', 'quiet', 'verbose', 'debug' ] paramsS = [ 'h', 'i', 'r', 'n', 'c', 'S', 'l', 'y', 'Y', 's', 't', 'p', 'a', 'w', 'x', 'W', 'X', 'u', 'C', 'f', 'v', 'd', 'R', 'o', 'e', 'e', 'A', 'B', 'T', 'j', 'J', '#', 'b', 'k', 'q', 'V', 'z' ] paramsRequireValue = ['pageUrl'] url = 'rtmp ' + url tmpTab = url.split(' ') parameter = None value = '' cmd = '' def _processItem(item, parameter, value, cmd): printDBG(item) if (item in paramsL and (parameter not in paramsRequireValue or '' != value)) or '##fake##' == item: if None != parameter: cmd += ' --' + parameter.strip() if '' != value: cmd += '="%s"' % value.strip() value = '' elif '' != value: printDBG( '_getCMD.RtmpDownloader no parameters for value[%s]' % value.strip()) if 0 < len(cmd): cmd = cmd[:-1] + ' %s"' % value.strip() value = '' parameter = item else: if '' != value: value += ' ' value += item return item, parameter, value, cmd # pre-processing params = [] for item in tmpTab: tmp = item.find('=') if -1 < tmp and item[:tmp] in paramsL: params.append(item[:tmp]) if 'live' != item[:tmp]: params.append(item[tmp + 1:]) else: params.append(item) for item in params: item, parameter, value, cmd = _processItem(item, parameter, value, cmd) item, parameter, value, cmd = _processItem('##fake##', parameter, value, cmd) return cmd def start(self, url, filePath, params={}, info_from=None, retries=0): ''' Owervrite start from BaseDownloader ''' self.url = url self.filePath = filePath self.downloaderParams = params self.fileExtension = '' # should be implemented in future rtmpdump_url = self._getCMD(url) if 0: #rtmpdump -r rtmp://5.79.71.195/stream/ --playpath=3001_goldvod --swfUrl=http://goldvod.tv:81/j/jwplayer/jwplayer.flash.swf --pageUrl=http://goldvod.tv/tv-online/tvp1.html -o tvp1.flv tmpTab = url.split(' ') rtmpdump_url = '"' + tmpTab[0].strip() + '"' del tmpTab[0] prevflashVer = '' for item in tmpTab: item = item.strip() # ignore empty and live params if '' != prevflashVer: rtmpdump_url += ' --' + prevflashVer[ 0:-1] + ' ' + item + '"' prevflashVer = '' continue idx = item.find('=') if -1 == idx: continue argName = item[:idx] argValue = item[idx + 1:] if 'live' in argName: item = 'live' else: item = '%s="%s"' % (argName, argValue) if 'flashVer' == argName: prevflashVer = item continue rtmpdump_url += ' --' + item cmd = DMHelper.GET_RTMPDUMP_PATH( ) + " " + rtmpdump_url + ' --realtime -o "' + self.filePath + '" > /dev/null 2>&1' printDBG("rtmpdump cmd[%s]" % cmd) self.console = eConsoleAppContainer() self.console_appClosed_conn = eConnectCallback(self.console.appClosed, self._cmdFinished) #self.console.stderrAvail.append( self._dataAvail ) self.console.execute(cmd) self.rtmpStatus = self.RTMP_STS.CONNECTING self.status = DMHelper.STS.DOWNLOADING self.onStart() return BaseDownloader.CODE_OK def _terminate(self): printDBG("WgetDownloader._terminate") if None != self.iptv_sys: self.iptv_sys.kill() self.iptv_sys = None if DMHelper.STS.DOWNLOADING == self.status: if self.console: self.console.sendCtrlC() # kill # produce zombies self._cmdFinished(-1, True) return BaseDownloader.CODE_OK return BaseDownloader.CODE_NOT_DOWNLOADING def _cmdFinished(self, code, terminated=False): printDBG("RtmpDownloader._cmdFinished code[%r] terminated[%r]" % (code, terminated)) # break circular references self.console_appClosed_conn = None self.console = None self.rtmpStatus = self.RTMP_STS.ENDED # When finished updateStatistic based on file sie on disk BaseDownloader.updateStatistic(self) if terminated: self.status = DMHelper.STS.INTERRUPTED elif 0 >= self.localFileSize: self.status = DMHelper.STS.ERROR elif self.remoteFileSize > 0 and self.remoteFileSize > self.localFileSize: self.status = DMHelper.STS.INTERRUPTED else: self.status = DMHelper.STS.DOWNLOADED if not terminated: self.onFinish() def updateStatistic(self): BaseDownloader.updateStatistic(self) return
class PwgetDownloader(BaseDownloader): # wget status WGET_STS = enum(NONE='WGET_NONE', CONNECTING='WGET_CONNECTING', DOWNLOADING='WGET_DOWNLOADING', ENDED='WGET_ENDED') def __init__(self): printDBG('PwgetDownloader.__init__ ----------------------------------') BaseDownloader.__init__(self) self.wgetStatus = self.WGET_STS.NONE # instance of E2 console self.console = None self.iptv_sys = None def __del__(self): printDBG("PwgetDownloader.__del__ ----------------------------------") def getName(self): return "pwget" def isWorkingCorrectly(self, callBackFun): self.iptv_sys = iptv_system( "python " + DMHelper.GET_PWGET_PATH() + " 2>&1", boundFunction(self._checkWorkingCallBack, callBackFun)) def _checkWorkingCallBack(self, callBackFun, code, data): reason = '' sts = True if 'Usage: python pwget url file' not in data: sts = False reason = data self.iptv_sys = None callBackFun(sts, reason) def start(self, url, filePath, params={}): ''' Owervrite start from BaseDownloader ''' self.url = url self.filePath = filePath self.downloaderParams = params self.fileExtension = '' # should be implemented in future self.outData = '' self.contentType = 'unknown' cmd = "python " + DMHelper.GET_PWGET_PATH( ) + ' "' + self.url + '" "' + self.filePath + '" > /dev/null' printDBG("Download cmd[%s]" % cmd) self.console = eConsoleAppContainer() self.console_appClosed_conn = eConnectCallback(self.console.appClosed, self._cmdFinished) self.console_stderrAvail_conn = eConnectCallback( self.console.stderrAvail, self._dataAvail) self.console.execute(E2PrioFix(cmd)) self.wgetStatus = self.WGET_STS.CONNECTING self.status = DMHelper.STS.DOWNLOADING self.onStart() return BaseDownloader.CODE_OK def _dataAvail(self, data): if None != data: self.outData += data def _terminate(self): printDBG("PwgetDownloader._terminate") if None != self.iptv_sys: self.iptv_sys.kill() self.iptv_sys = None if DMHelper.STS.DOWNLOADING == self.status: if self.console: self.console.sendCtrlC() # kill # produce zombies self._cmdFinished(-1, True) return BaseDownloader.CODE_OK return BaseDownloader.CODE_NOT_DOWNLOADING def _cmdFinished(self, code, terminated=False): printDBG("PwgetDownloader._cmdFinished code[%r] terminated[%r]" % (code, terminated)) # break circular references self.console_appClosed_conn = None self.console_stderrAvail_conn = None self.console = None self.wgetStatus = self.WGET_STS.ENDED # When finished updateStatistic based on file sie on disk BaseDownloader.updateStatistic(self) if not terminated: printDBG("PwgetDownloader._cmdFinished [%s]" % self.outData) match = re.search("Content-Length: ([0-9]+?)[^0-9]", self.outData) if match: self.remoteFileSize = int(match.group(1)) if terminated: self.status = DMHelper.STS.INTERRUPTED elif 0 >= self.localFileSize: self.status = DMHelper.STS.ERROR elif self.remoteFileSize > 0 and self.remoteFileSize > self.localFileSize: self.status = DMHelper.STS.INTERRUPTED else: self.status = DMHelper.STS.DOWNLOADED if not terminated: self.onFinish()
class DMHelper: STATUS_FILE_PATH = '/tmp/iptvdownload' STATUS_FILE_EXT = '.txt' STS = enum(WAITING='STS_WAITING', DOWNLOADING='STS_DOWNLOADING', DOWNLOADED='STS_DOWNLOADED', INTERRUPTED='STS_INTERRUPTED', ERROR='STS_ERROR') DOWNLOAD_TYPE = enum(INITIAL='INIT_DOWNLOAD', CONTINUE='CONTINUE_DOWNLOAD', RETRY='RETRY_DOWNLOAD') # DOWNLOADER_TYPE = enum(WGET='WGET_DOWNLOADER', F4F='F4F_DOWNLOADER') HEADER_PARAMS = [{ 'marker': 'Cookie=', 'name': 'Cookie' }, { 'marker': 'Referer=', 'name': 'Referer' }, { 'marker': 'User-Agent=', 'name': 'User-Agent' }, { 'marker': 'Range=', 'name': 'Range' }, { 'marker': 'Orgin=', 'name': 'Orgin' }, { 'marker': 'X-Forwarded-For=', 'name': 'X-Forwarded-For' }] HANDLED_HTTP_HEADER_PARAMS = [ 'Cookie', 'Referer', 'User-Agent', 'Range', 'Orgin', 'X-Forwarded-For' ] @staticmethod def GET_PWGET_PATH(): return GetPluginDir('iptvdm/pwget.py') @staticmethod def GET_WGET_PATH(): return config.plugins.iptvplayer.wgetpath.value @staticmethod def GET_F4M_PATH(): return config.plugins.iptvplayer.f4mdumppath.value @staticmethod def GET_RTMPDUMP_PATH(): return config.plugins.iptvplayer.rtmpdumppath.value @staticmethod def getDownloaderType(url): if url.endswith(".f4m"): return DMHelper.DOWNLOADER_TYPE.F4F else: return DMHelper.DOWNLOADER_TYPE.WGET @staticmethod def getDownloaderCMD(downItem): if downItem.downloaderType == DMHelper.DOWNLOADER_TYPE.F4F: return DMHelper.getF4fCMD(downItem) else: return DMHelper.getWgetCMD(downItem) @staticmethod def makeUnikalFileName(fileName, withTmpFileName=True, addDateToFileName=False): # if this function is called # no more than once per second # date and time (with second) # is sufficient to provide a unique name from time import gmtime, strftime date = strftime("%Y-%m-%d_%H:%M:%S_", gmtime()) if not addDateToFileName: tries = 10 for idx in range(tries): if idx > 0: uniqueID = str(idx + 1) + '. ' else: uniqueID = '' newFileName = os.path.dirname( fileName) + os.sep + uniqueID + os.path.basename(fileName) if fileExists(newFileName): continue if withTmpFileName: tmpFileName = os.path.dirname( fileName) + os.sep + "." + uniqueID + os.path.basename( fileName) if fileExists(tmpFileName): continue return newFileName, tmpFileName else: return newFileName newFileName = os.path.dirname(fileName) + os.sep + date.replace( ':', '.') + os.path.basename(fileName) if withTmpFileName: tmpFileName = os.path.dirname( fileName) + os.sep + "." + date.replace( ':', '.') + os.path.basename(fileName) return newFileName, tmpFileName else: return newFileName @staticmethod def getProgressFromF4fSTSFile(file): ret = 0 try: fo = open(file, "r") lines = fo.readlines() fo.close() except: return ret if 0 < len(lines): match = re.search("|PROGRESS|([0-9]+?)/([0-9]+?)|", lines[1]) if match: ret = 100 * int(match.group(1)) / int(match.group(2)) return ret @staticmethod def getFileSize(filename): try: st = os.stat(filename) ret = st.st_size except: ret = -1 return ret @staticmethod def getRemoteContentInfoByUrllib(url, addParams={}): remoteContentInfo = {} addParams = DMHelper.downloaderParams2UrllibParams(addParams) addParams['return_data'] = False cm = common() # only request sts, response = cm.getPage(url, addParams) if sts: tmpInfo = response.info() remoteContentInfo = { 'Content-Length': tmpInfo.get('Content-Length', -1), 'Content-Type': tmpInfo.get('Content-Type', '') } if response: try: response.close() except: pass printDBG("getRemoteContentInfoByUrllib: [%r]" % remoteContentInfo) return sts, remoteContentInfo @staticmethod def downloaderParams2UrllibParams(params): tmpParams = {} userAgent = params.get('User-Agent', '') if '' != userAgent: tmpParams['User-Agent'] = userAgent cookie = params.get('Cookie', '') if '' != cookie: tmpParams['Cookie'] = cookie if len(tmpParams) > 0: return {'header': tmpParams} else: return {} @staticmethod def getDownloaderParamFromUrlWithMeta(url): printDBG( "DMHelper.getDownloaderParamFromUrlWithMeta url[%s], url.meta[%r]" % (url, url.meta)) downloaderParams = {} for key in url.meta: if key in DMHelper.HANDLED_HTTP_HEADER_PARAMS: downloaderParams[key] = url.meta[key] elif key == 'http_proxy': downloaderParams[key] = url.meta[key] return url, downloaderParams @staticmethod def getDownloaderParamFromUrl(url): if isinstance(url, strwithmeta): return DMHelper.getDownloaderParamFromUrlWithMeta(url) downloaderParams = {} paramsTab = url.split('|') url = paramsTab[0] del paramsTab[0] for param in DMHelper.HEADER_PARAMS: for item in paramsTab: if item.startswith(param['marker']): downloaderParams[ param['name']] = item[len(param['marker']):] # ugly workaround the User-Agent param should be passed in url if -1 < url.find('apple.com'): downloaderParams['User-Agent'] = 'QuickTime/7.6.2' return url, downloaderParams @staticmethod def getBaseWgetCmd(downloaderParams={}): printDBG("getBaseWgetCmd downloaderParams[%r]" % downloaderParams) headerOptions = '' proxyOptions = '' defaultHeader = ' --header "User-Agent: Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:21.0) Gecko/20100101 Firefox/21.0" ' for key, value in downloaderParams.items(): if value != '': if key in DMHelper.HANDLED_HTTP_HEADER_PARAMS: if 'Cookie' == key: headerOptions += ' --cookies=off ' headerOptions += ' --header "%s: %s" ' % (key, value) if key == 'User-Agent': defaultHeader = '' elif key == 'http_proxy': proxyOptions += ' -e use_proxy=yes -e http_proxy="%s" -e https_proxy="%s" ' % ( value, value) cmd = DMHelper.GET_WGET_PATH( ) + defaultHeader + ' --no-check-certificate ' + headerOptions + proxyOptions printDBG("getBaseWgetCmd return cmd[%s]" % cmd) return cmd
def __init__(self, session, url, pathForBuffering, pathForDownloading, movieTitle, activMoviePlayer, requestedBuffSize, playerAdditionalParams={}, downloadManager=None, fileExtension=''): self.session = session Screen.__init__(self, session) self.onStartCalled = False self.downloadingPath = pathForDownloading self.bufferingPath = pathForBuffering self.filePath = pathForBuffering + '/.iptv_buffering.flv' self.url = url self.movieTitle = movieTitle self.downloadManager = downloadManager self.fileExtension = fileExtension self.currentService = self.session.nav.getCurrentlyPlayingServiceReference( ) self.activMoviePlayer = activMoviePlayer self.onClose.append(self.__onClose) #self.onLayoutFinish.append(self.doStart) self.onShow.append(self.onWindowShow) #self.onHide.append(self.onWindowHide) self["actions"] = ActionMap( [ "IPTVAlternateVideoPlayer", "WizardActions", "MoviePlayerActions" ], { "ok": self.ok_pressed, "back": self.back_pressed, "leavePlayer": self.back_pressed, "record": self.record_pressed, }, -1) self["console"] = Label() self["percentage"] = Label() self["addinfo"] = Label() self['ok_button'] = Cover3() self['rec_button'] = Cover3() self['exit_button'] = Cover3() self["icon"] = SimpleAnimatedCover() # prepare icon frames path frames = [] for idx in range(1, self.NUM_OF_ICON_FRAMES + 1): frames.append( resolveFilename( SCOPE_PLUGINS, 'Extensions/IPTVPlayer/icons/buffering/buffering_%d.png' % idx)) self["icon"].loadFrames(frames) self.inMoviePlayer = False self.canRunMoviePlayer = False # used in function updateDisplay, so must be first initialized #main Timer self.mainTimer = eTimer() self.mainTimerEnabled = False self.mainTimer_conn = eConnectCallback(self.mainTimer.timeout, self.updateDisplay) self.mainTimerInterval = 1000 # by default 1s self.requestedBuffSize = requestedBuffSize self.playerAdditionalParams = playerAdditionalParams self.clipLength = None self.lastPosition = None self.lastSize = 0 # Some MP4 files are not prepared for streaming # MOOV atom is needed to start playback # if it is located at the end of file, then # will try to download it separately to start playback # without downloading the whole file self.clouldBeMP4 = False self.isMOOVAtomAtTheBeginning = None self.checkMOOVAtom = True self.maxMOOVAtomSize = 10 * 1024 * 1024 # 10 MB max moov atom size self.moovAtomOffset = 0 self.moovAtomSize = 0 self.MOOV_STS = enum(UNKNOWN=0, WAITING=1, DOWNLOADING=2, DOWNLOADED=3, ERROR=4) self.moovAtomStatus = self.MOOV_STS.UNKNOWN self.moovAtomDownloader = None self.moovAtomPath = pathForBuffering + '/.iptv_buffering_moov.flv' self.closeRequestedByUser = None printDBG(">> activMoviePlayer[%s]" % self.activMoviePlayer)
class DMHelper: STATUS_FILE_PATH = '/tmp/iptvdownload' STATUS_FILE_EXT = '.txt' STS = enum( WAITING = 'STS_WAITING', DOWNLOADING = 'STS_DOWNLOADING', DOWNLOADED = 'STS_DOWNLOADED', INTERRUPTED = 'STS_INTERRUPTED', ERROR = 'STS_ERROR', POSTPROCESSING = 'STS_POSTPROCESSING') DOWNLOAD_TYPE = enum( INITIAL = 'INIT_DOWNLOAD', CONTINUE = 'CONTINUE_DOWNLOAD', RETRY = 'RETRY_DOWNLOAD' ) # DOWNLOADER_TYPE = enum( WGET = 'WGET_DOWNLOADER', F4F = 'F4F_DOWNLOADER' ) HEADER_PARAMS = [{'marker':'Host=', 'name':'Host'}, {'marker':'Accept=', 'name':'Accept'}, {'marker':'Cookie=', 'name':'Cookie'}, {'marker':'Referer=', 'name':'Referer'}, {'marker':'User-Agent=', 'name':'User-Agent'}, {'marker':'Range=', 'name':'Range'}, {'marker':'Orgin=', 'name':'Orgin'}, {'marker':'Origin=', 'name':'Origin'}, {'marker':'X-Playback-Session-Id=', 'name':'X-Playback-Session-Id'}, {'marker':'If-Modified-Since=','name':'If-Modified-Since'}, {'marker':'If-None-Match=', 'name':'If-None-Match'}, {'marker':'X-Forwarded-For=', 'name':'X-Forwarded-For'}, {'marker':'Authorization=', 'name':'Authorization'}, ] HANDLED_HTTP_HEADER_PARAMS = ['Host', 'Accept', 'Cookie', 'Referer', 'User-Agent', 'Range', 'Orgin', 'Origin', 'X-Playback-Session-Id', 'If-Modified-Since', 'If-None-Match', 'X-Forwarded-For', 'Authorization'] IPTV_DOWNLOADER_PARAMS = ['iptv_wget_continue', 'iptv_wget_timeout', 'iptv_wget_waitretry', 'iptv_wget_retry_on_http_error', 'iptv_wget_tries'] @staticmethod def GET_PWGET_PATH(): return GetPluginDir('iptvdm/pwget.py') @staticmethod def GET_WGET_PATH(): return config.plugins.iptvplayer.wgetpath.value @staticmethod def GET_F4M_PATH(): return config.plugins.iptvplayer.f4mdumppath.value @staticmethod def GET_HLSDL_PATH(): return config.plugins.iptvplayer.hlsdlpath.value @staticmethod def GET_FFMPEG_PATH(): altFFMPEGPath = '/iptvplayer_rootfs/usr/bin/ffmpeg' if IsExecutable(altFFMPEGPath): return altFFMPEGPath return "ffmpeg" @staticmethod def GET_RTMPDUMP_PATH(): return config.plugins.iptvplayer.rtmpdumppath.value @staticmethod def getDownloaderType(url): if url.endswith(".f4m"): return DMHelper.DOWNLOADER_TYPE.F4F else: return DMHelper.DOWNLOADER_TYPE.WGET @staticmethod def getDownloaderCMD(downItem): if downItem.downloaderType == DMHelper.DOWNLOADER_TYPE.F4F: return DMHelper.getF4fCMD(downItem) else: return DMHelper.getWgetCMD(downItem) @staticmethod def makeUnikalFileName(fileName, withTmpFileName = True, addDateToFileName=False): # if this function is called # no more than once per second # date and time (with second) # is sufficient to provide a unique name from time import gmtime, strftime date = strftime("%Y-%m-%d_%H:%M:%S_", gmtime()) if not addDateToFileName: tries = 10 for idx in range(tries): if idx > 0: uniqueID = str(idx+1) + '. ' else: uniqueID = '' newFileName = os.path.dirname(fileName) + os.sep + uniqueID + os.path.basename(fileName) if fileExists(newFileName): continue if withTmpFileName: tmpFileName = os.path.dirname(fileName) + os.sep + "." + uniqueID + os.path.basename(fileName) if fileExists(tmpFileName): continue return newFileName, tmpFileName else: return newFileName newFileName = os.path.dirname(fileName) + os.sep + date.replace(':', '.') + os.path.basename(fileName) if withTmpFileName: tmpFileName = os.path.dirname(fileName) + os.sep + "." + date.replace(':', '.') + os.path.basename(fileName) return newFileName, tmpFileName else: return newFileName @staticmethod def getProgressFromF4fSTSFile(file): ret = 0 try: fo = open(file, "r") lines = fo.readlines() fo.close() except Exception: return ret if 0 < len(lines): match = re.search("|PROGRESS|([0-9]+?)/([0-9]+?)|" , lines[1]) if match: ret = 100 * int(match.group(1)) / int(match.group(2)) return ret @staticmethod def getFileSize(filename): try: st = os.stat(filename) ret = st.st_size except Exception: ret = -1 return ret @staticmethod def getRemoteContentInfoByUrllib(url, addParams = {}): remoteContentInfo = {} addParams = DMHelper.downloaderParams2UrllibParams(addParams) addParams['max_data_size'] = 0 cm = common() # only request sts = cm.getPage(url, addParams)[0] if sts: remoteContentInfo = {'Content-Length': cm.meta.get('content-length', -1), 'Content-Type': cm.meta.get('content-type', '')} printDBG("getRemoteContentInfoByUrllib: [%r]" % remoteContentInfo) return sts,remoteContentInfo @staticmethod def downloaderParams2UrllibParams(params): tmpParams = {} userAgent = params.get('User-Agent', '') if '' != userAgent: tmpParams['User-Agent'] = userAgent cookie = params.get('Cookie', '') if '' != cookie: tmpParams['Cookie'] = cookie if len(tmpParams) > 0: return {'header': tmpParams} else: return {} @staticmethod def getDownloaderParamFromUrlWithMeta(url, httpHeadersOnly=False): printDBG("DMHelper.getDownloaderParamFromUrlWithMeta url[%s], url.meta[%r]" % (url, url.meta)) downloaderParams = {} for key in url.meta: if key in DMHelper.HANDLED_HTTP_HEADER_PARAMS: downloaderParams[key] = url.meta[key] elif key == 'http_proxy': downloaderParams[key] = url.meta[key] if not httpHeadersOnly: for key in DMHelper.IPTV_DOWNLOADER_PARAMS: if key in url.meta: downloaderParams[key] = url.meta[key] return url, downloaderParams @staticmethod def getDownloaderParamFromUrl(url): if isinstance(url, strwithmeta): return DMHelper.getDownloaderParamFromUrlWithMeta(url) downloaderParams = {} paramsTab = url.split('|') url = paramsTab[0] del paramsTab[0] for param in DMHelper.HEADER_PARAMS: for item in paramsTab: if item.startswith(param['marker']): downloaderParams[param['name']] = item[len(param['marker']):] # ugly workaround the User-Agent param should be passed in url if -1 < url.find('apple.com'): downloaderParams['User-Agent'] = 'QuickTime/7.6.2' return url,downloaderParams @staticmethod def getBaseWgetCmd(downloaderParams = {}): printDBG("getBaseWgetCmd downloaderParams[%r]" % downloaderParams) headerOptions = '' proxyOptions = '' #defaultHeader = ' --header "User-Agent: Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:21.0) Gecko/20100101 Firefox/21.0" ' defaultHeader = ' --header "User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36" ' for key, value in downloaderParams.items(): if value != '': if key in DMHelper.HANDLED_HTTP_HEADER_PARAMS: if 'Cookie' == key: headerOptions += ' --cookies=off ' headerOptions += ' --header "%s: %s" ' % (key, value) if key == 'User-Agent': defaultHeader = '' elif key == 'http_proxy': proxyOptions += ' -e use_proxy=yes -e http_proxy="%s" -e https_proxy="%s" ' % (value, value) wgetContinue = '' if downloaderParams.get('iptv_wget_continue', False): wgetContinue = ' -c --timeout=%s --waitretry=%s ' % (downloaderParams.get('iptv_wget_timeout', 30), downloaderParams.get('iptv_wget_waitretry', 1)) else: if 'iptv_wget_timeout' in downloaderParams: wgetContinue += ' --timeout=%s ' % downloaderParams['iptv_wget_timeout'] if 'iptv_wget_waitretry' in downloaderParams: wgetContinue += ' --waitretry=%s ' % downloaderParams['iptv_wget_waitretry'] if 'iptv_wget_retry_on_http_error' in downloaderParams: wgetContinue += ' --retry-on-http-error=%s ' % downloaderParams['iptv_wget_retry_on_http_error'] if 'iptv_wget_tries' in downloaderParams: wgetContinue += ' --tries=%s ' % downloaderParams['iptv_wget_tries'] if 'start_pos' in downloaderParams: wgetContinue = ' --start-pos=%s ' % downloaderParams['start_pos'] cmd = DMHelper.GET_WGET_PATH() + wgetContinue + defaultHeader + ' --no-check-certificate ' + headerOptions + proxyOptions printDBG("getBaseWgetCmd return cmd[%s]" % cmd) return cmd @staticmethod def getBaseHLSDLCmd(downloaderParams = {}): printDBG("getBaseWgetCmd downloaderParams[%r]" % downloaderParams) headerOptions = '' proxyOptions = '' #userAgent = ' -u "Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:21.0) Gecko/20100101 Firefox/21.0" ' userAgent = ' -u "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.103 Safari/537.36" ' for key, value in downloaderParams.items(): if value != '': if key in DMHelper.HANDLED_HTTP_HEADER_PARAMS: if key == 'User-Agent': userAgent = ' -u "%s" ' % value else: headerOptions += ' -h "%s: %s" ' % (key, value) elif key == 'http_proxy': proxyOptions += ' -e use_proxy=yes -e http_proxy="%s" -e https_proxy="%s" ' % (value, value) cmd = DMHelper.GET_HLSDL_PATH() + ' -q -f -b ' + userAgent + headerOptions + proxyOptions printDBG("getBaseHLSDLCmd return cmd[%s]" % cmd) return cmd
class WgetDownloader(BaseDownloader): # wget status WGET_STS = enum(NONE='WGET_NONE', CONNECTING='WGET_CONNECTING', DOWNLOADING='WGET_DOWNLOADING', ENDED='WGET_ENDED') # wget status INFO = enum(FROM_FILE='INFO_FROM_FILE', FROM_DOTS='INFO_FROM_DOTS') def __init__(self): printDBG('WgetDownloader.__init__ ----------------------------------') BaseDownloader.__init__(self) self.wgetStatus = self.WGET_STS.NONE # instance of E2 console self.console = None self.iptv_sys = None def __del__(self): printDBG("WgetDownloader.__del__ ----------------------------------") def getName(self): return "wget" def isWorkingCorrectly(self, callBackFun): self.iptv_sys = iptv_system( DMHelper.GET_WGET_PATH() + " -V 2>&1 ", boundFunction(self._checkWorkingCallBack, callBackFun)) def _checkWorkingCallBack(self, callBackFun, code, data): reason = '' sts = True if code != 0: sts = False reason = data self.iptv_sys = None callBackFun(sts, reason) def start(self, url, filePath, params={}, info_from=None, retries=0): ''' Owervrite start from BaseDownloader ''' self.url = url self.filePath = filePath self.downloaderParams = params self.fileExtension = '' # should be implemented in future self.outData = '' self.contentType = 'unknown' if None == info_from: info_from = WgetDownloader.INFO.FROM_FILE self.infoFrom = info_from cmd = DMHelper.getBaseWgetCmd(self.downloaderParams) + ( ' --progress=dot:default -t %d ' % retries ) + '"' + self.url + '" -O "' + self.filePath + '" > /dev/null' printDBG("Download cmd[%s]" % cmd) self.console = eConsoleAppContainer() self.console_appClosed_conn = eConnectCallback(self.console.appClosed, self._cmdFinished) self.console_stderrAvail_conn = eConnectCallback( self.console.stderrAvail, self._dataAvail) self.console.execute(cmd) self.wgetStatus = self.WGET_STS.CONNECTING self.status = DMHelper.STS.DOWNLOADING self.onStart() return BaseDownloader.CODE_OK def _dataAvail(self, data): if None != data: self.outData += data if self.WGET_STS.CONNECTING == self.wgetStatus: self.outData += data lines = self.outData.replace('\r', '\n').split('\n') for idx in range(len(lines)): if lines[idx].startswith('Length:'): match = re.search( "Length: ([0-9]+?) \([^)]+?\) (\[[^]]+?\])", lines[idx]) if match: self.remoteFileSize = int(match.group(1)) self.remoteContentType = match.group(2) elif lines[idx].startswith('Saving to:'): if len(lines) > idx: self.outData = '\n'.join(lines[idx + 1:]) else: self.outData = '' self.wgetStatus = self.WGET_STS.DOWNLOADING if self.infoFrom != WgetDownloader.INFO.FROM_DOTS: self.console_stderrAvail_conn = None break def _terminate(self): printDBG("WgetDownloader._terminate") if None != self.iptv_sys: self.iptv_sys.kill() self.iptv_sys = None if DMHelper.STS.DOWNLOADING == self.status: if self.console: self.console.sendCtrlC() # kill # produce zombies self._cmdFinished(-1, True) return BaseDownloader.CODE_OK return BaseDownloader.CODE_NOT_DOWNLOADING def _cmdFinished(self, code, terminated=False): printDBG("WgetDownloader._cmdFinished code[%r] terminated[%r]" % (code, terminated)) # break circular references self.console_appClosed_conn = None self.console_stderrAvail_conn = None self.console = None self.wgetStatus = self.WGET_STS.ENDED # When finished updateStatistic based on file sie on disk BaseDownloader.updateStatistic(self) if terminated: self.status = DMHelper.STS.INTERRUPTED elif 0 >= self.localFileSize: self.status = DMHelper.STS.ERROR elif self.remoteFileSize > 0 and self.remoteFileSize > self.localFileSize: self.status = DMHelper.STS.INTERRUPTED else: self.status = DMHelper.STS.DOWNLOADED if not terminated: self.onFinish() def updateStatistic(self): if self.infoFrom == WgetDownloader.INFO.FROM_FILE: BaseDownloader.updateStatistic(self) return if self.WGET_STS.DOWNLOADING == self.wgetStatus: print self.outData dataLen = len(self.outData) for idx in range(dataLen): if idx + 1 < dataLen: # default style - one dot = 1K if '.' == self.outData[idx] and self.outData[idx + 1] in [ '.', ' ' ]: self.localFileSize += 1024 else: self.outData = self.outData[idx:] break BaseDownloader._updateStatistic(self)