def _dataAvail(self, data): if None == data: return data = self.outData + data if '\n' != data[-1]: truncated = True else: truncated = False data = data.split('\n') if truncated: self.outData = data[-1] del data[-1] for item in data: printDBG(item) if item.startswith('{'): try: updateStatistic = False obj = json.loads(item.strip()) printDBG("Status object [%r]" % obj) if "d_s" in obj: self.localFileSize = obj["d_s"] updateStatistic = True if "t_d" in obj: self.totalDuration = obj["t_d"] updateStatistic = True if "d_d" in obj: self.downloadDuration = obj["d_d"] updateStatistic = True if "d_t" in obj and obj['d_t'] == 'live': self.liveStream = True if updateStatistic: BaseDownloader._updateStatistic(self) except Exception: printExc() continue
def _cmdFinished(self, code, terminated=False): printDBG("WgetDownloader._cmdFinished code[%r] terminated[%r]" % (code, terminated)) # When finished updateStatistic based on file sie on disk BaseDownloader.updateStatistic(self) if not terminated and self.remoteFileSize > 0 \ and self.remoteFileSize > self.localFileSize \ and self.curContinueRetry < self.maxContinueRetry: self.curContinueRetry += 1 self.console.execute(E2PrioFix(self.downloadCmd)) return # break circular references self.console_appClosed_conn = None self.console_stderrAvail_conn = None self.console = None self.wgetStatus = self.WGET_STS.ENDED if terminated: self.status = DMHelper.STS.INTERRUPTED elif 0 >= self.localFileSize: self.status = DMHelper.STS.ERROR elif self.remoteFileSize > 0 and self.remoteFileSize > self.localFileSize: self.status = DMHelper.STS.INTERRUPTED else: self.status = DMHelper.STS.DOWNLOADED if not terminated: self.onFinish()
def _cmdFinished(self, code, terminated=False): printDBG("PwgetDownloader._cmdFinished code[%r] terminated[%r]" % (code, terminated)) # break circular references self.console_appClosed_conn = None self.console_stderrAvail_conn = None self.console = None self.wgetStatus = self.WGET_STS.ENDED # When finished updateStatistic based on file sie on disk BaseDownloader.updateStatistic(self) if not terminated: printDBG("PwgetDownloader._cmdFinished [%s]" % self.outData) match = re.search("Content-Length: ([0-9]+?)[^0-9]", self.outData) if match: self.remoteFileSize = int(match.group(1)) if terminated: self.status = DMHelper.STS.INTERRUPTED elif 0 >= self.localFileSize: self.status = DMHelper.STS.ERROR elif self.remoteFileSize > 0 and self.remoteFileSize > self.localFileSize: self.status = DMHelper.STS.INTERRUPTED else: self.status = DMHelper.STS.DOWNLOADED if not terminated: self.onFinish()
def _cmdFinished(self, code, terminated=False): printDBG("WgetDownloader._cmdFinished code[%r] terminated[%r]" % (code, terminated)) # When finished updateStatistic based on file sie on disk BaseDownloader.updateStatistic(self) if not terminated and self.remoteFileSize > 0 \ and self.remoteFileSize > self.localFileSize \ and self.curContinueRetry < self.maxContinueRetry: self.curContinueRetry += 1 self.console.execute( E2PrioFix( self.downloadCmd ) ) return # break circular references self.console_appClosed_conn = None self.console_stderrAvail_conn = None self.console = None self.wgetStatus = self.WGET_STS.ENDED if terminated: self.status = DMHelper.STS.INTERRUPTED elif 0 >= self.localFileSize: self.status = DMHelper.STS.ERROR elif self.remoteFileSize > 0 and self.remoteFileSize > self.localFileSize: self.status = DMHelper.STS.INTERRUPTED else: self.status = DMHelper.STS.DOWNLOADED if not terminated: self.onFinish()
def __init__(self): printDBG('M3U8Downloader.__init__ ----------------------------------') BaseDownloader.__init__(self) self.wgetStatus = self.WGET_STS.NONE # instance of E2 console self.console = eConsoleAppContainer() self.iptv_sys = None # M3U8 list updater self.M3U8Updater = eConsoleAppContainer() self.M3U8Updater_appClosed_conn = eConnectCallback(self.M3U8Updater.appClosed, self._updateM3U8Finished) self.M3U8Updater_stdoutAvail_conn = eConnectCallback(self.M3U8Updater.stdoutAvail, self._updateM3U8DataAvail) self.M3U8ListData = '' self.M3U8UpdaterRefreshDelay = 0 self.refreshDelay = M3U8Downloader.MIN_REFRESH_DELAY # get only last fragments from first list, to satisfy specified duration in seconds # -1 means, starts from beginning self.startLiveDuration = M3U8Downloader.LIVE_START_OFFSET # 0 means, starts from beginning self.skipFirstSegFromList = 0 self.addStampToUrl = False self.totalDuration = -1 self.downloadDuration = 0 self.fragmentDurationList = []
def __init__(self): printDBG('M3U8Downloader.__init__ ----------------------------------') BaseDownloader.__init__(self) self.wgetStatus = self.WGET_STS.NONE # instance of E2 console self.console = eConsoleAppContainer() self.iptv_sys = None # M3U8 list updater self.M3U8Updater = eConsoleAppContainer() self.M3U8Updater_appClosed_conn = eConnectCallback( self.M3U8Updater.appClosed, self._updateM3U8Finished) self.M3U8Updater_stdoutAvail_conn = eConnectCallback( self.M3U8Updater.stdoutAvail, self._updateM3U8DataAvail) self.M3U8ListData = '' self.M3U8UpdaterRefreshDelay = 0 self.refreshDelay = M3U8Downloader.MIN_REFRESH_DELAY # get only last fragments from first list, to satisfy specified duration in seconds # -1 means, starts from beginning self.startLiveDuration = M3U8Downloader.LIVE_START_OFFSET # 0 means, starts from beginning self.skipFirstSegFromList = 0 self.addStampToUrl = False self.totalDuration = -1 self.downloadDuration = 0 self.fragmentDurationList = [] self.maxTriesAtStart = 0
def __init__(self): printDBG('F4mDownloader.__init__ ----------------------------------') BaseDownloader.__init__(self) # instance of E2 console self.console = None self.iptv_sys = None
def __init__(self): printDBG('PwgetDownloader.__init__ ----------------------------------') BaseDownloader.__init__(self) self.wgetStatus = self.WGET_STS.NONE # instance of E2 console self.console = None self.iptv_sys = None
def __init__(self): printDBG('RtmpDownloader.__init__ ----------------------------------') BaseDownloader.__init__(self) self.rtmpStatus = self.RTMP_STS.NONE # instance of E2 console self.console = None self.iptv_sys = None
def __init__(self): printDBG('MergeDownloader.__init__ ----------------------------------') BaseDownloader.__init__(self) # instance of E2 console self.console = None self.iptv_sys = None self.multi = {'urls':[], 'files':[], 'remote_size':[], 'remote_content_type':[], 'local_size':[]} self.currIdx = 0
def __init__(self): printDBG('HLSDownloader.__init__ ----------------------------------') BaseDownloader.__init__(self) # instance of E2 console self.console = None self.iptv_sys = None self.totalDuration = 0 self.downloadDuration = 0 self.liveStream = False
def __init__(self): printDBG('WgetDownloader.__init__ ----------------------------------') BaseDownloader.__init__(self) self.wgetStatus = self.WGET_STS.NONE # instance of E2 console self.console = None self.iptv_sys = None self.curContinueRetry = 0 self.maxContinueRetry = 0 self.downloadCmd = ''
def __init__(self): printDBG('WgetDownloader.__init__ ') BaseDownloader.__init__(self) self.wgetStatus = self.WGET_STS.NONE # instance of E2 console self.console = None self.iptv_sys = None self.curContinueRetry = 0 self.maxContinueRetry = 0 self.downloadCmd = '' self.remoteContentType = None self.lastErrorCode = None self.lastErrorDesc = ''
def updateStatistic(self): if self.infoFrom == WgetDownloader.INFO.FROM_FILE: BaseDownloader.updateStatistic(self) return if self.WGET_STS.DOWNLOADING == self.wgetStatus: print(self.outData) dataLen = len(self.outData) for idx in range(dataLen): if idx + 1 < dataLen: # default style - one dot = 1K if '.' == self.outData[idx] and self.outData[idx + 1] in ['.', ' ']: self.localFileSize += 1024 else: self.outData = self.outData[idx:] break BaseDownloader._updateStatistic(self)
def updateStatistic(self): if self.infoFrom == WgetDownloader.INFO.FROM_FILE: BaseDownloader.updateStatistic(self) return if self.WGET_STS.DOWNLOADING == self.wgetStatus: print self.outData dataLen = len(self.outData) for idx in range(dataLen): if idx+1 < dataLen: # default style - one dot = 1K if '.' == self.outData[idx] and self.outData[idx+1] in ['.', ' ']: self.localFileSize += 1024 else: self.outData = self.outData[idx:] break BaseDownloader._updateStatistic(self)
def __init__(self): printDBG('FFMPEGDownloader.__init__ ----------------------------------') BaseDownloader.__init__(self) # instance of E2 console self.console = None self.iptv_sys = None self.totalDuration = 0 self.downloadDuration = 0 self.liveStream = False self.headerReceived = False self.parseReObj = {} self.parseReObj['start_time'] = re.compile('\sstart\:\s*?([0-9]+?)\.') self.parseReObj['duration'] = re.compile('[\s=]([0-9]+?)\:([0-9]+?)\:([0-9]+?)\.') self.parseReObj['size'] = re.compile('size=\s*?([0-9]+?)kB') self.parseReObj['bitrate'] = re.compile('bitrate=\s*?([0-9]+?(?:\.[0-9]+?)?)kbits') self.parseReObj['speed'] = re.compile('speed=\s*?([0-9]+?(?:\.[0-9]+?)?)x') self.ffmpegOutputContener = 'matroska' self.fileCmdPath = ''
def _cmdFinished(self, code, terminated=False): printDBG("BuxyboxWgetDownloader._cmdFinished code[%r] terminated[%r]" % (code, terminated)) # break circular references self.console_appClosed_conn = None self.console = None self.wgetStatus = self.WGET_STS.ENDED # When finished updateStatistic based on file sie on disk BaseDownloader.updateStatistic(self) if terminated: self.status = DMHelper.STS.INTERRUPTED elif 0 >= self.localFileSize: self.status = DMHelper.STS.ERROR elif self.remoteFileSize > 0 and self.remoteFileSize > self.localFileSize: self.status = DMHelper.STS.INTERRUPTED else: self.status = DMHelper.STS.DOWNLOADED if not terminated: self.onFinish()
def _dataAvail(self, data): if None == data: return data = self.outData + data if '\n' != data[-1]: truncated = True else: truncated = False data = data.split('\n') if truncated: self.outData = data[-1] del data[-1] for item in data: printDBG(item) if item.startswith('{'): try: obj = json.loads(item.strip()) printDBG("Status object [%r]" % obj) if "total_download_size" in obj: self.localFileSize = obj["total_download_size"] BaseDownloader._updateStatistic(self) except: printExc() continue
def updateStatistic(self): BaseDownloader.updateStatistic(self) return
def _cmdFinished(self, code, terminated=False): printDBG("M3U8Downloader._cmdFinished code[%r] terminated[%r] downloadType[%s]" % (code, terminated, self.downloadType)) localStatus = DMHelper.STS.ERROR if terminated: BaseDownloader.updateStatistic(self) localStatus = DMHelper.STS.INTERRUPTED elif self.DOWNLOAD_TYPE.M3U8 == self.downloadType: self.console_appClosed_conn = None self.console_stdoutAvail_conn = None if 0 < len(self.outData): try: m3u8Obj = m3u8.inits(self.outData, self.m3u8Url) # uri given to m3u8 downloader should not be variant, # format should be selected before starting downloader # however if this was not done the firs one will be selected if m3u8Obj.is_variant: if 0 < len(m3u8Obj.playlists): self._startM3U8(self._segUri(m3u8Obj.playlists[-1].absolute_uri)) localStatus = DMHelper.STS.DOWNLOADING else: if 0 < len(m3u8Obj.segments): if not m3u8Obj.is_endlist: self.liveStream = True if -1 == self.startLiveDuration: self.fragmentList = [self._segUri(seg.absolute_uri) for seg in m3u8Obj.segments] else: # some live streams only add new fragments not removing old, # in this case most probably we not want to download old fragments # but start from last N fragments/seconds # self.startLiveDuration self.fragmentList = [] currentDuration = 0 maxFragDuration = m3u8Obj.target_duration for seg in reversed(m3u8Obj.segments): if None != seg.duration: currentDuration += seg.duration else: currentDuration += maxFragDuration self.fragmentList.append(self._segUri(seg.absolute_uri)) if currentDuration >= self.startLiveDuration: break self.fragmentList.reverse() # start update fragment list loop #self.fragmentList = self.fixFragmentsList(self.fragmentList) self._updateM3U8Finished(-1) else: self.fragmentList = [self._segUri(seg.absolute_uri) for seg in m3u8Obj.segments] localStatus = self._startFragment() except: pass elif self.liveStream and self.DOWNLOAD_TYPE.WAITTING == self.downloadType: printDBG("m3u8 liveStream waitting finished--------------------------------") localStatus = self._startFragment() else: BaseDownloader.updateStatistic(self) printDBG("m3u8 nextFragment finished: live[%r]: r[%d], l[%d], p[%d]" % (self.liveStream, self.remoteFragmentSize, self.localFileSize, self.m3u8_prevLocalFileSize )) if 0 >= self.localFileSize: if not self.liveStream: localStatus = DMHelper.STS.ERROR else: localStatus = self._startFragment() #elif not self.liveStream and self.remoteFragmentSize > 0 and self.remoteFragmentSize > (self.localFileSize - self.m3u8_prevLocalFileSize): # localStatus = DMHelper.STS.INTERRUPTED elif 0 < (self.localFileSize - self.m3u8_prevLocalFileSize): localStatus = self._startFragment() elif 0 == (self.localFileSize - self.m3u8_prevLocalFileSize): localStatus = self._startFragment(True) # retry else: localStatus = DMHelper.STS.INTERRUPTED self.status = localStatus if DMHelper.STS.DOWNLOADING == self.status: return # clean up at finish if self.M3U8Updater: self.M3U8Updater_appClosed_conn = None self.M3U8Updater_stdoutAvail_conn = None self.M3U8Updater = None self.liveStream = False if self.console: self.console_appClosed_conn = None self.console_stdoutAvail_conn = None self.console.sendCtrlC() # kill # produce zombies self.console = None ''' if None != self.updateThread: if self.updateThread.Thread.isAlive(): # give some time for update thread to finish sleep(self.MIN_REFRESH_DELAY) printDBG('m3u8 downloader killing update thread') ''' if not terminated: self.onFinish()
def _cmdFinished(self, code, terminated=False): printDBG( "M3U8Downloader._cmdFinished code[%r] terminated[%r] downloadType[%s]" % (code, terminated, self.downloadType)) localStatus = DMHelper.STS.ERROR if terminated: BaseDownloader.updateStatistic(self) localStatus = DMHelper.STS.INTERRUPTED elif self.DOWNLOAD_TYPE.M3U8 == self.downloadType: self.console_appClosed_conn = None self.console_stdoutAvail_conn = None if 0 < len(self.outData): try: m3u8Obj = m3u8.inits(self.outData, self.m3u8Url) # uri given to m3u8 downloader should not be variant, # format should be selected before starting downloader # however if this was not done the firs one will be selected if m3u8Obj.is_variant: if 0 < len(m3u8Obj.playlists): self.m3u8Url = self._segUri( m3u8Obj.playlists[-1].absolute_uri) self._startM3U8() localStatus = DMHelper.STS.DOWNLOADING else: if 0 < len(m3u8Obj.segments): if not m3u8Obj.is_endlist: self.liveStream = True if -1 == self.startLiveDuration: self.fragmentList = [ self._segUri(seg.absolute_uri) for seg in m3u8Obj.segments ] else: # some live streams only add new fragments not removing old, # in this case most probably we not want to download old fragments # but start from last N fragments/seconds # self.startLiveDuration self.fragmentList = [] currentDuration = 0 maxFragDuration = m3u8Obj.target_duration for seg in reversed(m3u8Obj.segments): if None != seg.duration: currentDuration += seg.duration else: currentDuration += maxFragDuration self.fragmentList.append( self._segUri(seg.absolute_uri)) if currentDuration >= self.startLiveDuration: break self.fragmentList.reverse() if len(m3u8Obj.segments) == len( self.fragmentList) and len( self.fragmentList ) > self.skipFirstSegFromList: self.fragmentList = self.fragmentList[ self.skipFirstSegFromList:] self.lastMediaSequence = m3u8Obj.media_sequence + len( m3u8Obj.segments) # start update fragment list loop #self.fragmentList = self.fixFragmentsList(self.fragmentList) self._updateM3U8Finished(-1) else: self.fragmentList = [ self._segUri(seg.absolute_uri) for seg in m3u8Obj.segments ] try: self.totalDuration = 0 self.fragmentDurationList = [] for seg in m3u8Obj.segments: self.totalDuration += seg.duration self.fragmentDurationList.append( seg.duration) except Exception: printExc() self.totalDuration = -1 self.fragmentDurationList = [] localStatus = self._startFragment() except Exception: pass printDBG(">>>>>>>>>>>>>>>>>> localStatus [%s] tries[%d]" % (localStatus, self.tries)) if localStatus == DMHelper.STS.ERROR and self.tries < self.maxTriesAtStart: self.console_appClosed_conn = None self.console_stdoutAvail_conn = None self.tries += 1 self._startM3U8(self.MIN_REFRESH_DELAY) return else: self.tries = 0 elif self.liveStream and self.DOWNLOAD_TYPE.WAITTING == self.downloadType: printDBG( "m3u8 liveStream waitting finished--------------------------------" ) localStatus = self._startFragment() else: BaseDownloader.updateStatistic(self) printDBG( "m3u8 nextFragment finished: live[%r]: r[%d], l[%d], p[%d]" % (self.liveStream, self.remoteFragmentSize, self.localFileSize, self.m3u8_prevLocalFileSize)) if 0 >= self.localFileSize: if not self.liveStream: localStatus = DMHelper.STS.ERROR else: localStatus = self._startFragment() #elif not self.liveStream and self.remoteFragmentSize > 0 and self.remoteFragmentSize > (self.localFileSize - self.m3u8_prevLocalFileSize): # localStatus = DMHelper.STS.INTERRUPTED elif 0 < (self.localFileSize - self.m3u8_prevLocalFileSize): if self.totalDuration > 0: try: self.downloadDuration += self.fragmentDurationList[ self.currentFragment] except Exception: printExc() localStatus = self._startFragment() elif 0 == (self.localFileSize - self.m3u8_prevLocalFileSize): localStatus = self._startFragment(True) # retry else: localStatus = DMHelper.STS.INTERRUPTED self.status = localStatus if DMHelper.STS.DOWNLOADING == self.status: return # clean up at finish if self.M3U8Updater: self.M3U8Updater_appClosed_conn = None self.M3U8Updater_stdoutAvail_conn = None self.M3U8Updater = None self.liveStream = False if self.console: self.console_appClosed_conn = None self.console_stdoutAvail_conn = None self.console.sendCtrlC() # kill # produce zombies self.console = None ''' if None != self.updateThread: if self.updateThread.Thread.isAlive(): # give some time for update thread to finish sleep(self.MIN_REFRESH_DELAY) printDBG('m3u8 downloader killing update thread') ''' if not terminated: self.onFinish()