def _dataAvail(self, data):
if None == data:
return
data = self.outData + data
if '\n' != data[-1]: truncated = True
else: truncated = False
data = data.split('\n')
if truncated:
self.outData = data[-1]
del data[-1]
for item in data:
printDBG(item)
if item.startswith('{'):
try:
updateStatistic = False
obj = json.loads(item.strip())
printDBG("Status object [%r]" % obj)
if "d_s" in obj:
self.localFileSize = obj["d_s"]
updateStatistic = True
if "t_d" in obj:
self.totalDuration = obj["t_d"]
updateStatistic = True
if "d_d" in obj:
self.downloadDuration = obj["d_d"]
updateStatistic = True
if "d_t" in obj and obj['d_t'] == 'live':
self.liveStream = True
if updateStatistic:
BaseDownloader._updateStatistic(self)
except Exception:
printExc()
continue
def _cmdFinished(self, code, terminated=False):
printDBG("WgetDownloader._cmdFinished code[%r] terminated[%r]" %
(code, terminated))
# When finished updateStatistic based on file sie on disk
BaseDownloader.updateStatistic(self)
if not terminated and self.remoteFileSize > 0 \
and self.remoteFileSize > self.localFileSize \
and self.curContinueRetry < self.maxContinueRetry:
self.curContinueRetry += 1
self.console.execute(E2PrioFix(self.downloadCmd))
return
# break circular references
self.console_appClosed_conn = None
self.console_stderrAvail_conn = None
self.console = None
self.wgetStatus = self.WGET_STS.ENDED
if terminated:
self.status = DMHelper.STS.INTERRUPTED
elif 0 >= self.localFileSize:
self.status = DMHelper.STS.ERROR
elif self.remoteFileSize > 0 and self.remoteFileSize > self.localFileSize:
self.status = DMHelper.STS.INTERRUPTED
else:
self.status = DMHelper.STS.DOWNLOADED
if not terminated:
self.onFinish()
def _cmdFinished(self, code, terminated=False):
printDBG("PwgetDownloader._cmdFinished code[%r] terminated[%r]" %
(code, terminated))
# break circular references
self.console_appClosed_conn = None
self.console_stderrAvail_conn = None
self.console = None
self.wgetStatus = self.WGET_STS.ENDED
# When finished updateStatistic based on file sie on disk
BaseDownloader.updateStatistic(self)
if not terminated:
printDBG("PwgetDownloader._cmdFinished [%s]" % self.outData)
match = re.search("Content-Length: ([0-9]+?)[^0-9]", self.outData)
if match:
self.remoteFileSize = int(match.group(1))
if terminated:
self.status = DMHelper.STS.INTERRUPTED
elif 0 >= self.localFileSize:
self.status = DMHelper.STS.ERROR
elif self.remoteFileSize > 0 and self.remoteFileSize > self.localFileSize:
self.status = DMHelper.STS.INTERRUPTED
else:
self.status = DMHelper.STS.DOWNLOADED
if not terminated:
self.onFinish()
def _cmdFinished(self, code, terminated=False):
printDBG("WgetDownloader._cmdFinished code[%r] terminated[%r]" % (code, terminated))
# When finished updateStatistic based on file sie on disk
BaseDownloader.updateStatistic(self)
if not terminated and self.remoteFileSize > 0 \
and self.remoteFileSize > self.localFileSize \
and self.curContinueRetry < self.maxContinueRetry:
self.curContinueRetry += 1
self.console.execute( E2PrioFix( self.downloadCmd ) )
return
# break circular references
self.console_appClosed_conn = None
self.console_stderrAvail_conn = None
self.console = None
self.wgetStatus = self.WGET_STS.ENDED
if terminated:
self.status = DMHelper.STS.INTERRUPTED
elif 0 >= self.localFileSize:
self.status = DMHelper.STS.ERROR
elif self.remoteFileSize > 0 and self.remoteFileSize > self.localFileSize:
self.status = DMHelper.STS.INTERRUPTED
else:
self.status = DMHelper.STS.DOWNLOADED
if not terminated:
self.onFinish()
def __init__(self):
printDBG('M3U8Downloader.__init__ ----------------------------------')
BaseDownloader.__init__(self)
self.wgetStatus = self.WGET_STS.NONE
# instance of E2 console
self.console = eConsoleAppContainer()
self.iptv_sys = None
# M3U8 list updater
self.M3U8Updater = eConsoleAppContainer()
self.M3U8Updater_appClosed_conn = eConnectCallback(self.M3U8Updater.appClosed, self._updateM3U8Finished)
self.M3U8Updater_stdoutAvail_conn = eConnectCallback(self.M3U8Updater.stdoutAvail, self._updateM3U8DataAvail)
self.M3U8ListData = ''
self.M3U8UpdaterRefreshDelay = 0
self.refreshDelay = M3U8Downloader.MIN_REFRESH_DELAY
# get only last fragments from first list, to satisfy specified duration in seconds
# -1 means, starts from beginning
self.startLiveDuration = M3U8Downloader.LIVE_START_OFFSET
# 0 means, starts from beginning
self.skipFirstSegFromList = 0
self.addStampToUrl = False
self.totalDuration = -1
self.downloadDuration = 0
self.fragmentDurationList = []
def __init__(self):
printDBG('M3U8Downloader.__init__ ----------------------------------')
BaseDownloader.__init__(self)
self.wgetStatus = self.WGET_STS.NONE
# instance of E2 console
self.console = eConsoleAppContainer()
self.iptv_sys = None
# M3U8 list updater
self.M3U8Updater = eConsoleAppContainer()
self.M3U8Updater_appClosed_conn = eConnectCallback(
self.M3U8Updater.appClosed, self._updateM3U8Finished)
self.M3U8Updater_stdoutAvail_conn = eConnectCallback(
self.M3U8Updater.stdoutAvail, self._updateM3U8DataAvail)
self.M3U8ListData = ''
self.M3U8UpdaterRefreshDelay = 0
self.refreshDelay = M3U8Downloader.MIN_REFRESH_DELAY
# get only last fragments from first list, to satisfy specified duration in seconds
# -1 means, starts from beginning
self.startLiveDuration = M3U8Downloader.LIVE_START_OFFSET
# 0 means, starts from beginning
self.skipFirstSegFromList = 0
self.addStampToUrl = False
self.totalDuration = -1
self.downloadDuration = 0
self.fragmentDurationList = []
self.maxTriesAtStart = 0
def __init__(self):
printDBG('F4mDownloader.__init__ ----------------------------------')
BaseDownloader.__init__(self)
# instance of E2 console
self.console = None
self.iptv_sys = None
def __init__(self):
printDBG('F4mDownloader.__init__ ----------------------------------')
BaseDownloader.__init__(self)
# instance of E2 console
self.console = None
self.iptv_sys = None
def _cmdFinished(self, code, terminated=False):
printDBG("PwgetDownloader._cmdFinished code[%r] terminated[%r]" % (code, terminated))
# break circular references
self.console_appClosed_conn = None
self.console_stderrAvail_conn = None
self.console = None
self.wgetStatus = self.WGET_STS.ENDED
# When finished updateStatistic based on file sie on disk
BaseDownloader.updateStatistic(self)
if not terminated:
printDBG("PwgetDownloader._cmdFinished [%s]" % self.outData)
match = re.search("Content-Length: ([0-9]+?)[^0-9]", self.outData)
if match:
self.remoteFileSize = int(match.group(1))
if terminated:
self.status = DMHelper.STS.INTERRUPTED
elif 0 >= self.localFileSize:
self.status = DMHelper.STS.ERROR
elif self.remoteFileSize > 0 and self.remoteFileSize > self.localFileSize:
self.status = DMHelper.STS.INTERRUPTED
else:
self.status = DMHelper.STS.DOWNLOADED
if not terminated:
self.onFinish()
def __init__(self):
printDBG('PwgetDownloader.__init__ ----------------------------------')
BaseDownloader.__init__(self)
self.wgetStatus = self.WGET_STS.NONE
# instance of E2 console
self.console = None
self.iptv_sys = None
def __init__(self):
printDBG('PwgetDownloader.__init__ ----------------------------------')
BaseDownloader.__init__(self)
self.wgetStatus = self.WGET_STS.NONE
# instance of E2 console
self.console = None
self.iptv_sys = None
def __init__(self):
printDBG('RtmpDownloader.__init__ ----------------------------------')
BaseDownloader.__init__(self)
self.rtmpStatus = self.RTMP_STS.NONE
# instance of E2 console
self.console = None
self.iptv_sys = None
def __init__(self):
printDBG('RtmpDownloader.__init__ ----------------------------------')
BaseDownloader.__init__(self)
self.rtmpStatus = self.RTMP_STS.NONE
# instance of E2 console
self.console = None
self.iptv_sys = None
def __init__(self):
printDBG('MergeDownloader.__init__ ----------------------------------')
BaseDownloader.__init__(self)
# instance of E2 console
self.console = None
self.iptv_sys = None
self.multi = {'urls':[], 'files':[], 'remote_size':[], 'remote_content_type':[], 'local_size':[]}
self.currIdx = 0
def __init__(self):
printDBG('HLSDownloader.__init__ ----------------------------------')
BaseDownloader.__init__(self)
# instance of E2 console
self.console = None
self.iptv_sys = None
self.totalDuration = 0
self.downloadDuration = 0
self.liveStream = False
def __init__(self):
printDBG('MergeDownloader.__init__ ----------------------------------')
BaseDownloader.__init__(self)
# instance of E2 console
self.console = None
self.iptv_sys = None
self.multi = {'urls':[], 'files':[], 'remote_size':[], 'remote_content_type':[], 'local_size':[]}
self.currIdx = 0
def __init__(self):
printDBG('WgetDownloader.__init__ ----------------------------------')
BaseDownloader.__init__(self)
self.wgetStatus = self.WGET_STS.NONE
# instance of E2 console
self.console = None
self.iptv_sys = None
self.curContinueRetry = 0
self.maxContinueRetry = 0
self.downloadCmd = ''
def __init__(self):
printDBG('WgetDownloader.__init__ ----------------------------------')
BaseDownloader.__init__(self)
self.wgetStatus = self.WGET_STS.NONE
# instance of E2 console
self.console = None
self.iptv_sys = None
self.curContinueRetry = 0
self.maxContinueRetry = 0
self.downloadCmd = ''
def __init__(self):
printDBG('WgetDownloader.__init__ ')
BaseDownloader.__init__(self)
self.wgetStatus = self.WGET_STS.NONE
# instance of E2 console
self.console = None
self.iptv_sys = None
self.curContinueRetry = 0
self.maxContinueRetry = 0
self.downloadCmd = ''
self.remoteContentType = None
self.lastErrorCode = None
self.lastErrorDesc = ''
def updateStatistic(self):
if self.infoFrom == WgetDownloader.INFO.FROM_FILE:
BaseDownloader.updateStatistic(self)
return
if self.WGET_STS.DOWNLOADING == self.wgetStatus:
print(self.outData)
dataLen = len(self.outData)
for idx in range(dataLen):
if idx + 1 < dataLen:
# default style - one dot = 1K
if '.' == self.outData[idx] and self.outData[idx + 1] in ['.', ' ']:
self.localFileSize += 1024
else:
self.outData = self.outData[idx:]
break
BaseDownloader._updateStatistic(self)
def updateStatistic(self):
if self.infoFrom == WgetDownloader.INFO.FROM_FILE:
BaseDownloader.updateStatistic(self)
return
if self.WGET_STS.DOWNLOADING == self.wgetStatus:
print self.outData
dataLen = len(self.outData)
for idx in range(dataLen):
if idx+1 < dataLen:
# default style - one dot = 1K
if '.' == self.outData[idx] and self.outData[idx+1] in ['.', ' ']:
self.localFileSize += 1024
else:
self.outData = self.outData[idx:]
break
BaseDownloader._updateStatistic(self)
def __init__(self):
printDBG('FFMPEGDownloader.__init__ ----------------------------------')
BaseDownloader.__init__(self)
# instance of E2 console
self.console = None
self.iptv_sys = None
self.totalDuration = 0
self.downloadDuration = 0
self.liveStream = False
self.headerReceived = False
self.parseReObj = {}
self.parseReObj['start_time'] = re.compile('\sstart\:\s*?([0-9]+?)\.')
self.parseReObj['duration'] = re.compile('[\s=]([0-9]+?)\:([0-9]+?)\:([0-9]+?)\.')
self.parseReObj['size'] = re.compile('size=\s*?([0-9]+?)kB')
self.parseReObj['bitrate'] = re.compile('bitrate=\s*?([0-9]+?(?:\.[0-9]+?)?)kbits')
self.parseReObj['speed'] = re.compile('speed=\s*?([0-9]+?(?:\.[0-9]+?)?)x')
self.ffmpegOutputContener = 'matroska'
self.fileCmdPath = ''
def _cmdFinished(self, code, terminated=False):
printDBG("BuxyboxWgetDownloader._cmdFinished code[%r] terminated[%r]" % (code, terminated))
# break circular references
self.console_appClosed_conn = None
self.console = None
self.wgetStatus = self.WGET_STS.ENDED
# When finished updateStatistic based on file sie on disk
BaseDownloader.updateStatistic(self)
if terminated:
self.status = DMHelper.STS.INTERRUPTED
elif 0 >= self.localFileSize:
self.status = DMHelper.STS.ERROR
elif self.remoteFileSize > 0 and self.remoteFileSize > self.localFileSize:
self.status = DMHelper.STS.INTERRUPTED
else:
self.status = DMHelper.STS.DOWNLOADED
if not terminated:
self.onFinish()
def _dataAvail(self, data):
if None == data:
return
data = self.outData + data
if '\n' != data[-1]: truncated = True
else: truncated = False
data = data.split('\n')
if truncated:
self.outData = data[-1]
del data[-1]
for item in data:
printDBG(item)
if item.startswith('{'):
try:
obj = json.loads(item.strip())
printDBG("Status object [%r]" % obj)
if "total_download_size" in obj:
self.localFileSize = obj["total_download_size"]
BaseDownloader._updateStatistic(self)
except:
printExc()
continue
def _dataAvail(self, data):
if None == data:
return
data = self.outData + data
if '\n' != data[-1]: truncated = True
else: truncated = False
data = data.split('\n')
if truncated:
self.outData = data[-1]
del data[-1]
for item in data:
printDBG(item)
if item.startswith('{'):
try:
obj = json.loads(item.strip())
printDBG("Status object [%r]" % obj)
if "total_download_size" in obj:
self.localFileSize = obj["total_download_size"]
BaseDownloader._updateStatistic(self)
except:
printExc()
continue
def updateStatistic(self):
BaseDownloader.updateStatistic(self)
return
def _cmdFinished(self, code, terminated=False):
printDBG("M3U8Downloader._cmdFinished code[%r] terminated[%r] downloadType[%s]" % (code, terminated, self.downloadType))
localStatus = DMHelper.STS.ERROR
if terminated:
BaseDownloader.updateStatistic(self)
localStatus = DMHelper.STS.INTERRUPTED
elif self.DOWNLOAD_TYPE.M3U8 == self.downloadType:
self.console_appClosed_conn = None
self.console_stdoutAvail_conn = None
if 0 < len(self.outData):
try:
m3u8Obj = m3u8.inits(self.outData, self.m3u8Url)
# uri given to m3u8 downloader should not be variant,
# format should be selected before starting downloader
# however if this was not done the firs one will be selected
if m3u8Obj.is_variant:
if 0 < len(m3u8Obj.playlists):
self._startM3U8(self._segUri(m3u8Obj.playlists[-1].absolute_uri))
localStatus = DMHelper.STS.DOWNLOADING
else:
if 0 < len(m3u8Obj.segments):
if not m3u8Obj.is_endlist:
self.liveStream = True
if -1 == self.startLiveDuration:
self.fragmentList = [self._segUri(seg.absolute_uri) for seg in m3u8Obj.segments]
else:
# some live streams only add new fragments not removing old,
# in this case most probably we not want to download old fragments
# but start from last N fragments/seconds
# self.startLiveDuration
self.fragmentList = []
currentDuration = 0
maxFragDuration = m3u8Obj.target_duration
for seg in reversed(m3u8Obj.segments):
if None != seg.duration:
currentDuration += seg.duration
else:
currentDuration += maxFragDuration
self.fragmentList.append(self._segUri(seg.absolute_uri))
if currentDuration >= self.startLiveDuration:
break
self.fragmentList.reverse()
# start update fragment list loop
#self.fragmentList = self.fixFragmentsList(self.fragmentList)
self._updateM3U8Finished(-1)
else:
self.fragmentList = [self._segUri(seg.absolute_uri) for seg in m3u8Obj.segments]
localStatus = self._startFragment()
except:
pass
elif self.liveStream and self.DOWNLOAD_TYPE.WAITTING == self.downloadType:
printDBG("m3u8 liveStream waitting finished--------------------------------")
localStatus = self._startFragment()
else:
BaseDownloader.updateStatistic(self)
printDBG("m3u8 nextFragment finished: live[%r]: r[%d], l[%d], p[%d]" % (self.liveStream, self.remoteFragmentSize, self.localFileSize, self.m3u8_prevLocalFileSize ))
if 0 >= self.localFileSize:
if not self.liveStream:
localStatus = DMHelper.STS.ERROR
else:
localStatus = self._startFragment()
#elif not self.liveStream and self.remoteFragmentSize > 0 and self.remoteFragmentSize > (self.localFileSize - self.m3u8_prevLocalFileSize):
# localStatus = DMHelper.STS.INTERRUPTED
elif 0 < (self.localFileSize - self.m3u8_prevLocalFileSize):
localStatus = self._startFragment()
elif 0 == (self.localFileSize - self.m3u8_prevLocalFileSize):
localStatus = self._startFragment(True) # retry
else:
localStatus = DMHelper.STS.INTERRUPTED
self.status = localStatus
if DMHelper.STS.DOWNLOADING == self.status:
return
# clean up at finish
if self.M3U8Updater:
self.M3U8Updater_appClosed_conn = None
self.M3U8Updater_stdoutAvail_conn = None
self.M3U8Updater = None
self.liveStream = False
if self.console:
self.console_appClosed_conn = None
self.console_stdoutAvail_conn = None
self.console.sendCtrlC() # kill # produce zombies
self.console = None
'''
if None != self.updateThread:
if self.updateThread.Thread.isAlive():
# give some time for update thread to finish
sleep(self.MIN_REFRESH_DELAY)
printDBG('m3u8 downloader killing update thread')
'''
if not terminated:
self.onFinish()
def _cmdFinished(self, code, terminated=False):
printDBG(
"M3U8Downloader._cmdFinished code[%r] terminated[%r] downloadType[%s]"
% (code, terminated, self.downloadType))
localStatus = DMHelper.STS.ERROR
if terminated:
BaseDownloader.updateStatistic(self)
localStatus = DMHelper.STS.INTERRUPTED
elif self.DOWNLOAD_TYPE.M3U8 == self.downloadType:
self.console_appClosed_conn = None
self.console_stdoutAvail_conn = None
if 0 < len(self.outData):
try:
m3u8Obj = m3u8.inits(self.outData, self.m3u8Url)
# uri given to m3u8 downloader should not be variant,
# format should be selected before starting downloader
# however if this was not done the firs one will be selected
if m3u8Obj.is_variant:
if 0 < len(m3u8Obj.playlists):
self.m3u8Url = self._segUri(
m3u8Obj.playlists[-1].absolute_uri)
self._startM3U8()
localStatus = DMHelper.STS.DOWNLOADING
else:
if 0 < len(m3u8Obj.segments):
if not m3u8Obj.is_endlist:
self.liveStream = True
if -1 == self.startLiveDuration:
self.fragmentList = [
self._segUri(seg.absolute_uri)
for seg in m3u8Obj.segments
]
else:
# some live streams only add new fragments not removing old,
# in this case most probably we not want to download old fragments
# but start from last N fragments/seconds
# self.startLiveDuration
self.fragmentList = []
currentDuration = 0
maxFragDuration = m3u8Obj.target_duration
for seg in reversed(m3u8Obj.segments):
if None != seg.duration:
currentDuration += seg.duration
else:
currentDuration += maxFragDuration
self.fragmentList.append(
self._segUri(seg.absolute_uri))
if currentDuration >= self.startLiveDuration:
break
self.fragmentList.reverse()
if len(m3u8Obj.segments) == len(
self.fragmentList) and len(
self.fragmentList
) > self.skipFirstSegFromList:
self.fragmentList = self.fragmentList[
self.skipFirstSegFromList:]
self.lastMediaSequence = m3u8Obj.media_sequence + len(
m3u8Obj.segments)
# start update fragment list loop
#self.fragmentList = self.fixFragmentsList(self.fragmentList)
self._updateM3U8Finished(-1)
else:
self.fragmentList = [
self._segUri(seg.absolute_uri)
for seg in m3u8Obj.segments
]
try:
self.totalDuration = 0
self.fragmentDurationList = []
for seg in m3u8Obj.segments:
self.totalDuration += seg.duration
self.fragmentDurationList.append(
seg.duration)
except Exception:
printExc()
self.totalDuration = -1
self.fragmentDurationList = []
localStatus = self._startFragment()
except Exception:
pass
printDBG(">>>>>>>>>>>>>>>>>> localStatus [%s] tries[%d]" %
(localStatus, self.tries))
if localStatus == DMHelper.STS.ERROR and self.tries < self.maxTriesAtStart:
self.console_appClosed_conn = None
self.console_stdoutAvail_conn = None
self.tries += 1
self._startM3U8(self.MIN_REFRESH_DELAY)
return
else:
self.tries = 0
elif self.liveStream and self.DOWNLOAD_TYPE.WAITTING == self.downloadType:
printDBG(
"m3u8 liveStream waitting finished--------------------------------"
)
localStatus = self._startFragment()
else:
BaseDownloader.updateStatistic(self)
printDBG(
"m3u8 nextFragment finished: live[%r]: r[%d], l[%d], p[%d]" %
(self.liveStream, self.remoteFragmentSize, self.localFileSize,
self.m3u8_prevLocalFileSize))
if 0 >= self.localFileSize:
if not self.liveStream:
localStatus = DMHelper.STS.ERROR
else:
localStatus = self._startFragment()
#elif not self.liveStream and self.remoteFragmentSize > 0 and self.remoteFragmentSize > (self.localFileSize - self.m3u8_prevLocalFileSize):
# localStatus = DMHelper.STS.INTERRUPTED
elif 0 < (self.localFileSize - self.m3u8_prevLocalFileSize):
if self.totalDuration > 0:
try:
self.downloadDuration += self.fragmentDurationList[
self.currentFragment]
except Exception:
printExc()
localStatus = self._startFragment()
elif 0 == (self.localFileSize - self.m3u8_prevLocalFileSize):
localStatus = self._startFragment(True) # retry
else:
localStatus = DMHelper.STS.INTERRUPTED
self.status = localStatus
if DMHelper.STS.DOWNLOADING == self.status:
return
# clean up at finish
if self.M3U8Updater:
self.M3U8Updater_appClosed_conn = None
self.M3U8Updater_stdoutAvail_conn = None
self.M3U8Updater = None
self.liveStream = False
if self.console:
self.console_appClosed_conn = None
self.console_stdoutAvail_conn = None
self.console.sendCtrlC() # kill # produce zombies
self.console = None
'''
if None != self.updateThread:
if self.updateThread.Thread.isAlive():
# give some time for update thread to finish
sleep(self.MIN_REFRESH_DELAY)
printDBG('m3u8 downloader killing update thread')
'''
if not terminated:
self.onFinish()
def updateStatistic(self):
BaseDownloader.updateStatistic(self)
return
