class BBCDL( object ): def __init__( self, url, proxy = None): try: self.outputfile = __settings__.getSetting('download_folder') + "/bbcsports.flv" except: print "Output Filename must be set" exit (-1) self.pipeHandle = None if os.name == 'nt': self.outputfile = '\\\\.\\pipe\\bbcsports.flv' # Create the named pipe import win32file, win32pipe self.pipeHandle = win32pipe.CreateNamedPipe(self.outputfile, win32pipe.PIPE_ACCESS_DUPLEX, win32pipe.PIPE_TYPE_BYTE | win32pipe.PIPE_READMODE_BYTE | win32pipe.PIPE_WAIT, 50, 4096, 4096, 10000, None) if self.pipeHandle == win32file.INVALID_HANDLE_VALUE: print "Fatal: Couldn't create named pipe. Exiting." exit (-1) else: # Create the FIFO if os.path.exists(self.outputfile): os.unlink(self.outputfile) os.mkfifo(self.outputfile) self.url = url self.proxy = proxy self.navigator = Navigator( self.proxy ) self.manifestURL = None self.drm = None self.segNum = 1 self.pos = 0 self.boxType = None self.boxSize = None self.fragNum = None self.rename = False self.prevTagSize = 4 self.tagHeaderLen = 11 self.baseTS = False self.prevAudioTS = -1 self.prevVideoTS = -1 self.TIMECODE_DURATION = 8 self.duration = 0 self.audio = False self.video = False self.prevAVC_Header = False self.prevAAC_Header = False self.AVC_HeaderWritten = False self.AAC_HeaderWritten = False self.dataQueue = Queue.Queue(50) self.dataThread = None self.dataThreadKill = False self.dataWritten = False self.needNewBootstrap = False self.flvHeader = bytearray.fromhex(unicode('464c5601050000000900000000')) self.flvHeaderLen = len(self.flvHeader) def run(self): print "Started download thread" flv = None self.baseUrl = self.url[0: self.url.rfind("/")] # Get the manifest self.manifest = self.navigator.getFile(self.url) # Parse the manifest self.parseManifest() segNum = self.segNum fragNum = self.fragNum #print segNum #print fragNum #print self.fragCount self.baseFilename = (self.streamid.attrib["streamId"] + "Seg%d" + "-Frag") % (segNum) self.fragUrl = self.baseUrl + "/" + self.streamid.attrib["url"] # Kick off the data fetching thread self.dataThread = threading.Thread(target=self.runDataThread) self.dataThread.start() while not self.dataThreadKill: fragNum, data = self.dataQueue.get() if fragNum is None: # New bootstrap info for us sys.stdout.write("Updating bootstrap info") self.UpdateBootstrapInfoWithData(data) else: sys.stdout.write("Decoding fragment %d of %d\r" % (fragNum, self.fragCount)) # Create the FLV if it doesn't exist if (flv is None): self.start = datetime.datetime.now() flvData = self.DecodeFragment(data, fragNum, None) #flv = self.WriteFlvFile("C:/" + self.baseFilename + ".flv", self.audio, self.video) self.dataWritten = True flv = self.WriteFlvFile(self.outputfile, self.audio, self.video) flv.write(flvData) else: self.DecodeFragment(data, fragNum, flv) self.dataQueue.task_done() if self.pipeHandle: self.pipeHandle.Close() self.pipeHandle = None exit(-1) def runDataThread(self): listeUserAgents = [ 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_5; fr-fr) AppleWebKit/525.18 (KHTML, like Gecko) Version/3.1.2 Safari/525.20.1', 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.186 Safari/535.1', 'Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/525.13 (KHTML, like Gecko) Chrome/0.2.149.27 Safari/525.13', 'Mozilla/5.0 (X11; U; Linux x86_64; en-us) AppleWebKit/528.5+ (KHTML, like Gecko, Safari/528.5+) midori', 'Mozilla/5.0 (Windows NT 6.0) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/13.0.782.107 Safari/535.1', 'Mozilla/5.0 (Macintosh; U; PPC Mac OS X; en-us) AppleWebKit/312.1 (KHTML, like Gecko) Safari/312', 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.12 Safari/535.11', 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.8 (KHTML, like Gecko) Chrome/17.0.940.0 Safari/535.8' ] # Init curl c = pycurl.Curl() data = CurlData() c.setopt(c.WRITEFUNCTION, data.body_callback) c.setopt(pycurl.TIMEOUT, 30) # Set up user agent c.setopt(c.USERAGENT, random.choice(listeUserAgents)) # And proxy if self.proxy: if self.proxy[0] != socks.PROXY_TYPE_HTTP_NO_TUNNEL: # Set generic options c.setopt(c.PROXY, self.proxy[1]) c.setopt(c.PROXYPORT, self.proxy[2]) c.setopt(c.PROXYUSERPWD, "%s:%s" % (self.proxy[4],self.proxy[5])) if self.proxy[0] == socks.PROXY_TYPE_HTTP: c.setopt(c.PROXYTYPE_HTTP) elif self.proxy[0] == socks.PROXY_TYPE_SOCKS4: c.setopt(c.PROXYTYPE_SOCKS4) elif self.proxy[0] == socks.PROXY_TYPE_SOCKS5: # pycurl doesn't seem to define c.PROXYTYPE_SOCKS5_HOSTNAME, so use the URL prefix method instead c.setopt(c.PROXY, "socks5h://%s" % (self.proxy[1])) #c.setopt(c.PROXYTYPE_SOCKS5) # Set up our fragment vars segNum = self.segNum fragNum = self.fragNum retries = 3 while not self.dataThreadKill: data.reset() if self.needNewBootstrap: # Download a new bit of bootstrap info c.setopt(c.URL, self.urlbootstrap) try: c.perform() self.dataQueue.put((None, data.contents)) self.needNewBootstrap = False except Exception as e: print "Curl failed to fetch new bootstrap data: %s" % e.strerror else: # Download the next fragment fragNum = fragNum + 1 if (self.segNum > 1): if (fragNum > (segNum * self.fragsPerSeg)): segNum = segNum + 1 # Segment change, update bootstrap to get new fragsPerSeg self.needNewBootstrap = True elif (fragNum <= ((segNum - 1) * self.fragsPerSeg)): segNum = segNum - 1 # Segment change, update bootstrap to get new fragsPerSeg self.needNewBootstrap = True filename1 = (self.fragUrl + "Seg%d" + "-Frag%d") % (segNum, fragNum) c.setopt(c.URL, filename1) try: c.perform() if(len(data.contents) < 1000): print "No more data available - waiting a bit." time.sleep(0.5) fragNum = fragNum - 1 continue retries = 3 self.dataQueue.put((fragNum, data.contents)) except Exception as e: retries = retries - 1 if retries > 0: # If we're retrying, roll back the fragNum fragNum = fragNum - 1 else: print "Couldn't download fragment %d" % fragNum c.close() def UpdateBootstrapInfoWithData(self, bootstrapData): bootstrapPos = 0 origFragCount = self.fragCount self.ReadBoxHeader(bootstrapData, bootstrapPos, None, None) bootstrapPos = self.pos if (self.boxType == "abst"): self.ParseBootstrapBox(bootstrapData, bootstrapPos) if origFragCount == self.fragCount: self.needNewBootstrap = True else: print "New fragment count is: %d" % self.fragCount def UpdateBootstrapInfo(self, bootstrapUrl): retries = 0 fragNum = self.fragCount while ((fragNum == self.fragCount) and (retries < 30)): bootstrapPos = 0 try: bootstrapfile = self.navigator.getFile(bootstrapUrl) except: exit(-1) self.ReadBoxHeader(bootstrapfile, bootstrapPos, None, None); #print self.boxType bootstrapPos = self.pos if (self.boxType == "abst"): self.ParseBootstrapBox(bootstrapfile, bootstrapPos); else: exit(-1) if (fragNum == self.fragCount): time.sleep(0.5) retries = retries + 1 if (retries == 30): print "Unable to update Bootstrap Information" exit (-1) def parseManifest( self ): try : tree = xml.etree.ElementTree.fromstring( self.manifest ) # Duration self.duration = float( tree.find( "{http://ns.adobe.com/f4m/1.0}duration" ).text ) self.media = tree.findall( "{http://ns.adobe.com/f4m/1.0}bootstrapInfo" )[ -1 ] self.streamid = tree.findall( "{http://ns.adobe.com/f4m/1.0}media" )[ -1 ] # Bootstrap URL self.urlbootstrap = self.media.attrib[ "url" ] except : print( "Not possible to parse the manifest" ) sys.exit( -1 ) self.urlbootstrap = self.baseUrl + "/" + self.urlbootstrap try : bootstrapfile = self.navigator.getFile(self.urlbootstrap) except : print( "Not possible to get the bootstrap file") sys.exit( -1 ) self.ReadBoxHeader(bootstrapfile, self.pos, self.boxType, self.boxSize) if (self.boxType == "abst"): self.ParseBootstrapBox(bootstrapfile, self.pos) def ReadBoxHeader(self, input_str, pos, boxType, boxSize): if (pos is None): pos = 0 self.boxSize = self.ReadInt32(input_str, pos) boxTypeString = (input_str[pos+4], input_str[pos+5], input_str[pos+6], input_str[pos+7]) self.boxType = string.join(boxTypeString, "") if (self.boxSize == 1): self.boxSize = self.ReadInt64(input_str, pos+8) -16 pos = pos + 16; else: self.boxSize = self.boxSize - 8 pos = pos + 8 self.pos = pos def ParseBootstrapBox(self, bootstrapinfo, pos): version = self.ReadByte(bootstrapinfo, pos) flags = self.ReadInt24(bootstrapinfo, (pos+1)) bootstrapVersion = self.ReadInt32(bootstrapinfo, (pos + 4)) byte = self.ReadByte(bootstrapinfo, (pos + 8)) profile = (byte & 0xC0) >> 6 self.live = (byte & 0x20) >> 5 update = (byte & 0x10) >> 4 timescale = self.ReadInt32(bootstrapinfo, (pos + 9)) currentMediaTime = self.ReadInt64(bootstrapinfo, 13) smpteTimeCodeOffset = self.ReadInt64(bootstrapinfo, 21) pos = pos + 29 movieIdentifier = self.ReadString(bootstrapinfo, pos) pos = self.pos serverEntryCount = self.ReadByte(bootstrapinfo, pos) pos += 1 qualityEntryCount = self.ReadByte(bootstrapinfo, pos) pos += 1 drmData = self.ReadString(bootstrapinfo, pos) pos = self.pos metadata = self.ReadString(bootstrapinfo, pos) pos = self.pos segRunTableCount = self.ReadByte(bootstrapinfo, pos) pos = pos + 1 for i in range(0, segRunTableCount): self.ReadBoxHeader(bootstrapinfo, pos, self.boxType, self.boxSize) i=i+1 if (self.boxType == "asrt"): self.ParseAsrtBox(bootstrapinfo, self.pos) #print "ASRT" pos = self.pos + self.boxSize fragRunTableCount = self.ReadByte(bootstrapinfo, pos) pos = pos + 1 for i in range(0, fragRunTableCount): self.ReadBoxHeader(bootstrapinfo, pos, self.boxType, self.boxSize) i=i+1 if (self.boxType == "afrt"): self.ParseAfrtBox(bootstrapinfo, self.pos) pos = self.pos + self.boxSize self.pos = pos def ReadInt32(self, input_str, pos): int32 = struct.unpack(">I", input_str[pos] + input_str[pos+1] + input_str[pos+2] + input_str[pos+3])[0] return int32 def ReadInt24(self, input_str, pos): int24 = struct.unpack(">I", '\x00' + input_str[pos] + input_str[pos+1] + input_str[pos+2])[0] return int24 def ReadByte(self, input_str, pos): unpacked_int = struct.unpack("B", input_str[pos])[0]; return unpacked_int def ReadInt64(self, input_str, pos): #hi = sprintf("%u", self.ReadInt32(str, pos)) #lo = sprintf("%u", self.ReadInt32(str, pos + 4)) hi = self.ReadInt32(input_str, pos) lo = self.ReadInt32(input_str, pos + 4) int64 = (hi * 4294967296 ) + lo #int64 = bcadd(bcmul(hi, "4294967296"), lo) return int64 def ReadString(self, frag, fragPos): strlen = 0 while (frag[fragPos + strlen] != "\x00"): strlen += 1 str = frag[fragPos: strlen] fragPos += strlen + 1 self.pos = fragPos return str def ParseAsrtBox(self, asrt, pos): version = self.ReadByte(asrt, pos) flags = self.ReadInt24(asrt, (pos + 1)) qualityEntryCount = self.ReadByte(asrt, (pos + 4)) pos = pos + 5 for i in range (0, qualityEntryCount): #qualitySegmentUrlModifiers[i] = self.ReadString(asrt, pos) i = i+1 self.segCount = self.ReadInt32(asrt, pos) self.segTable = dict( ((i,j),None) for i in range(self.segCount) for j in range(2) ) pos = pos + 4 for i in range (0, self.segCount): firstSegment = self.ReadInt32(asrt, pos) fragmentsPerSegment = self.ReadInt32(asrt, (pos + 4)) self.segTable[i, 0] = firstSegment self.segTable[i, 1] = fragmentsPerSegment pos = pos+8 i=i+1 lastSegment = self.segTable[self.segCount-1, 0] self.fragCount = self.segTable[self.segCount-1, 1] if (self.live == 1) and (self.segCount >1): try: secondLastSegment = self.segCount-2 except: pass if self.fragNum is None: self.segNum = lastSegment self.fragsPerSeg = self.segTable[secondLastSegment, 1] self.fragNum = self.segTable[secondLastSegment,0] * self.fragsPerSeg + self.fragCount - 2; self.fragCount = self.segTable[secondLastSegment,0] * self.fragsPerSeg + self.fragCount; else: self.fragCount = self.segTable[secondLastSegment,0] * self.fragsPerSeg + self.fragCount; def ParseAfrtBox(self, afrt, pos): version = self.ReadByte(afrt, pos) flags = self.ReadInt24(afrt, pos + 1) timescale = self.ReadInt32(afrt, pos + 4) qualityEntryCount = self.ReadByte(afrt, pos + 8) pos = pos + 9 self.fragEntries = self.ReadInt32(afrt, pos) self.fragTable = dict( ((i,j),None) for i in range(self.fragEntries) for j in range(4) ) pos = pos + 4 for i in range (0, self.fragEntries): firstFragment = self.ReadInt32(afrt, pos) self.fragTable[i,0] = firstFragment self.fragTable[i,1] = self.ReadInt64(afrt, pos + 4) self.fragTable[i,2] = self.ReadInt32(afrt, pos + 12) self.fragTable[i,3] = "" pos += 16 if self.fragTable[i,2] == 0: self.fragTable[i,3] = self.ReadByte(afrt, pos) pos = pos + 1 i = i + 1 if self.segCount == 1: firstFragment = 0 lastFragment = self.fragEntries if self.live == 1: if self.fragNum is None: self.fragNum = self.fragTable[lastfragment, 0] - 2 self.fragCount = self.fragTable[firstFragment, 0] else: self.fragCount = self.fragTable[lastFragment,0] elif self.fragNum is None: self.fragNum = self.fragTable[firstFragment,0] - 1 def DecodeFragment(self, frag, fragNum, flv): flvData = "" fragLen = 0 fragPos = 0 boxSize = 0 fragLen = len(frag) packetTS = None while (fragPos < fragLen): self.ReadBoxHeader(frag, fragPos, None, None) if (self.boxType == "mdat"): fragPos = self.pos break fragPos = self.pos + self.boxSize while (fragPos < fragLen): packetType = self.ReadByte(frag, fragPos) packetSize = self.ReadInt24(frag, fragPos + 1) packetTS = self.ReadInt24(frag, fragPos + 4) packetTS = (packetTS | (self.ReadByte(frag, fragPos + 7) << 24)) if (packetTS & 0x80000000): packetTS &= 0x7FFFFFFF #self.WriteFlvTimestamp(frag, fragPos, packetTS) if ((self.baseTS is False) and ((packetType == 0x08) or (packetType == 0x09))): self.baseTS = packetTS totalTagLen = self.tagHeaderLen + packetSize + self.prevTagSize if packetType == 0x08: if (packetTS >= self.prevAudioTS - self.TIMECODE_DURATION * 5): FrameInfo = self.ReadByte(frag, fragPos + self.tagHeaderLen) CodecID = (FrameInfo & 0xF0) >> 4 if (CodecID == 0x0A): AAC_PacketType = self.ReadByte(frag, fragPos + self.tagHeaderLen + 1) if (AAC_PacketType == 0x00): if (self.AAC_HeaderWritten is True): #break i = 1 #print "Skipping" else: #print("Writing AAC sequence header") self.AAC_HeaderWritten = True self.prevAAC_Header = True elif (self.AAC_HeaderWritten is False): i = 1 #print("Discarding audio packet received before AAC sequence header",) #break if (packetSize > 0): #Check for packets with non-monotonic audio timestamps and fix them if ((self.prevAAC_Header is False) and (packetTS <= self.prevAudioTS)): #print ("Fixing audio timestamp") packetTS = packetTS + self.TIMECODE_DURATION + (self.prevAudioTS - packetTS) #self.WriteFlvTimestamp(frag, fragPos, packetTS) if ((CodecID == 0x0A) and (AAC_PacketType != 0x00)): self.prevAAC_Header = False if (flv): #pAudioTagPos = flv.tell() if xbmc.Player().isPlaying(): flvData = frag[fragPos: fragPos + totalTagLen] flv.write(flvData) else: self.stop = datetime.datetime.now() elapsed = self.stop - self.start if (elapsed > datetime.timedelta(seconds=25)): self.dataThreadKill = True print "Downloads Killed" exit(-1) else: flvData = frag[fragPos: fragPos + totalTagLen] flv.write(flvData) #flvData = frag[fragPos: fragPos + totalTagLen] #flv.write(flvData) else: #flvData .= substr($frag, $fragPos, $totalTagLen); flvData = flvData + frag[fragPos: fragPos + totalTagLen] self.prevAudioTS = packetTS pAudioTagLen = totalTagLen else: i = 1 #print ("Skipping small sized audio packet") else: i= 1 #print "Test" if (self.audio is False): self.audio = True #break elif packetType == 0x09: if (packetTS >= self.prevVideoTS - self.TIMECODE_DURATION * 5): FrameInfo = self.ReadByte(frag, fragPos + self.tagHeaderLen) FrameType = (FrameInfo & 0xF0) >> 4 CodecID = FrameInfo & 0x0F if (FrameType == 0x05): printf("Skipping video info frame") if (CodecID == 0x07): AVC_PacketType = self.ReadByte(frag, fragPos + self.tagHeaderLen + 1) if (AVC_PacketType == 0x00): if (self.AVC_HeaderWritten is True): i = 1 #print ("Skipping AVC sequence header") else: #print("Writing AVC sequence header") self.AVC_HeaderWritten = True self.prevAVC_Header = True elif (self.AVC_HeaderWritten is False): i = 1 #print ("Discarding video packet received before AVC sequence header") if (packetSize > 0): #Check for packets with non-monotonic video timestamps and fix them if ((self.prevAVC_Header is False) and ((CodecID == 0x07) and (AVC_PacketType != 0x02)) and (packetTS <= self.prevVideoTS)): #print("Fixing video timestamp") packetTS = packetTS + self.TIMECODE_DURATION + (self.prevVideoTS - packetTS) #self.WriteFlvTimestamp(frag, fragPos, packetTS) if ((CodecID == 0x07) and (AVC_PacketType != 0x00)): self.prevAVC_Header = False if (flv): if xbmc.Player().isPlaying(): flvData = frag[fragPos: fragPos + totalTagLen] flv.write(flvData) else: self.stop = datetime.datetime.now() elapsed = self.stop - self.start if (elapsed > datetime.timedelta(seconds=25)): self.dataThreadKill = True print "Downloads Killed" exit(-1) else: flvData = frag[fragPos: fragPos + totalTagLen] flv.write(flvData) #flvData = frag[fragPos: fragPos + totalTagLen] #flv.write(flvData) #exit(-1) #flvData = frag[fragPos: fragPos + totalTagLen] #flv.write(flvData) #pVideoTagPos = flv.tell() #fwrite($flv, substr($frag, $fragPos, $totalTagLen), $totalTagLen) else: #flvData .= substr($frag, $fragPos, $totalTagLen) flvData = flvData + frag[fragPos:fragPos+totalTagLen] self.prevVideoTS = packetTS pVideoTagLen = totalTagLen else: print ("Skipping small sized video packet") if (self.video is False): self.video = True #break elif packetType == 0x12: i = 1