def extractElementsAlbum(table): x = [] i = 0 for value in table.split( '<li class="topListenedBox click js-lateral-info" onclick=\'albumClick(' ): if i > 0: element = {} title = Decoder.extract('<div class="topListenedBoxDiv">', '</div>', value).replace('<br>', " - ") link = Decoder.extract('","', '");\'>', value) img = Decoder.extract(' data-src="', '" ', value) element["title"] = title if link.find(".mp3") == -1: element["link"] = base64.standard_b64encode( Fildonet.ALBUM + link) else: element["link"] = link element["thumbnail"] = img logger.info("append3: " + title + ", link: " + element["link"] + ", thumbnail: " + element["thumbnail"]) x.append(element) i += 1 return x
def searchLists(param): url = ( "https://www.googleapis.com/customsearch/v1element?" "key=AIzaSyCVAXiUzRYsML1Pv6RwSG1gunmMikTzQqY" "&rsz=filtered_cse" "&num=20" "&hl=en" "&prettyPrint=false" "&source=gcsc" "&gss=.com" "&sig=8bdfc79787aa2b2b1ac464140255872c" "&cx=013305635491195529773:0ufpuq-fpt0" ) url += "&q=" + param + "&sort=date&googlehost=www.google.com&callback=google.search.Search.apiary846" results = Pastebin.getContentFromUrl(url) x = [] jsonString = Decoder.extract(',"results":', "]});", results) logger.debug(jsonString) for jsonResult in results.split('{"GsearchResultClass"'): element = {} link = Decoder.extract('"url":"', '","', jsonResult) if "pastebin.com" in link and "/raw/" not in link: link = link[: link.rfind("/")] + "/raw/" + link[link.rfind("/") + 1 :] title = Decoder.extract('"title":"', '","titleNoFormatting"', jsonResult) if "http" in link: logger.debug("appending result: " + title + ", url: " + link) element["title"] = title element["link"] = link x.append(element) return x
def getChannels(page): x = [] start = False if str(page) == '0': start = True page=Sports4u.MAIN_URL html = Sports4u.getContentFromUrl(page,"",Sports4u.cookie,"") #print html if start and 'live-channels-list">' in html: #it's a list, needs decode table = Decoder.extract('live-channels-list">','</li><br>',html) logger.debug("using menu table: "+table) x = Sports4u.extractElements(table) logger.debug("channel list logic done!") else: iframeUrl = Decoder.extract('<iframe frameborder="0" marginheight="0" marginwidth="0" height="490" ','"></iframe>',html) iframeUrl = Decoder.extract('src="','"',iframeUrl) logger.debug("iframeUrl is: "+iframeUrl) html2 = Sports4u.getContentFromUrl(url=iframeUrl,referer=page) logger.debug("html is: "+html2) file = Cricfreetv.seekIframeScript(html2,page,iframeUrl) logger.debug("Finished file logic, obtained file: "+file) element = {} element["link"] = file element["title"] = "Watch streaming" x.append(element) return x
def getChannels(page): x = [] if str(page) == "0": page = Live9net.MAIN_URL html = Live9net.getContentFromUrl(page, "", Live9net.cookie, "") # print html if html.find("ESPN</") > -1: # it's a list, needs decode table = Decoder.extract("ESPN</", "<div>", html) x = Live9net.extractElements(table) logger.debug("live9 channels logic done!") else: iframeUrl = Decoder.extract('src="', '"></iframe>', html) html2 = Live9net.getContentFromUrl(iframeUrl, "", Live9net.cookie, page) # print html2 if html2.find('src="http://sawlive.tv/') > -1 or html2.find('src="http://www3.sawlive') > -1: if html2.find('src="http://sawlive.tv/') > -1: scriptSrc = Decoder.extractWithRegex("http://sawlive", '"></script>', html2).replace( '"></script>', "" ) else: scriptSrc = Decoder.extractWithRegex("http://www3.sawlive", '"></script>', html2).replace( '"></script>', "" ) finalRtmpUrl = Decoder.extractSawlive(scriptSrc, iframeUrl) element = {} element["link"] = finalRtmpUrl element["title"] = "Watch channel" element["permalink"] = True logger.debug("finished append element!") x.append(element) return x
def getChannels(page): x = [] logger.debug("using tvshowme...") if str(page) == '0': page=Tvshowme.MAIN_URL html = Tvshowme.getContentFromUrl(page,"",Tvshowme.cookie,"") table = Decoder.extract('<span class="yawp_wim_title">Latest 150 Posts</span> <ul>','</ul>',html) x = Tvshowme.extractElements(table) elif page=="search": #display keyboard, it will wait for result keyboard = XBMCUtils.getKeyboard() keyboard.doModal() text = "" if (keyboard.isConfirmed()): text = keyboard.getText() page = "http://www.tvshow.me/?s="+urllib.quote(text) html = Tvshowme.getContentFromUrl(url=page) logger.debug(html) table = Decoder.extract('<div id="content"', '<h3 class="assistive-text">', html) logger.debug("table is: "+table) x = Tvshowme.extractLinks2(table) else: html = Tvshowme.getContentFromUrl(page, "", Tvshowme.cookie, Tvshowme.MAIN_URL) logger.debug(html) table = Decoder.extract('<div id="content"', '</article>', html) x = Tvshowme.extractLinks(table) return x
def getChannels(page): x = [] if str(page) == '0': html = Mamahdcom.getContentFromUrl(Mamahdcom.MAIN_URL,"",Mamahdcom.cookie,"") else: html = Mamahdcom.getContentFromUrl(page,"",Mamahdcom.cookie,"") #print html if page=='0': #menu table = Decoder.extract('<div class="standard row channels">','</div>',html) logger.debug("table is: "+table) x = Mamahdcom.extractElements(table) logger.debug("mamahd channels logic done!") else: iframeHtml = Decoder.extract("<iframe ","</iframe>",html) iframeUrl = Decoder.extract('src="','"',iframeHtml) html2 = Mamahdcom.getContentFromUrl(url=iframeUrl,referer=page) logger.debug("obtained html from iframe: "+iframeUrl+"; html: "+html2) if 'src="http://hdcast.org' in html2: logger.debug("found script, launching logic...") scriptUrl = Decoder.extract('<script type="text/javascript" src="','"></script>',html2) logger.debug("extracting script url... from: "+scriptUrl) iframeUrl2 = Mamahdcom.extractScriptIframeUrl(html2,scriptUrl,iframeUrl) logger.debug("script url extracted: "+iframeUrl2) finalRtmpUrl = Mamahdcom.extractFinalRtmpUrl(iframeUrl2,iframeUrl) logger.debug("rtmp extracted is: "+finalRtmpUrl) element = {} element["link"] = finalRtmpUrl element["title"] = "Watch channel" element["permaLink"] = True logger.debug("finished append element!") x.append(element) return x
def extractFinalRtmpUrl(url,referer): rtmpUrl = "" html = Mamahdcom.getContentFromUrl(url,"",Mamahdcom.cookie,referer) if 'file:\'' in html: file = Decoder.extract("file:'",'\',',html) rtmp = file[0:file.rfind("/") + 1] playpath = file[file.rfind("/") + 1:] swfUrl = "" secureToken = "SECURET0KEN#yw%.?()@W!" if url.find("hdcast.org") > -1: swfUrl = "http://player.hdcast.org/jws/jwplayer.flash.swf" rtmpUrl = rtmp + " playPath=" + playpath + " swfUrl=" + swfUrl + " pageUrl=" + url + " flashver=WIN/2019,0,0,226 live=true timeout=14 token=" + secureToken logger.debug("built final rtmp link: " + rtmpUrl) elif 'allowtransparency="true" src=' in html: logger.debug("using second way...") secondIframe = Decoder.extract('allowtransparency="true" src=', ' ', html).replace("&","&") logger.debug("found second way url: " + secondIframe+", referer: "+url) headers = { "User-Agent": Downloader.USER_AGENT, "Accept-Language": "en-US,en;q=0.8,es-ES;q=0.5,es;q=0.3", "Upgrade-Insecure-Requests" : "1", "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "Referer": url } html2 = Mamahdcom.getContentFromUrl(url=secondIframe,headers=headers) logger.debug("html2 is: "+html2) if 'file:"' in html2: rtmpUrl = Decoder.extract('file:"', '",', html2) logger.debug("using m3u8 for: "+rtmpUrl) return rtmpUrl
def getChannels(page='0'): x = [] if str(page) == '0': page = Yomvies.CHANNELS_PAGE logger.debug("loading json data from: "+page) bruteJSON = Yomvies.getContentFromUrl(page,"",Yomvies.cookie,Yomvies.MAIN_URL) logger.debug("parsing string to json...") i = 0 for jsonChannel in bruteJSON.split('{"CodCadenaTv":'): if i>0: element = {} codTv = Decoder.extract('"','"',jsonChannel) element["title"] = Decoder.extract('"Nombre":"','"',jsonChannel) element["thumbnail"] = Decoder.extract('"Logo":"','"',jsonChannel).replace("\\","") m3u8Url = Decoder.extract('"PuntoReproduccion":"','"',jsonChannel).replace("{network}",Yomvies.NETWORK).replace("\\","") logger.debug("Appending channel: "+element["title"]+", with url: "+m3u8Url+", img: "+element["thumbnail"]) headers = 'Referer='+codTv element["link"] = m3u8Url+"|"+headers x.append(element) i+=1 else: link = Yomvies.extractTargetVideo(page) element = {} element["title"] = page element["link"] = link element["finalLink"] = True x.append(element) return x
def extractChannel(html,referer): element = {} logger.debug('processing html...') if html.find('<iframe scrolling="no" marginwidth="0" marginheight="0" frameborder="0" width="650" height="400" src="')>-1: element = Cineestrenostv.extractIframeChannel(html,referer) elif html.find('.php')>-1 and referer.find(".php")==-1: logger.debug("proccessing level 1, cookie: "+Cineestrenostv.cookie) iframeUrl = Decoder.extractWithRegex('http://','.php',html) if iframeUrl.find('"')>-1: iframeUrl = iframeUrl[0:iframeUrl.find('"')] html2 = Cineestrenostv.getContentFromUrl(iframeUrl,"",Cineestrenostv.cookie,referer) if html2.find('<iframe scrolling="no" marginwidth="0" marginheight="0" frameborder="0" width="650" height="400" src="')>-1 or '<iframe scrolling="no" marginwidth="0" marginheight="0" frameborder="0" allowfullscreen width="653" height="403" src="' in html2: element = Cineestrenostv.extractIframeChannel(html2,iframeUrl) elif html.find('<iframe scrolling="no" marginwidth="0" marginheight="0" frameborder="0" width="650" height="500" src="')>-1: iframeUrl = Decoder.extract('<iframe scrolling="no" marginwidth="0" marginheight="0" frameborder="0" width="650" height="500" src="','"></iframe>',html) #same case with different width and height: TODO: change to regex!! html2 = Cineestrenostv.getContentFromUrl(iframeUrl,"","",referer) if html2.find('<th scope="col"><a href="/')>-1: partialLink = Decoder.extract('<th scope="col"><a href="/','"><font color="ffffff">',html2) completeLink = Cineestrenostv.MAIN_URL+"/"+partialLink html3 = Cineestrenostv.getContentFromUrl(completeLink,"",Cineestrenostv.cookie,iframeUrl) if html3.find('<iframe scrolling="no" marginwidth="0" marginheight="0" frameborder="0" width="650" height="400" src="')>-1: element = Cineestrenostv.extractIframeChannel(html3,completeLink) elif referer.find("php")!=-1: referer = referer.replace("ñ","%C3%B1") html2 = Cineestrenostv.getContentFromUrl(referer,"",Cineestrenostv.cookie,referer) element = Cineestrenostv.extractIframeChannel(html2,referer) return element
def extractTargetVideoJSON(page): title = '' link = '' thumbnail = '' headers = Youtube.buildHeaders() response = Youtube.getContentFromUrl(url=str(page + "?pbj=1"), headers=headers) logger.debug("response is: "+response) try: responseJ = Decoder.extract('ytplayer.config = ','};',response)+"}" #logger.debug("json extracted is: " + responseJ) jsonResponse = json.loads(responseJ) logger.debug("json loaded") bruteVideoInfo = jsonResponse["args"] logger.debug("obtained brute video info...") title = bruteVideoInfo["title"] url = bruteVideoInfo["adaptive_fmts"] url = Decoder.extract('url=',",",url) url = urllib.unquote(url) #url = url[:-1] thumbnail = bruteVideoInfo["thumbnail_url"] logger.debug("extracted final url: "+url) except: logger.error("error parsing video info") pass element = {} element["title"] = title element["link"] = link element["thumbnail"] = thumbnail element["finalLink"] = True return element
def getWidthAndHeightParams(html): subUrl = "" if html.find("; v_width=")>-1: width = Decoder.extract("; v_width=",";",html) height = Decoder.extract("; v_height=",";",html) subUrl = "&vw="+width+"&vh="+height logger.debug("width-height subUrl now is: "+subUrl) return subUrl
def extractElements(table): x = [] for value in table.split('\n'): if value.find("acestream://")>-1: element = {} element["title"] = unicode(Decoder.extract("// ",'(',value), errors='replace') element["link"] = Decoder.extractWithRegex("acestream:",'\"',value).replace('"',"") logger.debug("append: "+element["title"]+", link: "+element["link"]) x.append(element) return x
def getWidthAndHeightParams(html): subUrl = "" if html.find("; width='")>-1: width = Decoder.extract("; width='","'",html) height = Decoder.extract("; height='","'",html) subUrl = "&width="+width+"&height="+height elif html.find("; v_height=")>-1: width = Decoder.extract("; v_width=",";",html) height = Decoder.extract("; v_height=",";",html) subUrl = "&vw="+width+"&vh="+height return subUrl
def decode_file(self, input_filename: str, output_filename: str, encoding: str) -> NoReturn: self.create_new_output_file(output_filename) table = self.read_table_from_coded_file(input_filename, encoding) bits_iterator = self.read_coded_bits_from_coded_file(input_filename, encoding) decoder = Decoder(table) coded_chars_iterator = decoder.decode_lazy_from_bits(bits_iterator) with open(output_filename, encoding=encoding, mode='a') as file: for char in coded_chars_iterator: file.write(char)
def extractScriptLevel3(scriptUrl,referer=''): html4 = Cineestrenostv.getContentFromUrl(scriptUrl, "", Cineestrenostv.cookie, referer) finalIframeUrl = Decoder.extractWithRegex('http://', '%3D"', html4) finalIframeUrl = finalIframeUrl[0:len(finalIframeUrl) - 1] logger.debug("proccessing level 4, cookie: " + Cineestrenostv.cookie) finalHtml = Cineestrenostv.getContentFromUrl(finalIframeUrl, "", Cineestrenostv.cookie, referer) # print "final level5 html: "+finalHtml logger.debug("proccessing level 5, cookie: " + Cineestrenostv.cookie) playerUrl = Decoder.decodeBussinessApp(finalHtml, finalIframeUrl) return playerUrl
def extractElements(table): x = [] for fieldHtml in table.split('<li>'): if fieldHtml.find("<a href=")>-1: element = {} element["link"] = Decoder.extract('<a href="','"',fieldHtml) element["title"] = Decoder.extract('alt="','">',fieldHtml) element["thumbnail"] = Decoder.extract('src="','" ',fieldHtml) logger.debug("found title: "+element["title"]+", link: "+element["link"]+", thumbnail: "+element["thumbnail"]) if len(element["title"])>0: x.append(element) return x
def extractScriptIframeUrl(html,scriptUrl,referer): iframeUrl = "" logger.debug("extracting script iframe... url: "+scriptUrl) scriptContent = Mamahdcom.getContentFromUrl(scriptUrl,"",Mamahdcom.cookie,referer) #print scriptContent iframeUrl = Decoder.extract('src="',"'",scriptContent) logger.debug("brute iframeUrl is: "+iframeUrl) if iframeUrl.find("?u=")>-1: if '<script type="text/javascript"> fid="' in html: id = Decoder.extract('<script type="text/javascript"> fid="','"; ',html) iframeUrl = iframeUrl+id+Mamahdcom.getWidthAndHeightParams(html) return iframeUrl
def extractElements(table): x = [] for fieldHtml in table.split('</a>'): logger.debug("using html: "+fieldHtml) element = {} element["link"] = Decoder.extract('href="','"',fieldHtml) element["title"] = Decoder.extract("<span>","</span>",fieldHtml) element["thumbnail"] = Decoder.extract('<img src="','"',fieldHtml) element["permaLink"] = True logger.debug("found title: "+element["title"]+", link: "+element["link"]+", thumb: "+element["thumbnail"]) if "http" in element["link"]: x.append(element) return x
def extractLinks2(html): x = [] i=0 for value in html.split('<h2 class="entry-title ">'): if i>1: logger.debug("partial html is: "+value) element = {} title = Decoder.extract(' title="','"', value) link = Decoder.extract('href="','"', value) element["title"] = title.replace("Permalink to ","").replace("–","-") element["link"] = link x.append(element) i+=1 return x
def drawBbcCoUkNew(url): htmlContent = Downloader.getContentFromUrl(url=url) title = Decoder.extract('<p class="story-body__introduction">', "</p><div", htmlContent) if 'property="articleBody"' in htmlContent: body = Decoder.extract( 'property="articleBody"', " </div>", htmlContent, ) body = body.replace('<span class="off-screen">Image copyright</span>', "") body = body.replace('<span class="story-image-copyright">AFP</span>', "") body = body.replace('<span class="story-image-copyright">Reuters</span>', "") body = body.replace('<span class="off-screen">Image caption</span>', "") body = body.replace('<span class="off-screen">Media caption</span>', "") while '<span class="media-caption__text">' in body: line = Decoder.extractWithRegex('<span class="media-caption__text">', "</span>", body) body = body.replace(line, "") elif 'class="text-wrapper"' in htmlContent: # special content body = Decoder.extract('class="text-wrapper"', "</p>\n", htmlContent) dates = Decoder.extractWithRegex('<div class="date', "</div>", body) lastUpdate = Decoder.extractWithRegex('<p class="date ', "</p>", body) body = body.replace(dates, "") body = body.replace(lastUpdate, "") elif '<figcaption class="sp-media-asset' in htmlContent: body = Decoder.extract('<figcaption class="sp-media-asset', "</p><div ", htmlContent) if ">" in body: body = body[body.find(">") + 1 :] body = Decoder.removeHTML(body).replace(".", ".\n").replace(">", "") logger.debug("body is: " + body) drawNew(textContent=(body))
def extractIframeChannel(contentHtml,referer): logger.debug("proccessing level 2, cookie: "+Cineestrenostv.cookie) iframeUrl2 = "dummy url" if contentHtml.find('<iframe scrolling="no" marginwidth="0" marginheight="0" frameborder="0" width="653" height="403" src="')>-1: iframeUrl2 = Decoder.extract('<iframe scrolling="no" marginwidth="0" marginheight="0" frameborder="0" width="653" height="403" src="','"></iframe>',contentHtml).replace("ñ","%C3%B1") #same case with different width and height: TODO: change to regex!! elif contentHtml.find('<iframe scrolling="no" marginwidth="0" marginheight="0" frameborder="0" width="650" height="400" src="')>-1: iframeUrl2 = Decoder.extract('<iframe scrolling="no" marginwidth="0" marginheight="0" frameborder="0" width="650" height="400" src="','"></iframe>',contentHtml).replace("ñ","%C3%B1") #normal cases, standar width and height elif '<iframe scrolling="no" marginwidth="0" marginheight="0" frameborder="0" allowfullscreen width="653" height="403" src="' in contentHtml: iframeUrl2 = Decoder.extract('<iframe scrolling="no" marginwidth="0" marginheight="0" frameborder="0" allowfullscreen width="653" height="403" src="','"></iframe>', contentHtml).replace("ñ", "%C3%B1") # normal cases, standar width and height if iframeUrl2!="dummy url": html3 = Cineestrenostv.getContentFromUrl(iframeUrl2,"","",referer) return Cineestrenostv.mainLogicExtractIframeChannel(html3,iframeUrl2) else: return Cineestrenostv.mainLogicExtractIframeChannel(contentHtml,referer)
def extractElements(table,domain): x = [] i = 0 for value in table.split('"name"'): if i>0: element = {} title = Decoder.extract('"','"',value).replace('- ','') link = Decoder.extract('shortcut":"','"',value) element["title"] = title element["link"] = domain+"/channel/"+link+"/frame" logger.debug("append: "+title+", link: "+element["link"]) x.append(element) i+=1 return x
def extractElements(table): x = [] i = 0 for value in table.split('<li>'): logger.debug("loop: "+str(i)) if i>0: element = {} title = Decoder.extract(">",'</a></li>',value) link = Decoder.extract("href=\"",'"',value) element["title"] = title element["link"] = base64.b64encode(str(Zonasportsme.MAIN_URL+link)) logger.debug("append: "+title+", link: "+element["link"]) x.append(element) i+=1 return x
def extractElementsArtist(html): x = [] i=0 for value in html.split('<li class="topListenedBox click js-lateral-info " >'): if i>0: element = {} title = Decoder.extract('<div class="topListenedBoxDiv click js-lateral-info ">','</div>',value) element["title"] = title element["link"] = base64.standard_b64encode(Fildonet.ARTIST+title) if value.find('data-src="')!=-1: element["thumbnail"] = Fildonet.MAIN_URL+Decoder.extract('data-src="','" ',value) logger.info("append1: "+title+", link: "+element["link"]) x.append(element) i+=1 return x
def run(self): logger.debug("DownloadThread.run") self.running = True app = App.get_running_app() self.progressDialog.value = 0 self.progressDialogLabel.text = "decoding link: "+self.url decoded_link = Decoder.decodeLink(self.url) #copy to clipboard app.copy(decoded_link) if decoded_link!='' and decoded_link.find("http")>-1: app.message("Info","Link has been decoded (from "+self.url+"):\n"+decoded_link+" has been decoded and copied to clipboard.\nDownload should be started/resumed.") app.download_screen.ids.label_text_message.text=decoded_link elif self.aborted: app.message("Error","File "+app.target_file+" has not been downloaded, please try again and make sure remote url exists.") separationChar = '/' fileName = '' if self.url.find("/")>-1: fileName = self.url[self.url.rfind("/")+1:] if fileName.find(".")==-1 and decoded_link.find("/"): fileName = decoded_link[decoded_link.rfind("/")+1:] self.downloadfile(decoded_link,self.folder+separationChar+fileName,[],False,True,self.progressDialog,self.progressDialogLabel) self.running = False app.download_screen.ids.loading.opacity=0 if self.aborted: app.message("Info","Download proccess has been stopped for file: \n "+app.target_file)
def extractElementsAlbum(table): x = [] i = 0 for value in table.split('<div class="album">'): if i>0: element = {} title = Decoder.extract(' alt="','"/>',value) link = Decoder.extract('<a href="','" ',value) img = Decoder.extract('src="','" ',value) element["title"] = title element["link"] = Redmp3cc.MAIN_URL+link element["thumbnail"] = Redmp3cc.MAIN_URL+img logger.info("append: "+title+", link: "+element["link"]) x.append(element) i+=1 return x
def extractIframe(html,referer): iframeUrl = '' if '<iframe frameborder="0" marginheight="0" allowfullscreen="true" marginwidth="0" height="555" src="' in html: iframeUrl = Decoder.extract('<iframe frameborder="0" marginheight="0" allowfullscreen="true" marginwidth="0" height="555" src="','"',html) elif '<iframe frameborder="0" marginheight="0" marginwidth="0" height="490" src="' in html: iframeUrl = Decoder.extract('<iframe frameborder="0" marginheight="0" marginwidth="0" height="490" src="','"',html) if "'" in iframeUrl: iframeUrl = iframeUrl[0:iframeUrl.find("'")] logger.debug("level 1, iframeUrl: "+iframeUrl+", cookie: "+Cricfreetv.cookie) if iframeUrl!='': html = Cricfreetv.getContentFromUrl(iframeUrl,"",Cricfreetv.cookie,referer) file = Cricfreetv.seekIframeScript(html,referer,iframeUrl) item = {} item["title"] = referer item["link"] = file return item
def parseListSongs(songs,html): x = [] #jsonSongs = json.load(songs) for elementHtml in html.split("<Channel "): element = {} title = Decoder.extract('name="','"',elementHtml) channel = Decoder.extract('<RTMP>','</RTMP>',elementHtml) url = Decoder.extract("<URL>","</URL>",elementHtml) element["title"] = title element["link"] = channel+" playpath="+(url.replace(' ','%20'))+" pageUrl="+R977Musiccom.MAIN_URL+" swfUrl="+R977Musiccom.REFERER element["finalLink"] = True if element["link"].find("rtmp")==0: logger.info("found element: "+element["title"]+", with link: "+element["link"]) x.append(element) return x
def extractLinks(html): x = [] i=0 for value in html.split(' href='): if i>1: element = {} title = Decoder.extract('>','</a>', value).replace("–","-") link = Decoder.extract('"','"', value) element["title"] = title element["link"] = link element["finalLink"] = True if "<img" not in title and "tvshow.me" not in link: logger.debug("append: " + title + ", link: " + element["link"]) x.append(element) i+=1 return x
def extractNewIframeChannel(html3,iframeUrl2): element = {} if html3.find("http://telefivegb.com/")>-1: logger.debug("found telefivegb.com link, using that link to...") newUrl = Decoder.extractWithRegex('http://telefivegb.com/','"',html3).replace('"',"") elif html3.find("http://verlatelegratis.net")>-1: logger.debug("found verlatelegratis.net link, using that link to...") newUrl = Decoder.extractWithRegex('http://verlatelegratis.net','"',html3).replace('"',"") html4 = Cineestrenostv.getContentFromUrl(newUrl,"",Cineestrenostv.cookie,iframeUrl2) if html4.find("http://www.playerhd1.pw/")>-1: logger.debug("found playerhd1.pw, using that link, continue...") element = Cineestrenostv.extractScriptPlayerHd1pw(html4,newUrl) else: logger.debug("possible redirect to his domains: "+html4+", try again..."+newUrl) element = Cineestrenostv.extractNewIframeChannel(html4,newUrl) return element
def extractElements(page,tableHtml): x = [] i = 0 for value in tableHtml.split('<a '): if i>0: element = {} title = Decoder.extract(">","</a>",value) link = Decoder.extract("href=\"","\"",value) element["title"] = page+" ver. "+title element["link"] = page[0:1]+"00/"+link if "://" not in element["link"]: element["link"] = RTVE.MAIN_URL+element["link"] logger.debug("append: "+element["title"]+", link: "+element["link"]) x.append(element) i+=1 return x
def getChannels(page,cookie='',referer=''): x = [] html = "" if str(page) == '0': x = Redmp3cc.getMainSections() elif str(page) == 'songs.html': page=Redmp3cc.MAIN_URL+"/" html = Downloader.getContentFromUrl(page,"",cookie,"") x = Redmp3cc.extractElementsPlayer(html) elif str(page).find('search.html')!=-1: if str(page).find('search.html/')==-1: keyboard = xbmc.Keyboard("") keyboard.doModal() text = "" if (keyboard.isConfirmed()): text = keyboard.getText() x = Redmp3cc.search(text) else: text = Decoder.rExtract('search.html/','/',page) page = int(page[page.rfind('/')+1:]) x = Redmp3cc.search(text,page) elif str(page).find(".html")!=-1: if str(page) == 'albums.html'!=-1: page = Redmp3cc.MAIN_URL html = Downloader.getContentFromUrl(page,"",cookie,"") x = Redmp3cc.extractElementsAlbum(html) else: html = Downloader.getContentFromUrl(page,"",cookie,"") x = Redmp3cc.extractElementsPlayer(html) else: logger.info("page is: "+page) response = Redmp3cc.getContentFromUrl(page,"",cookie,Redmp3cc.MAIN_URL,True) #logger.info("will be used a mp3 url: "+Decoder.extract('<a href="','">here',response)) host = response[response.find("://")+len("://"):] if host.find("/")>-1: host = host[0:host.find("/")] cookie = Redmp3cc.cookie referer = page logger.info("cookie is: "+cookie+", referer is: "+referer) headers = downloadtools.buildMusicDownloadHeaders(host,cookie,referer) filename= Decoder.extract('filename=','&',response) #ROOT_DIR = xbmcaddon.Addon(id='org.harddevelop.kodi.juke').getAddonInfo('path') ROOT_DIR = xbmc.translatePath('special://temp/') logger.info("using special root folder: "+ROOT_DIR) downloadtools.downloadfile(response,ROOT_DIR+"/"+filename,headers,False,True) x.append(Redmp3cc.buildDownloadedFile(xbmc.makeLegalFilename(ROOT_DIR+"/"+filename))) return x
def getChannels(page): start = False #logger.debug("Current page is: "+page) if str(page) == '0' or str(page)=='1': if str(page)=='0': start = True page=Vigoal.MAIN_URL page = urllib.unquote_plus(page) html = Vigoal.getContentFromUrl(page,"",Vigoal.cookie,"") x = [] if page.find(".html")==-1: if start: element = {} element["link"] = '1' element["title"] = 'Display by event' x.append(element) table = Decoder.extract("<center><table><tr><td>","</tr></table></center>",html) for fieldHtml in table.split('<a href="'): element = {} element["link"] = urllib.quote_plus(str(fieldHtml[0:fieldHtml.find('"')])) element["title"] = fieldHtml[fieldHtml.find('title="')+len('title="'):] element["title"] = element["title"][0:element["title"].find('"')].replace("-"," ").replace("en directo","").replace("Live Stream","").replace("\n","").replace("\t","").replace(" ","").strip() #cleaned while element["title"].find("<")>-1: #clean tags tag = Decoder.extract("<",">",element["title"]) element["title"] = element["title"].replace("<"+tag+">","") element["thumbnail"] = fieldHtml[fieldHtml.find('<img src="')+len('<img src="'):] element["thumbnail"] = element["thumbnail"][0:element["thumbnail"].find('"')] logger.debug("found title: "+element["title"]+", link: "+element["link"]+", thumb: "+element["thumbnail"]) if element["link"].find("http")==0: x.append(element) else: #display program content table = Decoder.extract("<h2>Events Today:</h2>","</ul>",html) #instead could be used <div class="ppal"> but... fate I suppose i = 0 for fieldHtml in table.split('<li class="">'): if i>0: element = {} element["link"] = Decoder.extract('<a href="','">',fieldHtml) element["title"] = Decoder.extract(' - ','</div>',fieldHtml) if fieldHtml.find('"><h2>')>-1: titleLine = Decoder.extract('"><h2>',"</h2>",fieldHtml) else: titleLine = Decoder.rExtract('html">',"</a></div>",fieldHtml) timeLine = (element["title"].replace("</b>","").replace(" - ","")).replace("-"," ").replace("en directo","").replace("Live Stream","").replace("\n","").replace("\t","").replace(" ","").strip() #cleaned element["title"] = timeLine+" - "+titleLine while element["title"].find("<")>-1: #clean tags tag = Decoder.extract("<",">",element["title"]) element["title"] = element["title"].replace("<"+tag+">","") element["thumbnail"] = fieldHtml[fieldHtml.find('<img src="')+len('<img src="'):] element["thumbnail"] = Vigoal.MAIN_URL+element["thumbnail"][0:element["thumbnail"].find('"')] logger.debug("found title: "+element["title"]+", link: "+element["link"]+", thumb: "+element["thumbnail"]) element["link"] = urllib.quote_plus(str(Vigoal.MAIN_URL+element["link"])) x.append(element) i+=1 else: x.append(Vigoal.extractChannel(html,page)) return x
def getChannels(page): x = [] if str(page) == '0': page=Acetvru.MAIN_URL html = Acetvru.getContentFromUrl(page,"",Acetvru.cookie,"") html = Decoder.extract('var playlist = [',']',html) x = Acetvru.extractElements(html) return x
def extractElementsArtist(html): x = [] i = 0 for value in html.split( '<li class="topListenedBox click js-lateral-info " >'): if i > 0: element = {} title = Decoder.extract( '<div class="topListenedBoxDiv click js-lateral-info ">', '</div>', value) element["title"] = title element["link"] = base64.standard_b64encode(Fildonet.ARTIST + title) if value.find('data-src="') != -1: element["thumbnail"] = Fildonet.MAIN_URL + Decoder.extract( 'data-src="', '" ', value) logger.info("append1: " + title + ", link: " + element["link"]) x.append(element) i += 1 return x
def getChannels(page, cookie=''): x = [] print page if str(page) == '0': page = Radionet.MAIN_URL stations_lists = Radionet.getContentFromUrl( "http://www.radio.net/stations/", "", Radionet.cookie, Radionet.REFERER) x = Radionet.parseListStations(stations_lists) else: html = Radionet.getContentFromUrl(page, "", cookie, Radionet.REFERER) element = {} element["title"] = Decoder.extract('"seoTitle":"', '",', html) element["link"] = Decoder.extract('"streamUrl":"', '",', html) element["thumbnail"] = Decoder.extract('"logo100x100":"', '",', html) element["finalLink"] = True x.append(element) return x
def getCatalogSearchMenu(params=[]): text = "" page = 1 if type(params) is list: logger.debug("text") for element in params: logger.debug("text %s" % str(element)) if "text" in element: text = element["text"] if "page" in element: page = element["page"] url = 'https://www.lexaloffle.com/bbs/lister.php?use_hurl=1&cat=7&sub=2&page=%s&sub=2&mode=carts&orderby=featured&search=%s' % ( page, text) http = urllib3.PoolManager() logger.debug("lexaoffle url is: %s" % url) r = http.request('GET', url, preload_content=False) html = r.data.decode() bruteCards = Decoder.extract('pdat=[', '];', html) cards = [] for bruteCardLine in bruteCards.split(",``],"): logger.debug("line is %s" % bruteCardLine) if '[' in bruteCardLine: element = {} bruteCardLine = bruteCardLine[:bruteCardLine.find('],')] name = Decoder.extract(', `', '`,', bruteCardLine) id = Decoder.extract("['", "'", bruteCardLine) link = 'https://www.lexaloffle.com/bbs/get_cart.php?cat=7&play_src=0&lid=%s' % id out = re.sub('[^A-Za-z0-9.\-,\ ]+', '', name) + ".p8.png" out = ROMS_PATH + "/pico8/" + out command = 'curl "%s" --create-dirs --output "%s" \n' % (link, out) element["external"] = command element["action"] = 'command-exit' element["title"] = name cards.append(element) #back element = {} element["title"] = "Back" element["action"] = 'function' element["external"] = 'getCatalogMenu' cards.append(element) return cards
def extractElements(table,html=""): x = [] splitter = 'data-playlist-id="' splitter2 = 'Flash Player">' splitter3 = "new_player_block.nss_load_playlist('" for fieldHtml in table.split('<li '): if fieldHtml.find(splitter)>-1: element = {} playlistId = Decoder.extract(splitter,'" >',fieldHtml) title = Decoder.extract(splitter2,'</a>',fieldHtml).strip() url = Decoder.extract(splitter3,"' + jQuery(",fieldHtml).replace("&","&") rel = Decoder.rExtract('" rel="','">'+title+"</a>",html) url = R977Musiccom.MAIN_URL+url+rel+"&userId=0" element["title"] = title element["link"] = base64.standard_b64encode(url) logger.info("found title: "+element["title"]+", link: "+element["link"]) if len(element["title"])>0: #TODO: now we tries to extract the content from json 'html' x.append(element) return x
def extractElementsPlayer(html): x = [] i = 0 for value in html.split("<li class='hotSongsItem' songid="): if i > 0: element = {} title = Decoder.extract("songname='", "'", value) + " - " + Decoder.extract( "songartist='", "'", value) link = Decoder.extract("songmp3='", "'", value) element["title"] = title if link.find(".mp3") == -1: element["link"] = base64.standard_b64encode( Fildonet.MAIN_URL + link) else: element["link"] = link element["thumbnail"] = Fildonet.PLAY logger.info("append2: " + title + ", link: " + element["link"]) x.append(element) i += 1 return x
def parseListStations(html): x = [] html = Decoder.extract('<div class="col-sm-7 col-md-8 station-list">', 'aside class="col-sm-5 col-md-4">', html) for elementHtml in html.split( '<a class="stationinfo-info-toggle" href="" ng-click="toggle()"></a>' ): element = {} link = Decoder.extract('<a href="//', '" class="stationinfo-link">', elementHtml) title = Decoder.extract('<strong>', '</strong>', elementHtml) img = Decoder.extract('<img src="', '"', elementHtml) element["title"] = title element["link"] = link element["thumbnail"] = img if element["link"].find("http") != 0: element["link"] = "http://" + element["link"] if element["title"].find('</div>') == -1: logger.info("found element: " + element["title"] + ", with link: " + element["link"]) x.append(element) return x
def getChannels(page,cookie=''): x = [] referer = R977Musiccom.REFERER #print page if str(page) == '0': page=R977Musiccom.MAIN_URL html = R977Musiccom.getContentFromUrl(page,"",R977Musiccom.cookie,referer) stations_lists = R977Musiccom.getContentFromUrl("http://www.977music.com/ajax/stations_list_xml.php","",R977Musiccom.cookie,referer) #print "station list: "+stations_lists else: page = base64.standard_b64decode(page) logger.info("launching petition from page: "+page) #extract id id = Decoder.extract("?id=","&",page) songId = Decoder.extract("song_id=","&",page) if cookie=='': logger.info("launching to get an virgin cookie...") R977Musiccom.getContentFromUrl(R977Musiccom.MAIN_URL,"","","") #get a right cookie cookie = R977Musiccom.cookie else: logger.info("Detected a cookie: "+cookie) #logger.info("using always the same cookie: "+cookie) logger.info("simulating get referer (swf)... cookie: "+cookie) flashPlayer = R977Musiccom.getContentFromUrl(referer,"",cookie,R977Musiccom.MAIN_URL) logger.info("obtaining songs...") songs = R977Musiccom.getContentFromUrl("http://www.977music.com/ajax/rest/playlist.php/"+id+"/","",cookie,R977Musiccom.MAIN_URL,True) #print "songs: "+str(songs) logger.info("now is the vital petition, launching... ") html = R977Musiccom.getContentFromUrl(page,'',cookie,referer) x = R977Musiccom.parseListSongs(songs,html) #print "html: "+html if html.find('<div class="list_box">')>-1: #it's a list, needs decode table = Decoder.extract('<div class="list_box">','</ul>',html) x = R977Musiccom.extractElements(table,html) #print "done!" return x
def extractElementsPlayer(html): x = [] i = 0 for value in html.split('<div class="player"'): if i>0: element = {} title = Decoder.extract('data-title="','">',value) link = Decoder.extract('data-mp3url="','" ',value) element["title"] = title element["link"] = Redmp3cc.MAIN_URL+link if value.find('<img src="')!=-1: element["thumbnail"] = Redmp3cc.MAIN_URL+Decoder.extract('<img src="','" ',value) logger.info("append: "+title+", link: "+element["link"]) x.append(element) i+=1 if len(x)>0 and html.find(' class="button">Next page')!=-1: nextLink = "search.html/"+Decoder.rExtract('/mp3-','" class="button">Next page',html) nextText = "Next page" element = {} element["link"] = nextLink element["title"] = nextText logger.info("append next search with link: "+nextLink) x.append(element) return x
def search(text): start = 'window["ytInitialData"] = ' end = ";" page = Youtube.SEARCH_URL % urllib.parse.quote_plus(text) logger.debug("search url is: %s" % page) html = Youtube.getContentFromUrl(page) logger.debug("now extracting json search results...") jsonString = Decoder.extract(start,end,html) logger.debug("done, loading json...") jsonList = json.loads(jsonString) with open('/tmp/data.json', 'w') as outfile: json.dump(jsonList, outfile) logger.debug("parsed and saved!") listed = Youtube.extractVideosSearchFromJSON(jsonList) logger.debug("json to lists done") return listed
def main(opt): with open(opt.infos_path, 'rb') as f: infos = pickle.load(f) #override and collect parameters if len(opt.input_h5) == 0: opt.input_h5 = infos['opt'].input_h5 if len(opt.input_json) == 0: opt.input_json = infos['opt'].input_json if opt.batch_size == 0: opt.batch_size = infos['opt'].batch_size if len(opt.id) == 0: opt.id = infos['opt'].id ignore = ['id', 'batch_size', 'beam_size', 'strat_from', 'language_eval'] for key, value in vars(infos['opt']).items(): if key not in ignore: if key in vars(opt): assert vars(opt)[key] == vars(infos['opt'])[key],\ key+" option not consistent" else: vars(opt).update({key: value}) vocab = infos['vocab'] device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu') encoder = Encoder() decoder = Decoder(opt) encoder = encoder.to(device) decoder = decoder.to(device) decoder.load_state_dict(torch.load(opt.model, map_location=str(device))) encoder.eval() decoder.eval() criterion = utils.LanguageModelCriterion().to(device) if len(opt.image_folder) == 0: loader = get_loader(opt, 'test') loader.ix_to_word = vocab loss, split_predictions, lang_stats = \ eval_utils.eval_split(encoder, decoder, criterion, opt, vars(opt)) print('loss: ', loss) print(lang_stats) result_json_path = os.path.join(opt.checkpoint_path, "captions_"+opt.split+"2014_"+opt.id+"_results.json") with open(result_json_path, "w") as f: json.dump(split_predictions, f)
def train(opt): loader = get_loader(opt, 'train') opt.vocab_size = loader.vocab_size opt.seq_length = loader.seq_length summry_writer = tensorboardX.SummaryWriter() infos = {} histories = {} if opt.start_from is not None: infos_path = os.path.join(opt.start_from, 'infos_' + opt.id + '.pkl') histories_path = os.path.join(opt.start_from, 'histories_' + opt.id + '.pkl') # open infos and check if models are compatible with open(infos_path, 'rb') as f: infos = pickle.load(f) saved_model_opt = infos['opt'] need_be_same = ['hidden_size'] for checkme in need_be_same: assert vars(saved_model_opt)[checkme] == vars(opt)[checkme],\ "Command line argument and saved model disagree on %s"%(checkme) if os.path.isfile(histories_path): with open(histories_path, 'rb') as f: histories = pickle.load(f) iteration = infos.get('iter', 0) current_epoch = infos.get('epoch', 0) val_result_history = histories.get('val_result_history', {}) loss_history = histories.get('loss_history', {}) lr_history = histories.get('lr_history', {}) if opt.load_best_score == 1: best_val_score = infos.get("best_val_score", None) encoder = Encoder() decoder = Decoder(opt) device = torch.device("cuda" if torch.cuda.is_available() else "cpu") encoder = encoder.to(device) decoder = decoder.to(device) criterion = utils.LanguageModelCriterion().to(device) optimizer = optim.Adam(decoder.parameters(), lr=opt.learning_rate, weight_decay=opt.weight_decay) if vars(opt).get('start_from', None) is not None: optimizer_path = os.path.join(opt.start_from, 'optimizer.pth') optimizer.load_state_dict(torch.load(optimizer_path)) total_step = len(loader) start = time.time() for epoch in range(current_epoch, opt.max_epochs): if epoch > opt.learning_rate_decay_start and \ opt.learning_rate_decay_start >= 0: frac = (epoch - opt.learning_rate_decay_start ) // opt.learning_rate_decay_every deccay_factor = opt.learning_rate_decay_rate**frac opt.current_lr = opt.learning_rate * deccay_factor utils.set_lr(optimizer, opt.current_lr) print("learing rate change form {} to {}".format( opt.learning_rate, opt.current_lr)) else: opt.current_lr = opt.learning_rate for i, data in enumerate(loader, iteration): if i > total_step - 1: iteration = 0 break transform = transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)) imgs = [] for k in range(data['imgs'].shape[0]): img = torch.tensor(data['imgs'][k], dtype=torch.float) img = transform(img) imgs.append(img) imgs = torch.stack(imgs, dim=0).to(device) labels = torch.tensor(data['labels'].astype(np.int32), dtype=torch.long).to(device) masks = torch.tensor(data['masks'], dtype=torch.float).to(device) with torch.no_grad(): features = encoder(imgs) preds = decoder(features, labels) loss = criterion(preds, labels[:, 1:], masks[:, 1:]) optimizer.zero_grad() loss.backward() utils.clip_gradient(optimizer, opt.grad_clip) optimizer.step() train_loss = loss.item() print("iter: {}/{} (epoch {}), train loss = {:.3f}, time/batch = {}"\ .format(i, total_step, epoch, train_loss, utils.get_duration(start))) log_iter = i + epoch * total_step # write training loss summary if (i % opt.losses_log_every) == 0: summry_writer.add_scalar('train_loss', train_loss, log_iter) summry_writer.add_scalar('learning_rate', opt.current_lr, log_iter) # make evaluation on validation set, and save model if (i % opt.save_checkpoint_every == 0): #eval model eval_kwargs = {'split': 'val', 'dataset': opt.input_json} eval_kwargs.update(vars(opt)) val_loss,\ predictions,\ lang_stats = eval_utils.eval_split(encoder, decoder, criterion, opt, eval_kwargs) summry_writer.add_scalar('valaidation loss', val_loss, log_iter) if lang_stats is not None: for metric, score in lang_stats.items(): summry_writer.add_scalar(metric, score, log_iter) val_result_history[i] = { "loss": val_loss, "lang_stats": lang_stats, "predictions": predictions } if opt.language_eval == 1: current_score = lang_stats['CIDEr'] else: current_score = -val_loss.item() best_flag = False if best_val_score is None or current_score > best_val_score: best_val_score = current_score best_flag = True if not os.path.exists(opt.checkpoint_path): os.makedirs(opt.checkpoint_path) checkpoint_ptah = os.path.join(opt.checkpoint_path, 'model.pth') torch.save(decoder.state_dict(), checkpoint_ptah) print("model saved to {}".format(checkpoint_ptah)) optimizer_path = os.path.join(opt.checkpoint_path, 'optimizer.pth') torch.save(optimizer.state_dict(), optimizer_path) # Dump miscalleous informations infos['iter'] = i + 1 infos['epoch'] = epoch infos['best_val_score'] = best_val_score infos['opt'] = opt infos['vocab'] = loader.ix_to_word histories['val_result_history'] = val_result_history histories['loss_history'] = loss_history histories['lr_history'] = lr_history infos_path = os.path.join(opt.checkpoint_path, 'infos_' + opt.id + '.pkl') histories_path = os.path.join(opt.checkpoint_path, 'histories_' + opt.id + '.pkl') with open(infos_path, 'wb') as f: pickle.dump(infos, f) print("infos saved into {}".format(infos_path)) with open(histories_path, 'wb') as f: pickle.dump(histories, f) print('histories saved into {}'.format(histories_path)) if best_flag: checkpoint_path = os.path.join(opt.checkpoint_path, 'model-best.pth') torch.save(decoder.state_dict(), checkpoint_path) print("model saved to {}".format(checkpoint_path)) with open( os.path.join(opt.checkpoint_path, 'infos_' + opt.id + '-best.pkl'), 'wb') as f: pickle.dump(infos, f) summry_writer.close()
class Transformer(torch.nn.Module): """Text-to-Speech Transformer module. This is a module of text-to-speech Transformer described in `Neural Speech Synthesis with Transformer Network`_, which convert the sequence of characters or phonemes into the sequence of Mel-filterbanks. .. _`Neural Speech Synthesis with Transformer Network`: https://arxiv.org/pdf/1809.08895.pdf """ @property def attention_plot_class(self): """Return plot class for attention weight plot.""" return TTSPlot def __init__(self, idim, odim, args=None): """Initialize TTS-Transformer module. Args: idim (int): Dimension of the inputs. odim (int): Dimension of the outputs. """ # initialize base classes torch.nn.Module.__init__(self) # store hyperparameters self.idim = idim self.odim = odim self.use_scaled_pos_enc = hp.use_scaled_pos_enc self.reduction_factor = hp.reduction_factor self.loss_type = "L1" self.use_guided_attn_loss = True if self.use_guided_attn_loss: if hp.num_layers_applied_guided_attn == -1: self.num_layers_applied_guided_attn = hp.elayers else: self.num_layers_applied_guided_attn = hp.num_layers_applied_guided_attn if hp.num_heads_applied_guided_attn == -1: self.num_heads_applied_guided_attn = hp.aheads else: self.num_heads_applied_guided_attn = hp.num_heads_applied_guided_attn self.modules_applied_guided_attn = hp.modules_applied_guided_attn # use idx 0 as padding idx padding_idx = 0 # get positional encoding class pos_enc_class = ScaledPositionalEncoding if self.use_scaled_pos_enc else PositionalEncoding encoder_input_layer = torch.nn.Embedding( num_embeddings=idim, embedding_dim=hp.adim, padding_idx=padding_idx ) self.encoder = Encoder( idim=idim, attention_dim=hp.adim, attention_heads=hp.aheads, linear_units=hp.eunits, input_layer=encoder_input_layer, dropout_rate=hp.transformer_enc_dropout_rate, positional_dropout_rate=hp.transformer_enc_positional_dropout_rate, attention_dropout_rate=hp.transformer_enc_attn_dropout_rate, pos_enc_class=pos_enc_class, normalize_before=hp.encoder_normalize_before, concat_after=hp.encoder_concat_after ) # define core decoder if hp.dprenet_layers != 0: # decoder prenet decoder_input_layer = torch.nn.Sequential( DecoderPrenet( idim=odim, n_layers=hp.dprenet_layers, n_units=hp.dprenet_units, dropout_rate=hp.dprenet_dropout_rate ), torch.nn.Linear(hp.dprenet_units, hp.adim) ) else: decoder_input_layer = "linear" self.decoder = Decoder( odim=-1, attention_dim=hp.adim, attention_heads=hp.aheads, linear_units=hp.dunits, dropout_rate=hp.transformer_dec_dropout_rate, positional_dropout_rate=hp.transformer_dec_positional_dropout_rate, self_attention_dropout_rate=hp.transformer_dec_attn_dropout_rate, src_attention_dropout_rate=hp.transformer_enc_dec_attn_dropout_rate, input_layer=decoder_input_layer, use_output_layer=False, pos_enc_class=pos_enc_class, normalize_before=hp.decoder_normalize_before, concat_after=hp.decoder_concat_after ) # define final projection self.feat_out = torch.nn.Linear(hp.adim, odim * hp.reduction_factor) self.prob_out = torch.nn.Linear(hp.adim, hp.reduction_factor) # define postnet self.postnet = None if hp.postnet_layers == 0 else Postnet( idim=idim, odim=odim, n_layers=hp.postnet_layers, n_chans=hp.postnet_chans, n_filts=hp.postnet_filts, use_batch_norm=hp.use_batch_norm, dropout_rate=hp.postnet_dropout_rate ) # define loss function self.criterion = TransformerLoss(use_masking=hp.use_masking, bce_pos_weight=hp.bce_pos_weight) if self.use_guided_attn_loss: self.attn_criterion = GuidedMultiHeadAttentionLoss( sigma=0.4, alpha=1.0, ) # initialize parameters self._reset_parameters(init_type=hp.transformer_init, init_enc_alpha=hp.initial_encoder_alpha, init_dec_alpha=hp.initial_decoder_alpha) def _reset_parameters(self, init_type, init_enc_alpha=1.0, init_dec_alpha=1.0): # initialize parameters initialize(self, init_type) # initialize alpha in scaled positional encoding if self.use_scaled_pos_enc: self.encoder.embed[-1].alpha.data = torch.tensor(init_enc_alpha) self.decoder.embed[-1].alpha.data = torch.tensor(init_dec_alpha) def _add_first_frame_and_remove_last_frame(self, ys): ys_in = torch.cat([ys.new_zeros((ys.shape[0], 1, ys.shape[2])), ys[:, :-1]], dim=1) return ys_in def forward(self, xs, ilens, ys, labels, olens, spembs=None, *args, **kwargs): """Calculate forward propagation. Args: xs (Tensor): Batch of padded character ids (B, Tmax). ilens (LongTensor): Batch of lengths of each input batch (B,). ys (Tensor): Batch of padded target features (B, Lmax, odim). olens (LongTensor): Batch of the lengths of each target (B,). spembs (Tensor, optional): Batch of speaker embedding vectors (B, spk_embed_dim). Returns: Tensor: Loss value. """ # remove unnecessary padded part (for multi-gpus) max_ilen = max(ilens) max_olen = max(olens) if max_ilen != xs.shape[1]: xs = xs[:, :max_ilen] if max_olen != ys.shape[1]: ys = ys[:, :max_olen] labels = labels[:, :max_olen] # forward encoder x_masks = self._source_mask(ilens) hs, _ = self.encoder(xs, x_masks) # thin out frames for reduction factor (B, Lmax, odim) -> (B, Lmax//r, odim) if self.reduction_factor > 1: ys_in = ys[:, self.reduction_factor - 1::self.reduction_factor] olens_in = olens.new([olen // self.reduction_factor for olen in olens]) else: ys_in, olens_in = ys, olens # add first zero frame and remove last frame for auto-regressive ys_in = self._add_first_frame_and_remove_last_frame(ys_in) # forward decoder y_masks = self._target_mask(olens_in) xy_masks = self._source_to_target_mask(ilens, olens_in) zs, _ = self.decoder(ys_in, y_masks, hs, xy_masks) # (B, Lmax//r, odim * r) -> (B, Lmax//r * r, odim) before_outs = self.feat_out(zs).view(zs.size(0), -1, self.odim) # (B, Lmax//r, r) -> (B, Lmax//r * r) logits = self.prob_out(zs).view(zs.size(0), -1) # postnet -> (B, Lmax//r * r, odim) if self.postnet is None: after_outs = before_outs else: after_outs = before_outs + self.postnet(before_outs.transpose(1, 2)).transpose(1, 2) # modifiy mod part of groundtruth if self.reduction_factor > 1: olens = olens.new([olen - olen % self.reduction_factor for olen in olens]) max_olen = max(olens) ys = ys[:, :max_olen] labels = labels[:, :max_olen] labels[:, -1] = 1.0 # make sure at least one frame has 1 # caluculate loss values l1_loss, l2_loss, bce_loss = self.criterion( after_outs, before_outs, logits, ys, labels, olens) if self.loss_type == "L1": loss = l1_loss + bce_loss elif self.loss_type == "L2": loss = l2_loss + bce_loss elif self.loss_type == "L1+L2": loss = l1_loss + l2_loss + bce_loss else: raise ValueError("unknown --loss-type " + self.loss_type) report_keys = [ {"l1_loss": l1_loss.item()}, {"l2_loss": l2_loss.item()}, {"bce_loss": bce_loss.item()}, {"loss": loss.item()}, ] # calculate guided attention loss if self.use_guided_attn_loss: # calculate for encoder if "encoder" in self.modules_applied_guided_attn: att_ws = [] for idx, layer_idx in enumerate(reversed(range(len(self.encoder.encoders)))): att_ws += [self.encoder.encoders[layer_idx].self_attn.attn[:, :self.num_heads_applied_guided_attn]] if idx + 1 == self.num_layers_applied_guided_attn: break att_ws = torch.cat(att_ws, dim=1) # (B, H*L, T_in, T_in) enc_attn_loss = self.attn_criterion(att_ws, ilens, ilens) loss = loss + enc_attn_loss report_keys += [{"enc_attn_loss": enc_attn_loss.item()}] # calculate for decoder if "decoder" in self.modules_applied_guided_attn: att_ws = [] for idx, layer_idx in enumerate(reversed(range(len(self.decoder.decoders)))): att_ws += [self.decoder.decoders[layer_idx].self_attn.attn[:, :self.num_heads_applied_guided_attn]] if idx + 1 == self.num_layers_applied_guided_attn: break att_ws = torch.cat(att_ws, dim=1) # (B, H*L, T_out, T_out) dec_attn_loss = self.attn_criterion(att_ws, olens_in, olens_in) loss = loss + dec_attn_loss report_keys += [{"dec_attn_loss": dec_attn_loss.item()}] # calculate for encoder-decoder if "encoder_decoder" in self.modules_applied_guided_attn: att_ws = [] for idx, layer_idx in enumerate(reversed(range(len(self.decoder.decoders)))): att_ws += [self.decoder.decoders[layer_idx].src_attn.attn[:, :self.num_heads_applied_guided_attn]] if idx + 1 == self.num_layers_applied_guided_attn: break att_ws = torch.cat(att_ws, dim=1) # (B, H*L, T_out, T_in) enc_dec_attn_loss = self.attn_criterion(att_ws, ilens, olens_in) loss = loss + enc_dec_attn_loss report_keys += [{"enc_dec_attn_loss": enc_dec_attn_loss.item()}] # report extra information if self.use_scaled_pos_enc: report_keys += [ {"encoder_alpha": self.encoder.embed[-1].alpha.data.item()}, {"decoder_alpha": self.decoder.embed[-1].alpha.data.item()}, ] #self.reporter.report(report_keys) return loss, report_keys def inference(self, x, inference_args, *args, **kwargs): """Generate the sequence of features given the sequences of characters. Args: x (Tensor): Input sequence of characters (T,). inference_args (Namespace): - threshold (float): Threshold in inference. - minlenratio (float): Minimum length ratio in inference. - maxlenratio (float): Maximum length ratio in inference. Returns: Tensor: Output sequence of features (L, odim). Tensor: Output sequence of stop probabilities (L,). Tensor: Encoder-decoder (source) attention weights (#layers, #heads, L, T). """ # get options threshold = inference_args.threshold minlenratio = inference_args.minlenratio maxlenratio = inference_args.maxlenratio # forward encoder xs = x.unsqueeze(0) hs, _ = self.encoder(xs, None) # set limits of length maxlen = int(hs.size(1) * maxlenratio / self.reduction_factor) minlen = int(hs.size(1) * minlenratio / self.reduction_factor) # initialize idx = 0 ys = hs.new_zeros(1, 1, self.odim) outs, probs = [], [] # forward decoder step-by-step while True: # update index idx += 1 # calculate output and stop prob at idx-th step y_masks = subsequent_mask(idx).unsqueeze(0).to(x.device) z = self.decoder.recognize(ys, y_masks, hs) # (B, adim) outs += [self.feat_out(z).view(self.reduction_factor, self.odim)] # [(r, odim), ...] probs += [torch.sigmoid(self.prob_out(z))[0]] # [(r), ...] # update next inputs ys = torch.cat((ys, outs[-1][-1].view(1, 1, self.odim)), dim=1) # (1, idx + 1, odim) # check whether to finish generation if int(sum(probs[-1] >= threshold)) > 0 or idx >= maxlen: # check mininum length if idx < minlen: continue outs = torch.cat(outs, dim=0).unsqueeze(0).transpose(1, 2) # (L, odim) -> (1, L, odim) -> (1, odim, L) if self.postnet is not None: outs = outs + self.postnet(outs) # (1, odim, L) outs = outs.transpose(2, 1).squeeze(0) # (L, odim) probs = torch.cat(probs, dim=0) break # get attention weights att_ws = [] for name, m in self.named_modules(): if isinstance(m, MultiHeadedAttention) and "src" in name: att_ws += [m.attn] att_ws = torch.cat(att_ws, dim=0) return outs, probs, att_ws def calculate_all_attentions(self, xs, ilens, ys, olens, skip_output=False, keep_tensor=False, *args, **kwargs): """Calculate all of the attention weights. Args: xs (Tensor): Batch of padded character ids (B, Tmax). ilens (LongTensor): Batch of lengths of each input batch (B,). ys (Tensor): Batch of padded target features (B, Lmax, odim). olens (LongTensor): Batch of the lengths of each target (B,). spembs (Tensor, optional): Batch of speaker embedding vectors (B, spk_embed_dim). skip_output (bool, optional): Whether to skip calculate the final output. keep_tensor (bool, optional): Whether to keep original tensor. Returns: dict: Dict of attention weights and outputs. """ with torch.no_grad(): # forward encoder x_masks = self._source_mask(ilens) hs, _ = self.encoder(xs, x_masks) # thin out frames for reduction factor (B, Lmax, odim) -> (B, Lmax//r, odim) if self.reduction_factor > 1: ys_in = ys[:, self.reduction_factor - 1::self.reduction_factor] olens_in = olens.new([olen // self.reduction_factor for olen in olens]) else: ys_in, olens_in = ys, olens # add first zero frame and remove last frame for auto-regressive ys_in = self._add_first_frame_and_remove_last_frame(ys_in) # forward decoder y_masks = self._target_mask(olens_in) xy_masks = self._source_to_target_mask(ilens, olens_in) zs, _ = self.decoder(ys_in, y_masks, hs, xy_masks) # calculate final outputs if not skip_output: before_outs = self.feat_out(zs).view(zs.size(0), -1, self.odim) if self.postnet is None: after_outs = before_outs else: after_outs = before_outs + self.postnet(before_outs.transpose(1, 2)).transpose(1, 2) # modifiy mod part of output lengths due to reduction factor > 1 if self.reduction_factor > 1: olens = olens.new([olen - olen % self.reduction_factor for olen in olens]) # store into dict att_ws_dict = dict() if keep_tensor: for name, m in self.named_modules(): if isinstance(m, MultiHeadedAttention): att_ws_dict[name] = m.attn if not skip_output: att_ws_dict["before_postnet_fbank"] = before_outs att_ws_dict["after_postnet_fbank"] = after_outs else: for name, m in self.named_modules(): if isinstance(m, MultiHeadedAttention): attn = m.attn.cpu().numpy() if "encoder" in name: attn = [a[:, :l, :l] for a, l in zip(attn, ilens.tolist())] elif "decoder" in name: if "src" in name: attn = [a[:, :ol, :il] for a, il, ol in zip(attn, ilens.tolist(), olens_in.tolist())] elif "self" in name: attn = [a[:, :l, :l] for a, l in zip(attn, olens_in.tolist())] else: logging.warning("unknown attention module: " + name) else: logging.warning("unknown attention module: " + name) att_ws_dict[name] = attn if not skip_output: before_outs = before_outs.cpu().numpy() after_outs = after_outs.cpu().numpy() att_ws_dict["before_postnet_fbank"] = [m[:l].T for m, l in zip(before_outs, olens.tolist())] att_ws_dict["after_postnet_fbank"] = [m[:l].T for m, l in zip(after_outs, olens.tolist())] return att_ws_dict # # def _integrate_with_spk_embed(self, hs, spembs): # """Integrate speaker embedding with hidden states. # # Args: # hs (Tensor): Batch of hidden state sequences (B, Tmax, adim). # spembs (Tensor): Batch of speaker embeddings (B, spk_embed_dim). # # Returns: # Tensor: Batch of integrated hidden state sequences (B, Tmax, adim) # # """ # if self.spk_embed_integration_type == "add": # # apply projection and then add to hidden states # spembs = self.projection(F.normalize(spembs)) # hs = hs + spembs.unsqueeze(1) # elif self.spk_embed_integration_type == "concat": # # concat hidden states with spk embeds and then apply projection # spembs = F.normalize(spembs).unsqueeze(1).expand(-1, hs.size(1), -1) # hs = self.projection(torch.cat([hs, spembs], dim=-1)) # else: # raise NotImplementedError("support only add or concat.") # # return hs def _source_mask(self, ilens): """Make masks for self-attention. Examples: >>> ilens = [5, 3] >>> self._source_mask(ilens) tensor([[[1, 1, 1, 1, 1], [1, 1, 1, 1, 1], [1, 1, 1, 1, 1], [1, 1, 1, 1, 1], [1, 1, 1, 1, 1]], [[1, 1, 1, 0, 0], [1, 1, 1, 0, 0], [1, 1, 1, 0, 0], [0, 0, 0, 0, 0], [0, 0, 0, 0, 0]]], dtype=torch.uint8) """ x_masks = make_non_pad_mask(ilens).to(next(self.parameters()).device) return x_masks.unsqueeze(-2) & x_masks.unsqueeze(-1) def _target_mask(self, olens): """Make masks for masked self-attention. Examples: >>> olens = [5, 3] >>> self._target_mask(olens) tensor([[[1, 0, 0, 0, 0], [1, 1, 0, 0, 0], [1, 1, 1, 0, 0], [1, 1, 1, 1, 0], [1, 1, 1, 1, 1]], [[1, 0, 0, 0, 0], [1, 1, 0, 0, 0], [1, 1, 1, 0, 0], [0, 0, 0, 0, 0], [0, 0, 0, 0, 0]]], dtype=torch.uint8) """ #print("O lens:",olens) y_masks = make_non_pad_mask(olens).to(next(self.parameters()).device) s_masks = subsequent_mask(y_masks.size(-1), device=y_masks.device).unsqueeze(0) # y_masks = torch.mul(y_masks,1) # print("y masks ", y_masks) # print("s masks ", s_masks) # print("y mask",y_masks.size()) # print("s mask", s_masks.size()) # print("y mask", y_masks.unsqueeze(-2).size()) # print("y mask", y_masks.unsqueeze(-1).size()) # print("S marks : {} - {}".format(y_masks.unsqueeze(-2) & s_masks & y_masks.unsqueeze(-1),type(y_masks.unsqueeze(-2) & s_masks & y_masks.unsqueeze(-1)))) return y_masks.unsqueeze(-2) & s_masks & y_masks.unsqueeze(-1) def _source_to_target_mask(self, ilens, olens): """Make masks for encoder-decoder attention. Examples: >>> ilens = [4, 2] >>> olens = [5, 3] >>> self._source_to_target_mask(ilens) tensor([[[1, 1, 1, 1], [1, 1, 1, 1], [1, 1, 1, 1], [1, 1, 1, 1], [1, 1, 1, 1]], [[1, 1, 0, 0], [1, 1, 0, 0], [1, 1, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0]]], dtype=torch.uint8) """ x_masks = make_non_pad_mask(ilens).to(next(self.parameters()).device) y_masks = make_non_pad_mask(olens).to(next(self.parameters()).device) return x_masks.unsqueeze(-2) & y_masks.unsqueeze(-1) @property def base_plot_keys(self): """Return base key names to plot during training. keys should match what `chainer.reporter` reports. If you add the key `loss`, the reporter will report `main/loss` and `validation/main/loss` values. also `loss.png` will be created as a figure visulizing `main/loss` and `validation/main/loss` values. Returns: list: List of strings which are base keys to plot during training. """ plot_keys = ["loss", "l1_loss", "l2_loss", "bce_loss"] if self.use_scaled_pos_enc: plot_keys += ["encoder_alpha", "decoder_alpha"] if self.use_guided_attn_loss: if "encoder" in self.modules_applied_guided_attn: plot_keys += ["enc_attn_loss"] if "decoder" in self.modules_applied_guided_attn: plot_keys += ["dec_attn_loss"] if "encoder-decoder" in self.modules_applied_guided_attn: plot_keys += ["enc_dec_attn_loss"] return plot_keys
def __init__(self, idim, odim, args=None): """Initialize TTS-Transformer module. Args: idim (int): Dimension of the inputs. odim (int): Dimension of the outputs. """ # initialize base classes torch.nn.Module.__init__(self) # store hyperparameters self.idim = idim self.odim = odim self.use_scaled_pos_enc = hp.use_scaled_pos_enc self.reduction_factor = hp.reduction_factor self.loss_type = "L1" self.use_guided_attn_loss = True if self.use_guided_attn_loss: if hp.num_layers_applied_guided_attn == -1: self.num_layers_applied_guided_attn = hp.elayers else: self.num_layers_applied_guided_attn = hp.num_layers_applied_guided_attn if hp.num_heads_applied_guided_attn == -1: self.num_heads_applied_guided_attn = hp.aheads else: self.num_heads_applied_guided_attn = hp.num_heads_applied_guided_attn self.modules_applied_guided_attn = hp.modules_applied_guided_attn # use idx 0 as padding idx padding_idx = 0 # get positional encoding class pos_enc_class = ScaledPositionalEncoding if self.use_scaled_pos_enc else PositionalEncoding encoder_input_layer = torch.nn.Embedding( num_embeddings=idim, embedding_dim=hp.adim, padding_idx=padding_idx ) self.encoder = Encoder( idim=idim, attention_dim=hp.adim, attention_heads=hp.aheads, linear_units=hp.eunits, input_layer=encoder_input_layer, dropout_rate=hp.transformer_enc_dropout_rate, positional_dropout_rate=hp.transformer_enc_positional_dropout_rate, attention_dropout_rate=hp.transformer_enc_attn_dropout_rate, pos_enc_class=pos_enc_class, normalize_before=hp.encoder_normalize_before, concat_after=hp.encoder_concat_after ) # define core decoder if hp.dprenet_layers != 0: # decoder prenet decoder_input_layer = torch.nn.Sequential( DecoderPrenet( idim=odim, n_layers=hp.dprenet_layers, n_units=hp.dprenet_units, dropout_rate=hp.dprenet_dropout_rate ), torch.nn.Linear(hp.dprenet_units, hp.adim) ) else: decoder_input_layer = "linear" self.decoder = Decoder( odim=-1, attention_dim=hp.adim, attention_heads=hp.aheads, linear_units=hp.dunits, dropout_rate=hp.transformer_dec_dropout_rate, positional_dropout_rate=hp.transformer_dec_positional_dropout_rate, self_attention_dropout_rate=hp.transformer_dec_attn_dropout_rate, src_attention_dropout_rate=hp.transformer_enc_dec_attn_dropout_rate, input_layer=decoder_input_layer, use_output_layer=False, pos_enc_class=pos_enc_class, normalize_before=hp.decoder_normalize_before, concat_after=hp.decoder_concat_after ) # define final projection self.feat_out = torch.nn.Linear(hp.adim, odim * hp.reduction_factor) self.prob_out = torch.nn.Linear(hp.adim, hp.reduction_factor) # define postnet self.postnet = None if hp.postnet_layers == 0 else Postnet( idim=idim, odim=odim, n_layers=hp.postnet_layers, n_chans=hp.postnet_chans, n_filts=hp.postnet_filts, use_batch_norm=hp.use_batch_norm, dropout_rate=hp.postnet_dropout_rate ) # define loss function self.criterion = TransformerLoss(use_masking=hp.use_masking, bce_pos_weight=hp.bce_pos_weight) if self.use_guided_attn_loss: self.attn_criterion = GuidedMultiHeadAttentionLoss( sigma=0.4, alpha=1.0, ) # initialize parameters self._reset_parameters(init_type=hp.transformer_init, init_enc_alpha=hp.initial_encoder_alpha, init_dec_alpha=hp.initial_decoder_alpha)
def getCatalogMenu(params=[]): page = 1 if type(params) is list: logger.debug("page") for element in params: logger.debug("page %s" % str(element)) if "page" in element: page = element["page"] #/bbs/cposts/sc/ http = urllib3.PoolManager() url = "https://www.lexaloffle.com/bbs/lister.php?use_hurl=1&cat=7&sub=2&page=%s&mode=carts&orderby=featured" % page logger.debug("lexaoffle url is: %s" % url) r = http.request('GET', url, preload_content=False) html = r.data.decode() bruteCards = Decoder.extract('pdat=[', '];', html) cards = [] if page > 1: element = {} element["title"] = "next p. %s" % (int(page) - 1) element["action"] = "function" element["external"] = 'getCatalogMenu' element["params"] = [{'page': int(page) - 1}] cards.append(element) for bruteCardLine in bruteCards.split(",``],"): logger.debug("line is %s" % bruteCardLine) if '[' in bruteCardLine: element = {} bruteCardLine = bruteCardLine[:bruteCardLine.find('],')] name = Decoder.extract(', `', '`,', bruteCardLine) id = Decoder.extract("['", "'", bruteCardLine) link = 'https://www.lexaloffle.com/bbs/get_cart.php?cat=7&play_src=0&lid=%s' % id out = re.sub('[^A-Za-z0-9.\-,\ ]+', '', name) + ".p8.png" out = ROMS_PATH + "/pico8/" + out command = 'curl "%s" --create-dirs --output "%s" \n' % (link, out) element["external"] = command element["action"] = 'command-exit' element["title"] = name cards.append(element) #next page if len(cards) > 0: element = {} element["title"] = "next p. %s" % (int(page) + 1) element["action"] = "function" element["external"] = 'getCatalogMenu' element["params"] = [{'page': int(page) + 1}] cards.append(element) element = {} element["title"] = "Search" element["action"] = 'function-text' element["external"] = 'getCatalogSearchMenu' element["params"] = [{'webpage': link, 'final': False}] #TODO append getCatalogSearchMenu search cards.append(element) #back element = {} element["title"] = "Back" if int(page) != 1: element["action"] = 'function' element["external"] = 'getCatalogMenu' else: element["action"] = 'menu' element["external"] = 'webpages' cards.append(element) return cards
def init(): params = get_params() url = "" mode = None page = "" cookie = "" try: page = urllib.unquote_plus(params["page"]) except: pass try: url = urllib.unquote_plus(params["url"]) except: pass try: mode = int(params["mode"]) except: pass try: provider = urllib.unquote_plus(params["provider"]) except: pass try: logger.info("cookie was filled with: " + params["cookie"]) cookie = urllib.unquote_plus(params["cookie"]) except: pass #print "Mode: "+str(mode) print "URL: " + str(url) print "cookie: " + str(cookie) if mode == None: #init get_main_dirs() elif mode == 1: #get channels get_dirs(url, '', page) elif mode == 2: #open multimedia file in player open(url, page) elif mode == 3: browse_channels(url, page) elif mode == 4: browse_channel(url, page, provider, cookie) #main logic elif mode == 5: open_channel(url, page) elif mode == 0: #update if xbmcgui.Dialog().yesno(addon.getLocalizedString(10011), updater.getUpdateInfo(), "", "", addon.getLocalizedString(11013), addon.getLocalizedString(11014)): updater.update() get_main_dirs() elif mode == 100: #decode provider link logger.info("decoding: " + url) link = Decoder.decodeLink(url) logger.info("decoded: " + link) open(link, page) elif mode == 101: jsonChannels = R977Musiccom.getChannels(page, cookie) url = jsonChannels[0]["link"] logger.info("found link: " + url + ", launching...") open(url, page) #same that 2, but reserved for rtmp elif mode == 102: jsonChannels = Radionet.getChannels(page, cookie) url = jsonChannels[0]["link"] logger.info("found link: " + url + ", launching...") open(url, page) #same that 2, but reserved for rtmp elif mode == 103: jsonChannels = Redmp3cc.getChannels(url, cookie) url = jsonChannels[0]["link"] logger.info("found link: " + url + ", launching...") open(url, page) #same that 2, but reserved for rtmp logger.info("remove temp: " + url) #os.remove(url) xbmcplugin.endOfDirectory(int(sys.argv[1]))