def extractElementsAlbum(table):
x = []
i = 0
for value in table.split(
'<li class="topListenedBox click js-lateral-info" onclick=\'albumClick('
):
if i > 0:
element = {}
title = Decoder.extract('<div class="topListenedBoxDiv">',
'</div>',
value).replace('<br>', " - ")
link = Decoder.extract('","', '");\'>', value)
img = Decoder.extract(' data-src="', '" ', value)
element["title"] = title
if link.find(".mp3") == -1:
element["link"] = base64.standard_b64encode(
Fildonet.ALBUM + link)
else:
element["link"] = link
element["thumbnail"] = img
logger.info("append3: " + title + ", link: " +
element["link"] + ", thumbnail: " +
element["thumbnail"])
x.append(element)
i += 1
return x
def searchLists(param):
url = (
"https://www.googleapis.com/customsearch/v1element?"
"key=AIzaSyCVAXiUzRYsML1Pv6RwSG1gunmMikTzQqY"
"&rsz=filtered_cse"
"&num=20"
"&hl=en"
"&prettyPrint=false"
"&source=gcsc"
"&gss=.com"
"&sig=8bdfc79787aa2b2b1ac464140255872c"
"&cx=013305635491195529773:0ufpuq-fpt0"
)
url += "&q=" + param + "&sort=date&googlehost=www.google.com&callback=google.search.Search.apiary846"
results = Pastebin.getContentFromUrl(url)
x = []
jsonString = Decoder.extract(',"results":', "]});", results)
logger.debug(jsonString)
for jsonResult in results.split('{"GsearchResultClass"'):
element = {}
link = Decoder.extract('"url":"', '","', jsonResult)
if "pastebin.com" in link and "/raw/" not in link:
link = link[: link.rfind("/")] + "/raw/" + link[link.rfind("/") + 1 :]
title = Decoder.extract('"title":"', '","titleNoFormatting"', jsonResult)
if "http" in link:
logger.debug("appending result: " + title + ", url: " + link)
element["title"] = title
element["link"] = link
x.append(element)
return x
def getChannels(page):
x = []
start = False
if str(page) == '0':
start = True
page=Sports4u.MAIN_URL
html = Sports4u.getContentFromUrl(page,"",Sports4u.cookie,"")
#print html
if start and 'live-channels-list">' in html: #it's a list, needs decode
table = Decoder.extract('live-channels-list">','</li><br>',html)
logger.debug("using menu table: "+table)
x = Sports4u.extractElements(table)
logger.debug("channel list logic done!")
else:
iframeUrl = Decoder.extract('<iframe frameborder="0" marginheight="0" marginwidth="0" height="490" ','"></iframe>',html)
iframeUrl = Decoder.extract('src="','"',iframeUrl)
logger.debug("iframeUrl is: "+iframeUrl)
html2 = Sports4u.getContentFromUrl(url=iframeUrl,referer=page)
logger.debug("html is: "+html2)
file = Cricfreetv.seekIframeScript(html2,page,iframeUrl)
logger.debug("Finished file logic, obtained file: "+file)
element = {}
element["link"] = file
element["title"] = "Watch streaming"
x.append(element)
return x
def getChannels(page):
x = []
if str(page) == "0":
page = Live9net.MAIN_URL
html = Live9net.getContentFromUrl(page, "", Live9net.cookie, "")
# print html
if html.find("ESPN</") > -1: # it's a list, needs decode
table = Decoder.extract("ESPN</", "<div>", html)
x = Live9net.extractElements(table)
logger.debug("live9 channels logic done!")
else:
iframeUrl = Decoder.extract('src="', '"></iframe>', html)
html2 = Live9net.getContentFromUrl(iframeUrl, "", Live9net.cookie, page)
# print html2
if html2.find('src="http://sawlive.tv/') > -1 or html2.find('src="http://www3.sawlive') > -1:
if html2.find('src="http://sawlive.tv/') > -1:
scriptSrc = Decoder.extractWithRegex("http://sawlive", '"></script>', html2).replace(
'"></script>', ""
)
else:
scriptSrc = Decoder.extractWithRegex("http://www3.sawlive", '"></script>', html2).replace(
'"></script>', ""
)
finalRtmpUrl = Decoder.extractSawlive(scriptSrc, iframeUrl)
element = {}
element["link"] = finalRtmpUrl
element["title"] = "Watch channel"
element["permalink"] = True
logger.debug("finished append element!")
x.append(element)
return x
def getChannels(page):
x = []
logger.debug("using tvshowme...")
if str(page) == '0':
page=Tvshowme.MAIN_URL
html = Tvshowme.getContentFromUrl(page,"",Tvshowme.cookie,"")
table = Decoder.extract('<span class="yawp_wim_title">Latest 150 Posts</span> <ul>','</ul>',html)
x = Tvshowme.extractElements(table)
elif page=="search":
#display keyboard, it will wait for result
keyboard = XBMCUtils.getKeyboard()
keyboard.doModal()
text = ""
if (keyboard.isConfirmed()):
text = keyboard.getText()
page = "http://www.tvshow.me/?s="+urllib.quote(text)
html = Tvshowme.getContentFromUrl(url=page)
logger.debug(html)
table = Decoder.extract('<div id="content"', '<h3 class="assistive-text">', html)
logger.debug("table is: "+table)
x = Tvshowme.extractLinks2(table)
else:
html = Tvshowme.getContentFromUrl(page, "", Tvshowme.cookie, Tvshowme.MAIN_URL)
logger.debug(html)
table = Decoder.extract('<div id="content"', '</article>', html)
x = Tvshowme.extractLinks(table)
return x
def getChannels(page):
x = []
if str(page) == '0':
html = Mamahdcom.getContentFromUrl(Mamahdcom.MAIN_URL,"",Mamahdcom.cookie,"")
else:
html = Mamahdcom.getContentFromUrl(page,"",Mamahdcom.cookie,"")
#print html
if page=='0': #menu
table = Decoder.extract('<div class="standard row channels">','</div>',html)
logger.debug("table is: "+table)
x = Mamahdcom.extractElements(table)
logger.debug("mamahd channels logic done!")
else:
iframeHtml = Decoder.extract("<iframe ","</iframe>",html)
iframeUrl = Decoder.extract('src="','"',iframeHtml)
html2 = Mamahdcom.getContentFromUrl(url=iframeUrl,referer=page)
logger.debug("obtained html from iframe: "+iframeUrl+"; html: "+html2)
if 'src="http://hdcast.org' in html2:
logger.debug("found script, launching logic...")
scriptUrl = Decoder.extract('<script type="text/javascript" src="','"></script>',html2)
logger.debug("extracting script url... from: "+scriptUrl)
iframeUrl2 = Mamahdcom.extractScriptIframeUrl(html2,scriptUrl,iframeUrl)
logger.debug("script url extracted: "+iframeUrl2)
finalRtmpUrl = Mamahdcom.extractFinalRtmpUrl(iframeUrl2,iframeUrl)
logger.debug("rtmp extracted is: "+finalRtmpUrl)
element = {}
element["link"] = finalRtmpUrl
element["title"] = "Watch channel"
element["permaLink"] = True
logger.debug("finished append element!")
x.append(element)
return x
def extractFinalRtmpUrl(url,referer):
rtmpUrl = ""
html = Mamahdcom.getContentFromUrl(url,"",Mamahdcom.cookie,referer)
if 'file:\'' in html:
file = Decoder.extract("file:'",'\',',html)
rtmp = file[0:file.rfind("/") + 1]
playpath = file[file.rfind("/") + 1:]
swfUrl = ""
secureToken = "SECURET0KEN#yw%.?()@W!"
if url.find("hdcast.org") > -1:
swfUrl = "http://player.hdcast.org/jws/jwplayer.flash.swf"
rtmpUrl = rtmp + " playPath=" + playpath + " swfUrl=" + swfUrl + " pageUrl=" + url + " flashver=WIN/2019,0,0,226 live=true timeout=14 token=" + secureToken
logger.debug("built final rtmp link: " + rtmpUrl)
elif 'allowtransparency="true" src=' in html:
logger.debug("using second way...")
secondIframe = Decoder.extract('allowtransparency="true" src=', ' ', html).replace("&","&")
logger.debug("found second way url: " + secondIframe+", referer: "+url)
headers = {
"User-Agent": Downloader.USER_AGENT,
"Accept-Language": "en-US,en;q=0.8,es-ES;q=0.5,es;q=0.3",
"Upgrade-Insecure-Requests" : "1",
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Referer": url
}
html2 = Mamahdcom.getContentFromUrl(url=secondIframe,headers=headers)
logger.debug("html2 is: "+html2)
if 'file:"' in html2:
rtmpUrl = Decoder.extract('file:"', '",', html2)
logger.debug("using m3u8 for: "+rtmpUrl)
return rtmpUrl
def getChannels(page='0'):
x = []
if str(page) == '0':
page = Yomvies.CHANNELS_PAGE
logger.debug("loading json data from: "+page)
bruteJSON = Yomvies.getContentFromUrl(page,"",Yomvies.cookie,Yomvies.MAIN_URL)
logger.debug("parsing string to json...")
i = 0
for jsonChannel in bruteJSON.split('{"CodCadenaTv":'):
if i>0:
element = {}
codTv = Decoder.extract('"','"',jsonChannel)
element["title"] = Decoder.extract('"Nombre":"','"',jsonChannel)
element["thumbnail"] = Decoder.extract('"Logo":"','"',jsonChannel).replace("\\","")
m3u8Url = Decoder.extract('"PuntoReproduccion":"','"',jsonChannel).replace("{network}",Yomvies.NETWORK).replace("\\","")
logger.debug("Appending channel: "+element["title"]+", with url: "+m3u8Url+", img: "+element["thumbnail"])
headers = 'Referer='+codTv
element["link"] = m3u8Url+"|"+headers
x.append(element)
i+=1
else:
link = Yomvies.extractTargetVideo(page)
element = {}
element["title"] = page
element["link"] = link
element["finalLink"] = True
x.append(element)
return x
def extractChannel(html,referer):
element = {}
logger.debug('processing html...')
if html.find('<iframe scrolling="no" marginwidth="0" marginheight="0" frameborder="0" width="650" height="400" src="')>-1:
element = Cineestrenostv.extractIframeChannel(html,referer)
elif html.find('.php')>-1 and referer.find(".php")==-1:
logger.debug("proccessing level 1, cookie: "+Cineestrenostv.cookie)
iframeUrl = Decoder.extractWithRegex('http://','.php',html)
if iframeUrl.find('"')>-1:
iframeUrl = iframeUrl[0:iframeUrl.find('"')]
html2 = Cineestrenostv.getContentFromUrl(iframeUrl,"",Cineestrenostv.cookie,referer)
if html2.find('<iframe scrolling="no" marginwidth="0" marginheight="0" frameborder="0" width="650" height="400" src="')>-1 or '<iframe scrolling="no" marginwidth="0" marginheight="0" frameborder="0" allowfullscreen width="653" height="403" src="' in html2:
element = Cineestrenostv.extractIframeChannel(html2,iframeUrl)
elif html.find('<iframe scrolling="no" marginwidth="0" marginheight="0" frameborder="0" width="650" height="500" src="')>-1:
iframeUrl = Decoder.extract('<iframe scrolling="no" marginwidth="0" marginheight="0" frameborder="0" width="650" height="500" src="','"></iframe>',html) #same case with different width and height: TODO: change to regex!!
html2 = Cineestrenostv.getContentFromUrl(iframeUrl,"","",referer)
if html2.find('<th scope="col"><a href="/')>-1:
partialLink = Decoder.extract('<th scope="col"><a href="/','"><font color="ffffff">',html2)
completeLink = Cineestrenostv.MAIN_URL+"/"+partialLink
html3 = Cineestrenostv.getContentFromUrl(completeLink,"",Cineestrenostv.cookie,iframeUrl)
if html3.find('<iframe scrolling="no" marginwidth="0" marginheight="0" frameborder="0" width="650" height="400" src="')>-1:
element = Cineestrenostv.extractIframeChannel(html3,completeLink)
elif referer.find("php")!=-1:
referer = referer.replace("ñ","%C3%B1")
html2 = Cineestrenostv.getContentFromUrl(referer,"",Cineestrenostv.cookie,referer)
element = Cineestrenostv.extractIframeChannel(html2,referer)
return element
def extractTargetVideoJSON(page):
title = ''
link = ''
thumbnail = ''
headers = Youtube.buildHeaders()
response = Youtube.getContentFromUrl(url=str(page + "?pbj=1"), headers=headers)
logger.debug("response is: "+response)
try:
responseJ = Decoder.extract('ytplayer.config = ','};',response)+"}"
#logger.debug("json extracted is: " + responseJ)
jsonResponse = json.loads(responseJ)
logger.debug("json loaded")
bruteVideoInfo = jsonResponse["args"]
logger.debug("obtained brute video info...")
title = bruteVideoInfo["title"]
url = bruteVideoInfo["adaptive_fmts"]
url = Decoder.extract('url=',",",url)
url = urllib.unquote(url)
#url = url[:-1]
thumbnail = bruteVideoInfo["thumbnail_url"]
logger.debug("extracted final url: "+url)
except:
logger.error("error parsing video info")
pass
element = {}
element["title"] = title
element["link"] = link
element["thumbnail"] = thumbnail
element["finalLink"] = True
return element
def getWidthAndHeightParams(html):
subUrl = ""
if html.find("; v_width=")>-1:
width = Decoder.extract("; v_width=",";",html)
height = Decoder.extract("; v_height=",";",html)
subUrl = "&vw="+width+"&vh="+height
logger.debug("width-height subUrl now is: "+subUrl)
return subUrl
def extractElements(table):
x = []
for value in table.split('\n'):
if value.find("acestream://")>-1:
element = {}
element["title"] = unicode(Decoder.extract("// ",'(',value), errors='replace')
element["link"] = Decoder.extractWithRegex("acestream:",'\"',value).replace('"',"")
logger.debug("append: "+element["title"]+", link: "+element["link"])
x.append(element)
return x
def getWidthAndHeightParams(html):
subUrl = ""
if html.find("; width='")>-1:
width = Decoder.extract("; width='","'",html)
height = Decoder.extract("; height='","'",html)
subUrl = "&width="+width+"&height="+height
elif html.find("; v_height=")>-1:
width = Decoder.extract("; v_width=",";",html)
height = Decoder.extract("; v_height=",";",html)
subUrl = "&vw="+width+"&vh="+height
return subUrl
def decode_file(self, input_filename: str,
output_filename: str, encoding: str) -> NoReturn:
self.create_new_output_file(output_filename)
table = self.read_table_from_coded_file(input_filename, encoding)
bits_iterator = self.read_coded_bits_from_coded_file(input_filename,
encoding)
decoder = Decoder(table)
coded_chars_iterator = decoder.decode_lazy_from_bits(bits_iterator)
with open(output_filename, encoding=encoding, mode='a') as file:
for char in coded_chars_iterator:
file.write(char)
def extractScriptLevel3(scriptUrl,referer=''):
html4 = Cineestrenostv.getContentFromUrl(scriptUrl, "", Cineestrenostv.cookie, referer)
finalIframeUrl = Decoder.extractWithRegex('http://', '%3D"', html4)
finalIframeUrl = finalIframeUrl[0:len(finalIframeUrl) - 1]
logger.debug("proccessing level 4, cookie: " + Cineestrenostv.cookie)
finalHtml = Cineestrenostv.getContentFromUrl(finalIframeUrl, "", Cineestrenostv.cookie, referer)
# print "final level5 html: "+finalHtml
logger.debug("proccessing level 5, cookie: " + Cineestrenostv.cookie)
playerUrl = Decoder.decodeBussinessApp(finalHtml, finalIframeUrl)
return playerUrl
def extractElements(table):
x = []
for fieldHtml in table.split('<li>'):
if fieldHtml.find("<a href=")>-1:
element = {}
element["link"] = Decoder.extract('<a href="','"',fieldHtml)
element["title"] = Decoder.extract('alt="','">',fieldHtml)
element["thumbnail"] = Decoder.extract('src="','" ',fieldHtml)
logger.debug("found title: "+element["title"]+", link: "+element["link"]+", thumbnail: "+element["thumbnail"])
if len(element["title"])>0:
x.append(element)
return x
def extractScriptIframeUrl(html,scriptUrl,referer):
iframeUrl = ""
logger.debug("extracting script iframe... url: "+scriptUrl)
scriptContent = Mamahdcom.getContentFromUrl(scriptUrl,"",Mamahdcom.cookie,referer)
#print scriptContent
iframeUrl = Decoder.extract('src="',"'",scriptContent)
logger.debug("brute iframeUrl is: "+iframeUrl)
if iframeUrl.find("?u=")>-1:
if '<script type="text/javascript"> fid="' in html:
id = Decoder.extract('<script type="text/javascript"> fid="','"; ',html)
iframeUrl = iframeUrl+id+Mamahdcom.getWidthAndHeightParams(html)
return iframeUrl
def extractElements(table):
x = []
for fieldHtml in table.split('</a>'):
logger.debug("using html: "+fieldHtml)
element = {}
element["link"] = Decoder.extract('href="','"',fieldHtml)
element["title"] = Decoder.extract("<span>","</span>",fieldHtml)
element["thumbnail"] = Decoder.extract('<img src="','"',fieldHtml)
element["permaLink"] = True
logger.debug("found title: "+element["title"]+", link: "+element["link"]+", thumb: "+element["thumbnail"])
if "http" in element["link"]:
x.append(element)
return x
def extractLinks2(html):
x = []
i=0
for value in html.split('<h2 class="entry-title ">'):
if i>1:
logger.debug("partial html is: "+value)
element = {}
title = Decoder.extract(' title="','"', value)
link = Decoder.extract('href="','"', value)
element["title"] = title.replace("Permalink to ","").replace("–","-")
element["link"] = link
x.append(element)
i+=1
return x
def drawBbcCoUkNew(url):
htmlContent = Downloader.getContentFromUrl(url=url)
title = Decoder.extract('<p class="story-body__introduction">', "</p><div", htmlContent)
if 'property="articleBody"' in htmlContent:
body = Decoder.extract(
'property="articleBody"',
" </div>",
htmlContent,
)
body = body.replace('<span class="off-screen">Image copyright</span>', "")
body = body.replace('<span class="story-image-copyright">AFP</span>', "")
body = body.replace('<span class="story-image-copyright">Reuters</span>', "")
body = body.replace('<span class="off-screen">Image caption</span>', "")
body = body.replace('<span class="off-screen">Media caption</span>', "")
while '<span class="media-caption__text">' in body:
line = Decoder.extractWithRegex('<span class="media-caption__text">', "</span>", body)
body = body.replace(line, "")
elif 'class="text-wrapper"' in htmlContent:
# special content
body = Decoder.extract('class="text-wrapper"', "</p>\n", htmlContent)
dates = Decoder.extractWithRegex('<div class="date', "</div>", body)
lastUpdate = Decoder.extractWithRegex('<p class="date ', "</p>", body)
body = body.replace(dates, "")
body = body.replace(lastUpdate, "")
elif '<figcaption class="sp-media-asset' in htmlContent:
body = Decoder.extract('<figcaption class="sp-media-asset', "</p><div ", htmlContent)
if ">" in body:
body = body[body.find(">") + 1 :]
body = Decoder.removeHTML(body).replace(".", ".\n").replace(">", "")
logger.debug("body is: " + body)
drawNew(textContent=(body))
def extractIframeChannel(contentHtml,referer):
logger.debug("proccessing level 2, cookie: "+Cineestrenostv.cookie)
iframeUrl2 = "dummy url"
if contentHtml.find('<iframe scrolling="no" marginwidth="0" marginheight="0" frameborder="0" width="653" height="403" src="')>-1:
iframeUrl2 = Decoder.extract('<iframe scrolling="no" marginwidth="0" marginheight="0" frameborder="0" width="653" height="403" src="','"></iframe>',contentHtml).replace("ñ","%C3%B1") #same case with different width and height: TODO: change to regex!!
elif contentHtml.find('<iframe scrolling="no" marginwidth="0" marginheight="0" frameborder="0" width="650" height="400" src="')>-1:
iframeUrl2 = Decoder.extract('<iframe scrolling="no" marginwidth="0" marginheight="0" frameborder="0" width="650" height="400" src="','"></iframe>',contentHtml).replace("ñ","%C3%B1") #normal cases, standar width and height
elif '<iframe scrolling="no" marginwidth="0" marginheight="0" frameborder="0" allowfullscreen width="653" height="403" src="' in contentHtml:
iframeUrl2 = Decoder.extract('<iframe scrolling="no" marginwidth="0" marginheight="0" frameborder="0" allowfullscreen width="653" height="403" src="','"></iframe>', contentHtml).replace("ñ", "%C3%B1") # normal cases, standar width and height
if iframeUrl2!="dummy url":
html3 = Cineestrenostv.getContentFromUrl(iframeUrl2,"","",referer)
return Cineestrenostv.mainLogicExtractIframeChannel(html3,iframeUrl2)
else:
return Cineestrenostv.mainLogicExtractIframeChannel(contentHtml,referer)
def extractElements(table,domain):
x = []
i = 0
for value in table.split('"name"'):
if i>0:
element = {}
title = Decoder.extract('"','"',value).replace('- ','')
link = Decoder.extract('shortcut":"','"',value)
element["title"] = title
element["link"] = domain+"/channel/"+link+"/frame"
logger.debug("append: "+title+", link: "+element["link"])
x.append(element)
i+=1
return x
def extractElements(table):
x = []
i = 0
for value in table.split('<li>'):
logger.debug("loop: "+str(i))
if i>0:
element = {}
title = Decoder.extract(">",'</a></li>',value)
link = Decoder.extract("href=\"",'"',value)
element["title"] = title
element["link"] = base64.b64encode(str(Zonasportsme.MAIN_URL+link))
logger.debug("append: "+title+", link: "+element["link"])
x.append(element)
i+=1
return x
def extractElementsArtist(html):
x = []
i=0
for value in html.split('<li class="topListenedBox click js-lateral-info " >'):
if i>0:
element = {}
title = Decoder.extract('<div class="topListenedBoxDiv click js-lateral-info ">','</div>',value)
element["title"] = title
element["link"] = base64.standard_b64encode(Fildonet.ARTIST+title)
if value.find('data-src="')!=-1:
element["thumbnail"] = Fildonet.MAIN_URL+Decoder.extract('data-src="','" ',value)
logger.info("append1: "+title+", link: "+element["link"])
x.append(element)
i+=1
return x
def run(self):
logger.debug("DownloadThread.run")
self.running = True
app = App.get_running_app()
self.progressDialog.value = 0
self.progressDialogLabel.text = "decoding link: "+self.url
decoded_link = Decoder.decodeLink(self.url)
#copy to clipboard
app.copy(decoded_link)
if decoded_link!='' and decoded_link.find("http")>-1:
app.message("Info","Link has been decoded (from "+self.url+"):\n"+decoded_link+" has been decoded and copied to clipboard.\nDownload should be started/resumed.")
app.download_screen.ids.label_text_message.text=decoded_link
elif self.aborted:
app.message("Error","File "+app.target_file+" has not been downloaded, please try again and make sure remote url exists.")
separationChar = '/'
fileName = ''
if self.url.find("/")>-1:
fileName = self.url[self.url.rfind("/")+1:]
if fileName.find(".")==-1 and decoded_link.find("/"):
fileName = decoded_link[decoded_link.rfind("/")+1:]
self.downloadfile(decoded_link,self.folder+separationChar+fileName,[],False,True,self.progressDialog,self.progressDialogLabel)
self.running = False
app.download_screen.ids.loading.opacity=0
if self.aborted:
app.message("Info","Download proccess has been stopped for file: \n "+app.target_file)
def extractElementsAlbum(table):
x = []
i = 0
for value in table.split('<div class="album">'):
if i>0:
element = {}
title = Decoder.extract(' alt="','"/>',value)
link = Decoder.extract('<a href="','" ',value)
img = Decoder.extract('src="','" ',value)
element["title"] = title
element["link"] = Redmp3cc.MAIN_URL+link
element["thumbnail"] = Redmp3cc.MAIN_URL+img
logger.info("append: "+title+", link: "+element["link"])
x.append(element)
i+=1
return x
def extractIframe(html,referer):
iframeUrl = ''
if '<iframe frameborder="0" marginheight="0" allowfullscreen="true" marginwidth="0" height="555" src="' in html:
iframeUrl = Decoder.extract('<iframe frameborder="0" marginheight="0" allowfullscreen="true" marginwidth="0" height="555" src="','"',html)
elif '<iframe frameborder="0" marginheight="0" marginwidth="0" height="490" src="' in html:
iframeUrl = Decoder.extract('<iframe frameborder="0" marginheight="0" marginwidth="0" height="490" src="','"',html)
if "'" in iframeUrl:
iframeUrl = iframeUrl[0:iframeUrl.find("'")]
logger.debug("level 1, iframeUrl: "+iframeUrl+", cookie: "+Cricfreetv.cookie)
if iframeUrl!='':
html = Cricfreetv.getContentFromUrl(iframeUrl,"",Cricfreetv.cookie,referer)
file = Cricfreetv.seekIframeScript(html,referer,iframeUrl)
item = {}
item["title"] = referer
item["link"] = file
return item
def parseListSongs(songs,html):
x = []
#jsonSongs = json.load(songs)
for elementHtml in html.split("<Channel "):
element = {}
title = Decoder.extract('name="','"',elementHtml)
channel = Decoder.extract('<RTMP>','</RTMP>',elementHtml)
url = Decoder.extract("<URL>","</URL>",elementHtml)
element["title"] = title
element["link"] = channel+" playpath="+(url.replace(' ','%20'))+" pageUrl="+R977Musiccom.MAIN_URL+" swfUrl="+R977Musiccom.REFERER
element["finalLink"] = True
if element["link"].find("rtmp")==0:
logger.info("found element: "+element["title"]+", with link: "+element["link"])
x.append(element)
return x
def extractLinks(html):
x = []
i=0
for value in html.split(' href='):
if i>1:
element = {}
title = Decoder.extract('>','</a>', value).replace("–","-")
link = Decoder.extract('"','"', value)
element["title"] = title
element["link"] = link
element["finalLink"] = True
if "<img" not in title and "tvshow.me" not in link:
logger.debug("append: " + title + ", link: " + element["link"])
x.append(element)
i+=1
return x
def extractNewIframeChannel(html3,iframeUrl2):
element = {}
if html3.find("http://telefivegb.com/")>-1:
logger.debug("found telefivegb.com link, using that link to...")
newUrl = Decoder.extractWithRegex('http://telefivegb.com/','"',html3).replace('"',"")
elif html3.find("http://verlatelegratis.net")>-1:
logger.debug("found verlatelegratis.net link, using that link to...")
newUrl = Decoder.extractWithRegex('http://verlatelegratis.net','"',html3).replace('"',"")
html4 = Cineestrenostv.getContentFromUrl(newUrl,"",Cineestrenostv.cookie,iframeUrl2)
if html4.find("http://www.playerhd1.pw/")>-1:
logger.debug("found playerhd1.pw, using that link, continue...")
element = Cineestrenostv.extractScriptPlayerHd1pw(html4,newUrl)
else:
logger.debug("possible redirect to his domains: "+html4+", try again..."+newUrl)
element = Cineestrenostv.extractNewIframeChannel(html4,newUrl)
return element
def extractElements(page,tableHtml):
x = []
i = 0
for value in tableHtml.split('<a '):
if i>0:
element = {}
title = Decoder.extract(">","</a>",value)
link = Decoder.extract("href=\"","\"",value)
element["title"] = page+" ver. "+title
element["link"] = page[0:1]+"00/"+link
if "://" not in element["link"]:
element["link"] = RTVE.MAIN_URL+element["link"]
logger.debug("append: "+element["title"]+", link: "+element["link"])
x.append(element)
i+=1
return x
def getChannels(page,cookie='',referer=''):
x = []
html = ""
if str(page) == '0':
x = Redmp3cc.getMainSections()
elif str(page) == 'songs.html':
page=Redmp3cc.MAIN_URL+"/"
html = Downloader.getContentFromUrl(page,"",cookie,"")
x = Redmp3cc.extractElementsPlayer(html)
elif str(page).find('search.html')!=-1:
if str(page).find('search.html/')==-1:
keyboard = xbmc.Keyboard("")
keyboard.doModal()
text = ""
if (keyboard.isConfirmed()):
text = keyboard.getText()
x = Redmp3cc.search(text)
else:
text = Decoder.rExtract('search.html/','/',page)
page = int(page[page.rfind('/')+1:])
x = Redmp3cc.search(text,page)
elif str(page).find(".html")!=-1:
if str(page) == 'albums.html'!=-1:
page = Redmp3cc.MAIN_URL
html = Downloader.getContentFromUrl(page,"",cookie,"")
x = Redmp3cc.extractElementsAlbum(html)
else:
html = Downloader.getContentFromUrl(page,"",cookie,"")
x = Redmp3cc.extractElementsPlayer(html)
else:
logger.info("page is: "+page)
response = Redmp3cc.getContentFromUrl(page,"",cookie,Redmp3cc.MAIN_URL,True)
#logger.info("will be used a mp3 url: "+Decoder.extract('<a href="','">here',response))
host = response[response.find("://")+len("://"):]
if host.find("/")>-1:
host = host[0:host.find("/")]
cookie = Redmp3cc.cookie
referer = page
logger.info("cookie is: "+cookie+", referer is: "+referer)
headers = downloadtools.buildMusicDownloadHeaders(host,cookie,referer)
filename= Decoder.extract('filename=','&',response)
#ROOT_DIR = xbmcaddon.Addon(id='org.harddevelop.kodi.juke').getAddonInfo('path')
ROOT_DIR = xbmc.translatePath('special://temp/')
logger.info("using special root folder: "+ROOT_DIR)
downloadtools.downloadfile(response,ROOT_DIR+"/"+filename,headers,False,True)
x.append(Redmp3cc.buildDownloadedFile(xbmc.makeLegalFilename(ROOT_DIR+"/"+filename)))
return x
def getChannels(page):
start = False
#logger.debug("Current page is: "+page)
if str(page) == '0' or str(page)=='1':
if str(page)=='0':
start = True
page=Vigoal.MAIN_URL
page = urllib.unquote_plus(page)
html = Vigoal.getContentFromUrl(page,"",Vigoal.cookie,"")
x = []
if page.find(".html")==-1:
if start:
element = {}
element["link"] = '1'
element["title"] = 'Display by event'
x.append(element)
table = Decoder.extract("<center><table><tr><td>","</tr></table></center>",html)
for fieldHtml in table.split('<a href="'):
element = {}
element["link"] = urllib.quote_plus(str(fieldHtml[0:fieldHtml.find('"')]))
element["title"] = fieldHtml[fieldHtml.find('title="')+len('title="'):]
element["title"] = element["title"][0:element["title"].find('"')].replace("-"," ").replace("en directo","").replace("Live Stream","").replace("\n","").replace("\t","").replace(" ","").strip() #cleaned
while element["title"].find("<")>-1: #clean tags
tag = Decoder.extract("<",">",element["title"])
element["title"] = element["title"].replace("<"+tag+">","")
element["thumbnail"] = fieldHtml[fieldHtml.find('<img src="')+len('<img src="'):]
element["thumbnail"] = element["thumbnail"][0:element["thumbnail"].find('"')]
logger.debug("found title: "+element["title"]+", link: "+element["link"]+", thumb: "+element["thumbnail"])
if element["link"].find("http")==0:
x.append(element)
else: #display program content
table = Decoder.extract("<h2>Events Today:</h2>","</ul>",html) #instead could be used <div class="ppal"> but... fate I suppose
i = 0
for fieldHtml in table.split('<li class="">'):
if i>0:
element = {}
element["link"] = Decoder.extract('<a href="','">',fieldHtml)
element["title"] = Decoder.extract(' - ','</div>',fieldHtml)
if fieldHtml.find('"><h2>')>-1:
titleLine = Decoder.extract('"><h2>',"</h2>",fieldHtml)
else:
titleLine = Decoder.rExtract('html">',"</a></div>",fieldHtml)
timeLine = (element["title"].replace("</b>","").replace(" - ","")).replace("-"," ").replace("en directo","").replace("Live Stream","").replace("\n","").replace("\t","").replace(" ","").strip() #cleaned
element["title"] = timeLine+" - "+titleLine
while element["title"].find("<")>-1: #clean tags
tag = Decoder.extract("<",">",element["title"])
element["title"] = element["title"].replace("<"+tag+">","")
element["thumbnail"] = fieldHtml[fieldHtml.find('<img src="')+len('<img src="'):]
element["thumbnail"] = Vigoal.MAIN_URL+element["thumbnail"][0:element["thumbnail"].find('"')]
logger.debug("found title: "+element["title"]+", link: "+element["link"]+", thumb: "+element["thumbnail"])
element["link"] = urllib.quote_plus(str(Vigoal.MAIN_URL+element["link"]))
x.append(element)
i+=1
else:
x.append(Vigoal.extractChannel(html,page))
return x
def getChannels(page):
x = []
if str(page) == '0':
page=Acetvru.MAIN_URL
html = Acetvru.getContentFromUrl(page,"",Acetvru.cookie,"")
html = Decoder.extract('var playlist = [',']',html)
x = Acetvru.extractElements(html)
return x
def extractElementsArtist(html):
x = []
i = 0
for value in html.split(
'<li class="topListenedBox click js-lateral-info " >'):
if i > 0:
element = {}
title = Decoder.extract(
'<div class="topListenedBoxDiv click js-lateral-info ">',
'</div>', value)
element["title"] = title
element["link"] = base64.standard_b64encode(Fildonet.ARTIST +
title)
if value.find('data-src="') != -1:
element["thumbnail"] = Fildonet.MAIN_URL + Decoder.extract(
'data-src="', '" ', value)
logger.info("append1: " + title + ", link: " + element["link"])
x.append(element)
i += 1
return x
def getChannels(page, cookie=''):
x = []
print page
if str(page) == '0':
page = Radionet.MAIN_URL
stations_lists = Radionet.getContentFromUrl(
"http://www.radio.net/stations/", "", Radionet.cookie,
Radionet.REFERER)
x = Radionet.parseListStations(stations_lists)
else:
html = Radionet.getContentFromUrl(page, "", cookie,
Radionet.REFERER)
element = {}
element["title"] = Decoder.extract('"seoTitle":"', '",', html)
element["link"] = Decoder.extract('"streamUrl":"', '",', html)
element["thumbnail"] = Decoder.extract('"logo100x100":"', '",',
html)
element["finalLink"] = True
x.append(element)
return x
def getCatalogSearchMenu(params=[]):
text = ""
page = 1
if type(params) is list:
logger.debug("text")
for element in params:
logger.debug("text %s" % str(element))
if "text" in element:
text = element["text"]
if "page" in element:
page = element["page"]
url = 'https://www.lexaloffle.com/bbs/lister.php?use_hurl=1&cat=7&sub=2&page=%s&sub=2&mode=carts&orderby=featured&search=%s' % (
page, text)
http = urllib3.PoolManager()
logger.debug("lexaoffle url is: %s" % url)
r = http.request('GET', url, preload_content=False)
html = r.data.decode()
bruteCards = Decoder.extract('pdat=[', '];', html)
cards = []
for bruteCardLine in bruteCards.split(",``],"):
logger.debug("line is %s" % bruteCardLine)
if '[' in bruteCardLine:
element = {}
bruteCardLine = bruteCardLine[:bruteCardLine.find('],')]
name = Decoder.extract(', `', '`,', bruteCardLine)
id = Decoder.extract("['", "'", bruteCardLine)
link = 'https://www.lexaloffle.com/bbs/get_cart.php?cat=7&play_src=0&lid=%s' % id
out = re.sub('[^A-Za-z0-9.\-,\ ]+', '', name) + ".p8.png"
out = ROMS_PATH + "/pico8/" + out
command = 'curl "%s" --create-dirs --output "%s" \n' % (link, out)
element["external"] = command
element["action"] = 'command-exit'
element["title"] = name
cards.append(element)
#back
element = {}
element["title"] = "Back"
element["action"] = 'function'
element["external"] = 'getCatalogMenu'
cards.append(element)
return cards
def extractElements(table,html=""):
x = []
splitter = 'data-playlist-id="'
splitter2 = 'Flash Player">'
splitter3 = "new_player_block.nss_load_playlist('"
for fieldHtml in table.split('<li '):
if fieldHtml.find(splitter)>-1:
element = {}
playlistId = Decoder.extract(splitter,'" >',fieldHtml)
title = Decoder.extract(splitter2,'</a>',fieldHtml).strip()
url = Decoder.extract(splitter3,"' + jQuery(",fieldHtml).replace("&","&")
rel = Decoder.rExtract('" rel="','">'+title+"</a>",html)
url = R977Musiccom.MAIN_URL+url+rel+"&userId=0"
element["title"] = title
element["link"] = base64.standard_b64encode(url)
logger.info("found title: "+element["title"]+", link: "+element["link"])
if len(element["title"])>0:
#TODO: now we tries to extract the content from json 'html'
x.append(element)
return x
def extractElementsPlayer(html):
x = []
i = 0
for value in html.split("<li class='hotSongsItem' songid="):
if i > 0:
element = {}
title = Decoder.extract("songname='", "'",
value) + " - " + Decoder.extract(
"songartist='", "'", value)
link = Decoder.extract("songmp3='", "'", value)
element["title"] = title
if link.find(".mp3") == -1:
element["link"] = base64.standard_b64encode(
Fildonet.MAIN_URL + link)
else:
element["link"] = link
element["thumbnail"] = Fildonet.PLAY
logger.info("append2: " + title + ", link: " + element["link"])
x.append(element)
i += 1
return x
def parseListStations(html):
x = []
html = Decoder.extract('<div class="col-sm-7 col-md-8 station-list">',
'aside class="col-sm-5 col-md-4">', html)
for elementHtml in html.split(
'<a class="stationinfo-info-toggle" href="" ng-click="toggle()"></a>'
):
element = {}
link = Decoder.extract('<a href="//',
'" class="stationinfo-link">', elementHtml)
title = Decoder.extract('<strong>', '</strong>', elementHtml)
img = Decoder.extract('<img src="', '"', elementHtml)
element["title"] = title
element["link"] = link
element["thumbnail"] = img
if element["link"].find("http") != 0:
element["link"] = "http://" + element["link"]
if element["title"].find('</div>') == -1:
logger.info("found element: " + element["title"] +
", with link: " + element["link"])
x.append(element)
return x
def getChannels(page,cookie=''):
x = []
referer = R977Musiccom.REFERER
#print page
if str(page) == '0':
page=R977Musiccom.MAIN_URL
html = R977Musiccom.getContentFromUrl(page,"",R977Musiccom.cookie,referer)
stations_lists = R977Musiccom.getContentFromUrl("http://www.977music.com/ajax/stations_list_xml.php","",R977Musiccom.cookie,referer)
#print "station list: "+stations_lists
else:
page = base64.standard_b64decode(page)
logger.info("launching petition from page: "+page)
#extract id
id = Decoder.extract("?id=","&",page)
songId = Decoder.extract("song_id=","&",page)
if cookie=='':
logger.info("launching to get an virgin cookie...")
R977Musiccom.getContentFromUrl(R977Musiccom.MAIN_URL,"","","") #get a right cookie
cookie = R977Musiccom.cookie
else:
logger.info("Detected a cookie: "+cookie)
#logger.info("using always the same cookie: "+cookie)
logger.info("simulating get referer (swf)... cookie: "+cookie)
flashPlayer = R977Musiccom.getContentFromUrl(referer,"",cookie,R977Musiccom.MAIN_URL)
logger.info("obtaining songs...")
songs = R977Musiccom.getContentFromUrl("http://www.977music.com/ajax/rest/playlist.php/"+id+"/","",cookie,R977Musiccom.MAIN_URL,True)
#print "songs: "+str(songs)
logger.info("now is the vital petition, launching... ")
html = R977Musiccom.getContentFromUrl(page,'',cookie,referer)
x = R977Musiccom.parseListSongs(songs,html)
#print "html: "+html
if html.find('<div class="list_box">')>-1: #it's a list, needs decode
table = Decoder.extract('<div class="list_box">','</ul>',html)
x = R977Musiccom.extractElements(table,html)
#print "done!"
return x
def extractElementsPlayer(html):
x = []
i = 0
for value in html.split('<div class="player"'):
if i>0:
element = {}
title = Decoder.extract('data-title="','">',value)
link = Decoder.extract('data-mp3url="','" ',value)
element["title"] = title
element["link"] = Redmp3cc.MAIN_URL+link
if value.find('<img src="')!=-1:
element["thumbnail"] = Redmp3cc.MAIN_URL+Decoder.extract('<img src="','" ',value)
logger.info("append: "+title+", link: "+element["link"])
x.append(element)
i+=1
if len(x)>0 and html.find(' class="button">Next page')!=-1:
nextLink = "search.html/"+Decoder.rExtract('/mp3-','" class="button">Next page',html)
nextText = "Next page"
element = {}
element["link"] = nextLink
element["title"] = nextText
logger.info("append next search with link: "+nextLink)
x.append(element)
return x
def search(text):
start = 'window["ytInitialData"] = '
end = ";"
page = Youtube.SEARCH_URL % urllib.parse.quote_plus(text)
logger.debug("search url is: %s" % page)
html = Youtube.getContentFromUrl(page)
logger.debug("now extracting json search results...")
jsonString = Decoder.extract(start,end,html)
logger.debug("done, loading json...")
jsonList = json.loads(jsonString)
with open('/tmp/data.json', 'w') as outfile:
json.dump(jsonList, outfile)
logger.debug("parsed and saved!")
listed = Youtube.extractVideosSearchFromJSON(jsonList)
logger.debug("json to lists done")
return listed
def main(opt):
with open(opt.infos_path, 'rb') as f:
infos = pickle.load(f)
#override and collect parameters
if len(opt.input_h5) == 0:
opt.input_h5 = infos['opt'].input_h5
if len(opt.input_json) == 0:
opt.input_json = infos['opt'].input_json
if opt.batch_size == 0:
opt.batch_size = infos['opt'].batch_size
if len(opt.id) == 0:
opt.id = infos['opt'].id
ignore = ['id', 'batch_size', 'beam_size', 'strat_from', 'language_eval']
for key, value in vars(infos['opt']).items():
if key not in ignore:
if key in vars(opt):
assert vars(opt)[key] == vars(infos['opt'])[key],\
key+" option not consistent"
else:
vars(opt).update({key: value})
vocab = infos['vocab']
device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
encoder = Encoder()
decoder = Decoder(opt)
encoder = encoder.to(device)
decoder = decoder.to(device)
decoder.load_state_dict(torch.load(opt.model, map_location=str(device)))
encoder.eval()
decoder.eval()
criterion = utils.LanguageModelCriterion().to(device)
if len(opt.image_folder) == 0:
loader = get_loader(opt, 'test')
loader.ix_to_word = vocab
loss, split_predictions, lang_stats = \
eval_utils.eval_split(encoder, decoder, criterion, opt, vars(opt))
print('loss: ', loss)
print(lang_stats)
result_json_path = os.path.join(opt.checkpoint_path, "captions_"+opt.split+"2014_"+opt.id+"_results.json")
with open(result_json_path, "w") as f:
json.dump(split_predictions, f)
def train(opt):
loader = get_loader(opt, 'train')
opt.vocab_size = loader.vocab_size
opt.seq_length = loader.seq_length
summry_writer = tensorboardX.SummaryWriter()
infos = {}
histories = {}
if opt.start_from is not None:
infos_path = os.path.join(opt.start_from, 'infos_' + opt.id + '.pkl')
histories_path = os.path.join(opt.start_from,
'histories_' + opt.id + '.pkl')
# open infos and check if models are compatible
with open(infos_path, 'rb') as f:
infos = pickle.load(f)
saved_model_opt = infos['opt']
need_be_same = ['hidden_size']
for checkme in need_be_same:
assert vars(saved_model_opt)[checkme] == vars(opt)[checkme],\
"Command line argument and saved model disagree on %s"%(checkme)
if os.path.isfile(histories_path):
with open(histories_path, 'rb') as f:
histories = pickle.load(f)
iteration = infos.get('iter', 0)
current_epoch = infos.get('epoch', 0)
val_result_history = histories.get('val_result_history', {})
loss_history = histories.get('loss_history', {})
lr_history = histories.get('lr_history', {})
if opt.load_best_score == 1:
best_val_score = infos.get("best_val_score", None)
encoder = Encoder()
decoder = Decoder(opt)
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
encoder = encoder.to(device)
decoder = decoder.to(device)
criterion = utils.LanguageModelCriterion().to(device)
optimizer = optim.Adam(decoder.parameters(),
lr=opt.learning_rate,
weight_decay=opt.weight_decay)
if vars(opt).get('start_from', None) is not None:
optimizer_path = os.path.join(opt.start_from, 'optimizer.pth')
optimizer.load_state_dict(torch.load(optimizer_path))
total_step = len(loader)
start = time.time()
for epoch in range(current_epoch, opt.max_epochs):
if epoch > opt.learning_rate_decay_start and \
opt.learning_rate_decay_start >= 0:
frac = (epoch - opt.learning_rate_decay_start
) // opt.learning_rate_decay_every
deccay_factor = opt.learning_rate_decay_rate**frac
opt.current_lr = opt.learning_rate * deccay_factor
utils.set_lr(optimizer, opt.current_lr)
print("learing rate change form {} to {}".format(
opt.learning_rate, opt.current_lr))
else:
opt.current_lr = opt.learning_rate
for i, data in enumerate(loader, iteration):
if i > total_step - 1:
iteration = 0
break
transform = transforms.Normalize((0.485, 0.456, 0.406),
(0.229, 0.224, 0.225))
imgs = []
for k in range(data['imgs'].shape[0]):
img = torch.tensor(data['imgs'][k], dtype=torch.float)
img = transform(img)
imgs.append(img)
imgs = torch.stack(imgs, dim=0).to(device)
labels = torch.tensor(data['labels'].astype(np.int32),
dtype=torch.long).to(device)
masks = torch.tensor(data['masks'], dtype=torch.float).to(device)
with torch.no_grad():
features = encoder(imgs)
preds = decoder(features, labels)
loss = criterion(preds, labels[:, 1:], masks[:, 1:])
optimizer.zero_grad()
loss.backward()
utils.clip_gradient(optimizer, opt.grad_clip)
optimizer.step()
train_loss = loss.item()
print("iter: {}/{} (epoch {}), train loss = {:.3f}, time/batch = {}"\
.format(i, total_step, epoch, train_loss, utils.get_duration(start)))
log_iter = i + epoch * total_step
# write training loss summary
if (i % opt.losses_log_every) == 0:
summry_writer.add_scalar('train_loss', train_loss, log_iter)
summry_writer.add_scalar('learning_rate', opt.current_lr,
log_iter)
# make evaluation on validation set, and save model
if (i % opt.save_checkpoint_every == 0):
#eval model
eval_kwargs = {'split': 'val', 'dataset': opt.input_json}
eval_kwargs.update(vars(opt))
val_loss,\
predictions,\
lang_stats = eval_utils.eval_split(encoder, decoder, criterion,
opt, eval_kwargs)
summry_writer.add_scalar('valaidation loss', val_loss,
log_iter)
if lang_stats is not None:
for metric, score in lang_stats.items():
summry_writer.add_scalar(metric, score, log_iter)
val_result_history[i] = {
"loss": val_loss,
"lang_stats": lang_stats,
"predictions": predictions
}
if opt.language_eval == 1:
current_score = lang_stats['CIDEr']
else:
current_score = -val_loss.item()
best_flag = False
if best_val_score is None or current_score > best_val_score:
best_val_score = current_score
best_flag = True
if not os.path.exists(opt.checkpoint_path):
os.makedirs(opt.checkpoint_path)
checkpoint_ptah = os.path.join(opt.checkpoint_path,
'model.pth')
torch.save(decoder.state_dict(), checkpoint_ptah)
print("model saved to {}".format(checkpoint_ptah))
optimizer_path = os.path.join(opt.checkpoint_path,
'optimizer.pth')
torch.save(optimizer.state_dict(), optimizer_path)
# Dump miscalleous informations
infos['iter'] = i + 1
infos['epoch'] = epoch
infos['best_val_score'] = best_val_score
infos['opt'] = opt
infos['vocab'] = loader.ix_to_word
histories['val_result_history'] = val_result_history
histories['loss_history'] = loss_history
histories['lr_history'] = lr_history
infos_path = os.path.join(opt.checkpoint_path,
'infos_' + opt.id + '.pkl')
histories_path = os.path.join(opt.checkpoint_path,
'histories_' + opt.id + '.pkl')
with open(infos_path, 'wb') as f:
pickle.dump(infos, f)
print("infos saved into {}".format(infos_path))
with open(histories_path, 'wb') as f:
pickle.dump(histories, f)
print('histories saved into {}'.format(histories_path))
if best_flag:
checkpoint_path = os.path.join(opt.checkpoint_path,
'model-best.pth')
torch.save(decoder.state_dict(), checkpoint_path)
print("model saved to {}".format(checkpoint_path))
with open(
os.path.join(opt.checkpoint_path,
'infos_' + opt.id + '-best.pkl'),
'wb') as f:
pickle.dump(infos, f)
summry_writer.close()
class Transformer(torch.nn.Module):
"""Text-to-Speech Transformer module.
This is a module of text-to-speech Transformer described in `Neural Speech Synthesis with Transformer Network`_,
which convert the sequence of characters or phonemes into the sequence of Mel-filterbanks.
.. _`Neural Speech Synthesis with Transformer Network`:
https://arxiv.org/pdf/1809.08895.pdf
"""
@property
def attention_plot_class(self):
"""Return plot class for attention weight plot."""
return TTSPlot
def __init__(self, idim, odim, args=None):
"""Initialize TTS-Transformer module.
Args:
idim (int): Dimension of the inputs.
odim (int): Dimension of the outputs.
"""
# initialize base classes
torch.nn.Module.__init__(self)
# store hyperparameters
self.idim = idim
self.odim = odim
self.use_scaled_pos_enc = hp.use_scaled_pos_enc
self.reduction_factor = hp.reduction_factor
self.loss_type = "L1"
self.use_guided_attn_loss = True
if self.use_guided_attn_loss:
if hp.num_layers_applied_guided_attn == -1:
self.num_layers_applied_guided_attn = hp.elayers
else:
self.num_layers_applied_guided_attn = hp.num_layers_applied_guided_attn
if hp.num_heads_applied_guided_attn == -1:
self.num_heads_applied_guided_attn = hp.aheads
else:
self.num_heads_applied_guided_attn = hp.num_heads_applied_guided_attn
self.modules_applied_guided_attn = hp.modules_applied_guided_attn
# use idx 0 as padding idx
padding_idx = 0
# get positional encoding class
pos_enc_class = ScaledPositionalEncoding if self.use_scaled_pos_enc else PositionalEncoding
encoder_input_layer = torch.nn.Embedding(
num_embeddings=idim,
embedding_dim=hp.adim,
padding_idx=padding_idx
)
self.encoder = Encoder(
idim=idim,
attention_dim=hp.adim,
attention_heads=hp.aheads,
linear_units=hp.eunits,
input_layer=encoder_input_layer,
dropout_rate=hp.transformer_enc_dropout_rate,
positional_dropout_rate=hp.transformer_enc_positional_dropout_rate,
attention_dropout_rate=hp.transformer_enc_attn_dropout_rate,
pos_enc_class=pos_enc_class,
normalize_before=hp.encoder_normalize_before,
concat_after=hp.encoder_concat_after
)
# define core decoder
if hp.dprenet_layers != 0:
# decoder prenet
decoder_input_layer = torch.nn.Sequential(
DecoderPrenet(
idim=odim,
n_layers=hp.dprenet_layers,
n_units=hp.dprenet_units,
dropout_rate=hp.dprenet_dropout_rate
),
torch.nn.Linear(hp.dprenet_units, hp.adim)
)
else:
decoder_input_layer = "linear"
self.decoder = Decoder(
odim=-1,
attention_dim=hp.adim,
attention_heads=hp.aheads,
linear_units=hp.dunits,
dropout_rate=hp.transformer_dec_dropout_rate,
positional_dropout_rate=hp.transformer_dec_positional_dropout_rate,
self_attention_dropout_rate=hp.transformer_dec_attn_dropout_rate,
src_attention_dropout_rate=hp.transformer_enc_dec_attn_dropout_rate,
input_layer=decoder_input_layer,
use_output_layer=False,
pos_enc_class=pos_enc_class,
normalize_before=hp.decoder_normalize_before,
concat_after=hp.decoder_concat_after
)
# define final projection
self.feat_out = torch.nn.Linear(hp.adim, odim * hp.reduction_factor)
self.prob_out = torch.nn.Linear(hp.adim, hp.reduction_factor)
# define postnet
self.postnet = None if hp.postnet_layers == 0 else Postnet(
idim=idim,
odim=odim,
n_layers=hp.postnet_layers,
n_chans=hp.postnet_chans,
n_filts=hp.postnet_filts,
use_batch_norm=hp.use_batch_norm,
dropout_rate=hp.postnet_dropout_rate
)
# define loss function
self.criterion = TransformerLoss(use_masking=hp.use_masking,
bce_pos_weight=hp.bce_pos_weight)
if self.use_guided_attn_loss:
self.attn_criterion = GuidedMultiHeadAttentionLoss(
sigma=0.4,
alpha=1.0,
)
# initialize parameters
self._reset_parameters(init_type=hp.transformer_init,
init_enc_alpha=hp.initial_encoder_alpha,
init_dec_alpha=hp.initial_decoder_alpha)
def _reset_parameters(self, init_type, init_enc_alpha=1.0, init_dec_alpha=1.0):
# initialize parameters
initialize(self, init_type)
# initialize alpha in scaled positional encoding
if self.use_scaled_pos_enc:
self.encoder.embed[-1].alpha.data = torch.tensor(init_enc_alpha)
self.decoder.embed[-1].alpha.data = torch.tensor(init_dec_alpha)
def _add_first_frame_and_remove_last_frame(self, ys):
ys_in = torch.cat([ys.new_zeros((ys.shape[0], 1, ys.shape[2])), ys[:, :-1]], dim=1)
return ys_in
def forward(self, xs, ilens, ys, labels, olens, spembs=None, *args, **kwargs):
"""Calculate forward propagation.
Args:
xs (Tensor): Batch of padded character ids (B, Tmax).
ilens (LongTensor): Batch of lengths of each input batch (B,).
ys (Tensor): Batch of padded target features (B, Lmax, odim).
olens (LongTensor): Batch of the lengths of each target (B,).
spembs (Tensor, optional): Batch of speaker embedding vectors (B, spk_embed_dim).
Returns:
Tensor: Loss value.
"""
# remove unnecessary padded part (for multi-gpus)
max_ilen = max(ilens)
max_olen = max(olens)
if max_ilen != xs.shape[1]:
xs = xs[:, :max_ilen]
if max_olen != ys.shape[1]:
ys = ys[:, :max_olen]
labels = labels[:, :max_olen]
# forward encoder
x_masks = self._source_mask(ilens)
hs, _ = self.encoder(xs, x_masks)
# thin out frames for reduction factor (B, Lmax, odim) -> (B, Lmax//r, odim)
if self.reduction_factor > 1:
ys_in = ys[:, self.reduction_factor - 1::self.reduction_factor]
olens_in = olens.new([olen // self.reduction_factor for olen in olens])
else:
ys_in, olens_in = ys, olens
# add first zero frame and remove last frame for auto-regressive
ys_in = self._add_first_frame_and_remove_last_frame(ys_in)
# forward decoder
y_masks = self._target_mask(olens_in)
xy_masks = self._source_to_target_mask(ilens, olens_in)
zs, _ = self.decoder(ys_in, y_masks, hs, xy_masks)
# (B, Lmax//r, odim * r) -> (B, Lmax//r * r, odim)
before_outs = self.feat_out(zs).view(zs.size(0), -1, self.odim)
# (B, Lmax//r, r) -> (B, Lmax//r * r)
logits = self.prob_out(zs).view(zs.size(0), -1)
# postnet -> (B, Lmax//r * r, odim)
if self.postnet is None:
after_outs = before_outs
else:
after_outs = before_outs + self.postnet(before_outs.transpose(1, 2)).transpose(1, 2)
# modifiy mod part of groundtruth
if self.reduction_factor > 1:
olens = olens.new([olen - olen % self.reduction_factor for olen in olens])
max_olen = max(olens)
ys = ys[:, :max_olen]
labels = labels[:, :max_olen]
labels[:, -1] = 1.0 # make sure at least one frame has 1
# caluculate loss values
l1_loss, l2_loss, bce_loss = self.criterion(
after_outs, before_outs, logits, ys, labels, olens)
if self.loss_type == "L1":
loss = l1_loss + bce_loss
elif self.loss_type == "L2":
loss = l2_loss + bce_loss
elif self.loss_type == "L1+L2":
loss = l1_loss + l2_loss + bce_loss
else:
raise ValueError("unknown --loss-type " + self.loss_type)
report_keys = [
{"l1_loss": l1_loss.item()},
{"l2_loss": l2_loss.item()},
{"bce_loss": bce_loss.item()},
{"loss": loss.item()},
]
# calculate guided attention loss
if self.use_guided_attn_loss:
# calculate for encoder
if "encoder" in self.modules_applied_guided_attn:
att_ws = []
for idx, layer_idx in enumerate(reversed(range(len(self.encoder.encoders)))):
att_ws += [self.encoder.encoders[layer_idx].self_attn.attn[:, :self.num_heads_applied_guided_attn]]
if idx + 1 == self.num_layers_applied_guided_attn:
break
att_ws = torch.cat(att_ws, dim=1) # (B, H*L, T_in, T_in)
enc_attn_loss = self.attn_criterion(att_ws, ilens, ilens)
loss = loss + enc_attn_loss
report_keys += [{"enc_attn_loss": enc_attn_loss.item()}]
# calculate for decoder
if "decoder" in self.modules_applied_guided_attn:
att_ws = []
for idx, layer_idx in enumerate(reversed(range(len(self.decoder.decoders)))):
att_ws += [self.decoder.decoders[layer_idx].self_attn.attn[:, :self.num_heads_applied_guided_attn]]
if idx + 1 == self.num_layers_applied_guided_attn:
break
att_ws = torch.cat(att_ws, dim=1) # (B, H*L, T_out, T_out)
dec_attn_loss = self.attn_criterion(att_ws, olens_in, olens_in)
loss = loss + dec_attn_loss
report_keys += [{"dec_attn_loss": dec_attn_loss.item()}]
# calculate for encoder-decoder
if "encoder_decoder" in self.modules_applied_guided_attn:
att_ws = []
for idx, layer_idx in enumerate(reversed(range(len(self.decoder.decoders)))):
att_ws += [self.decoder.decoders[layer_idx].src_attn.attn[:, :self.num_heads_applied_guided_attn]]
if idx + 1 == self.num_layers_applied_guided_attn:
break
att_ws = torch.cat(att_ws, dim=1) # (B, H*L, T_out, T_in)
enc_dec_attn_loss = self.attn_criterion(att_ws, ilens, olens_in)
loss = loss + enc_dec_attn_loss
report_keys += [{"enc_dec_attn_loss": enc_dec_attn_loss.item()}]
# report extra information
if self.use_scaled_pos_enc:
report_keys += [
{"encoder_alpha": self.encoder.embed[-1].alpha.data.item()},
{"decoder_alpha": self.decoder.embed[-1].alpha.data.item()},
]
#self.reporter.report(report_keys)
return loss, report_keys
def inference(self, x, inference_args, *args, **kwargs):
"""Generate the sequence of features given the sequences of characters.
Args:
x (Tensor): Input sequence of characters (T,).
inference_args (Namespace):
- threshold (float): Threshold in inference.
- minlenratio (float): Minimum length ratio in inference.
- maxlenratio (float): Maximum length ratio in inference.
Returns:
Tensor: Output sequence of features (L, odim).
Tensor: Output sequence of stop probabilities (L,).
Tensor: Encoder-decoder (source) attention weights (#layers, #heads, L, T).
"""
# get options
threshold = inference_args.threshold
minlenratio = inference_args.minlenratio
maxlenratio = inference_args.maxlenratio
# forward encoder
xs = x.unsqueeze(0)
hs, _ = self.encoder(xs, None)
# set limits of length
maxlen = int(hs.size(1) * maxlenratio / self.reduction_factor)
minlen = int(hs.size(1) * minlenratio / self.reduction_factor)
# initialize
idx = 0
ys = hs.new_zeros(1, 1, self.odim)
outs, probs = [], []
# forward decoder step-by-step
while True:
# update index
idx += 1
# calculate output and stop prob at idx-th step
y_masks = subsequent_mask(idx).unsqueeze(0).to(x.device)
z = self.decoder.recognize(ys, y_masks, hs) # (B, adim)
outs += [self.feat_out(z).view(self.reduction_factor, self.odim)] # [(r, odim), ...]
probs += [torch.sigmoid(self.prob_out(z))[0]] # [(r), ...]
# update next inputs
ys = torch.cat((ys, outs[-1][-1].view(1, 1, self.odim)), dim=1) # (1, idx + 1, odim)
# check whether to finish generation
if int(sum(probs[-1] >= threshold)) > 0 or idx >= maxlen:
# check mininum length
if idx < minlen:
continue
outs = torch.cat(outs, dim=0).unsqueeze(0).transpose(1, 2) # (L, odim) -> (1, L, odim) -> (1, odim, L)
if self.postnet is not None:
outs = outs + self.postnet(outs) # (1, odim, L)
outs = outs.transpose(2, 1).squeeze(0) # (L, odim)
probs = torch.cat(probs, dim=0)
break
# get attention weights
att_ws = []
for name, m in self.named_modules():
if isinstance(m, MultiHeadedAttention) and "src" in name:
att_ws += [m.attn]
att_ws = torch.cat(att_ws, dim=0)
return outs, probs, att_ws
def calculate_all_attentions(self, xs, ilens, ys, olens,
skip_output=False, keep_tensor=False, *args, **kwargs):
"""Calculate all of the attention weights.
Args:
xs (Tensor): Batch of padded character ids (B, Tmax).
ilens (LongTensor): Batch of lengths of each input batch (B,).
ys (Tensor): Batch of padded target features (B, Lmax, odim).
olens (LongTensor): Batch of the lengths of each target (B,).
spembs (Tensor, optional): Batch of speaker embedding vectors (B, spk_embed_dim).
skip_output (bool, optional): Whether to skip calculate the final output.
keep_tensor (bool, optional): Whether to keep original tensor.
Returns:
dict: Dict of attention weights and outputs.
"""
with torch.no_grad():
# forward encoder
x_masks = self._source_mask(ilens)
hs, _ = self.encoder(xs, x_masks)
# thin out frames for reduction factor (B, Lmax, odim) -> (B, Lmax//r, odim)
if self.reduction_factor > 1:
ys_in = ys[:, self.reduction_factor - 1::self.reduction_factor]
olens_in = olens.new([olen // self.reduction_factor for olen in olens])
else:
ys_in, olens_in = ys, olens
# add first zero frame and remove last frame for auto-regressive
ys_in = self._add_first_frame_and_remove_last_frame(ys_in)
# forward decoder
y_masks = self._target_mask(olens_in)
xy_masks = self._source_to_target_mask(ilens, olens_in)
zs, _ = self.decoder(ys_in, y_masks, hs, xy_masks)
# calculate final outputs
if not skip_output:
before_outs = self.feat_out(zs).view(zs.size(0), -1, self.odim)
if self.postnet is None:
after_outs = before_outs
else:
after_outs = before_outs + self.postnet(before_outs.transpose(1, 2)).transpose(1, 2)
# modifiy mod part of output lengths due to reduction factor > 1
if self.reduction_factor > 1:
olens = olens.new([olen - olen % self.reduction_factor for olen in olens])
# store into dict
att_ws_dict = dict()
if keep_tensor:
for name, m in self.named_modules():
if isinstance(m, MultiHeadedAttention):
att_ws_dict[name] = m.attn
if not skip_output:
att_ws_dict["before_postnet_fbank"] = before_outs
att_ws_dict["after_postnet_fbank"] = after_outs
else:
for name, m in self.named_modules():
if isinstance(m, MultiHeadedAttention):
attn = m.attn.cpu().numpy()
if "encoder" in name:
attn = [a[:, :l, :l] for a, l in zip(attn, ilens.tolist())]
elif "decoder" in name:
if "src" in name:
attn = [a[:, :ol, :il] for a, il, ol in zip(attn, ilens.tolist(), olens_in.tolist())]
elif "self" in name:
attn = [a[:, :l, :l] for a, l in zip(attn, olens_in.tolist())]
else:
logging.warning("unknown attention module: " + name)
else:
logging.warning("unknown attention module: " + name)
att_ws_dict[name] = attn
if not skip_output:
before_outs = before_outs.cpu().numpy()
after_outs = after_outs.cpu().numpy()
att_ws_dict["before_postnet_fbank"] = [m[:l].T for m, l in zip(before_outs, olens.tolist())]
att_ws_dict["after_postnet_fbank"] = [m[:l].T for m, l in zip(after_outs, olens.tolist())]
return att_ws_dict
#
# def _integrate_with_spk_embed(self, hs, spembs):
# """Integrate speaker embedding with hidden states.
#
# Args:
# hs (Tensor): Batch of hidden state sequences (B, Tmax, adim).
# spembs (Tensor): Batch of speaker embeddings (B, spk_embed_dim).
#
# Returns:
# Tensor: Batch of integrated hidden state sequences (B, Tmax, adim)
#
# """
# if self.spk_embed_integration_type == "add":
# # apply projection and then add to hidden states
# spembs = self.projection(F.normalize(spembs))
# hs = hs + spembs.unsqueeze(1)
# elif self.spk_embed_integration_type == "concat":
# # concat hidden states with spk embeds and then apply projection
# spembs = F.normalize(spembs).unsqueeze(1).expand(-1, hs.size(1), -1)
# hs = self.projection(torch.cat([hs, spembs], dim=-1))
# else:
# raise NotImplementedError("support only add or concat.")
#
# return hs
def _source_mask(self, ilens):
"""Make masks for self-attention.
Examples:
>>> ilens = [5, 3]
>>> self._source_mask(ilens)
tensor([[[1, 1, 1, 1, 1],
[1, 1, 1, 1, 1],
[1, 1, 1, 1, 1],
[1, 1, 1, 1, 1],
[1, 1, 1, 1, 1]],
[[1, 1, 1, 0, 0],
[1, 1, 1, 0, 0],
[1, 1, 1, 0, 0],
[0, 0, 0, 0, 0],
[0, 0, 0, 0, 0]]], dtype=torch.uint8)
"""
x_masks = make_non_pad_mask(ilens).to(next(self.parameters()).device)
return x_masks.unsqueeze(-2) & x_masks.unsqueeze(-1)
def _target_mask(self, olens):
"""Make masks for masked self-attention.
Examples:
>>> olens = [5, 3]
>>> self._target_mask(olens)
tensor([[[1, 0, 0, 0, 0],
[1, 1, 0, 0, 0],
[1, 1, 1, 0, 0],
[1, 1, 1, 1, 0],
[1, 1, 1, 1, 1]],
[[1, 0, 0, 0, 0],
[1, 1, 0, 0, 0],
[1, 1, 1, 0, 0],
[0, 0, 0, 0, 0],
[0, 0, 0, 0, 0]]], dtype=torch.uint8)
"""
#print("O lens:",olens)
y_masks = make_non_pad_mask(olens).to(next(self.parameters()).device)
s_masks = subsequent_mask(y_masks.size(-1), device=y_masks.device).unsqueeze(0)
# y_masks = torch.mul(y_masks,1)
# print("y masks ", y_masks)
# print("s masks ", s_masks)
# print("y mask",y_masks.size())
# print("s mask", s_masks.size())
# print("y mask", y_masks.unsqueeze(-2).size())
# print("y mask", y_masks.unsqueeze(-1).size())
# print("S marks : {} - {}".format(y_masks.unsqueeze(-2) & s_masks & y_masks.unsqueeze(-1),type(y_masks.unsqueeze(-2) & s_masks & y_masks.unsqueeze(-1))))
return y_masks.unsqueeze(-2) & s_masks & y_masks.unsqueeze(-1)
def _source_to_target_mask(self, ilens, olens):
"""Make masks for encoder-decoder attention.
Examples:
>>> ilens = [4, 2]
>>> olens = [5, 3]
>>> self._source_to_target_mask(ilens)
tensor([[[1, 1, 1, 1],
[1, 1, 1, 1],
[1, 1, 1, 1],
[1, 1, 1, 1],
[1, 1, 1, 1]],
[[1, 1, 0, 0],
[1, 1, 0, 0],
[1, 1, 0, 0],
[0, 0, 0, 0],
[0, 0, 0, 0]]], dtype=torch.uint8)
"""
x_masks = make_non_pad_mask(ilens).to(next(self.parameters()).device)
y_masks = make_non_pad_mask(olens).to(next(self.parameters()).device)
return x_masks.unsqueeze(-2) & y_masks.unsqueeze(-1)
@property
def base_plot_keys(self):
"""Return base key names to plot during training. keys should match what `chainer.reporter` reports.
If you add the key `loss`, the reporter will report `main/loss` and `validation/main/loss` values.
also `loss.png` will be created as a figure visulizing `main/loss` and `validation/main/loss` values.
Returns:
list: List of strings which are base keys to plot during training.
"""
plot_keys = ["loss", "l1_loss", "l2_loss", "bce_loss"]
if self.use_scaled_pos_enc:
plot_keys += ["encoder_alpha", "decoder_alpha"]
if self.use_guided_attn_loss:
if "encoder" in self.modules_applied_guided_attn:
plot_keys += ["enc_attn_loss"]
if "decoder" in self.modules_applied_guided_attn:
plot_keys += ["dec_attn_loss"]
if "encoder-decoder" in self.modules_applied_guided_attn:
plot_keys += ["enc_dec_attn_loss"]
return plot_keys
def __init__(self, idim, odim, args=None):
"""Initialize TTS-Transformer module.
Args:
idim (int): Dimension of the inputs.
odim (int): Dimension of the outputs.
"""
# initialize base classes
torch.nn.Module.__init__(self)
# store hyperparameters
self.idim = idim
self.odim = odim
self.use_scaled_pos_enc = hp.use_scaled_pos_enc
self.reduction_factor = hp.reduction_factor
self.loss_type = "L1"
self.use_guided_attn_loss = True
if self.use_guided_attn_loss:
if hp.num_layers_applied_guided_attn == -1:
self.num_layers_applied_guided_attn = hp.elayers
else:
self.num_layers_applied_guided_attn = hp.num_layers_applied_guided_attn
if hp.num_heads_applied_guided_attn == -1:
self.num_heads_applied_guided_attn = hp.aheads
else:
self.num_heads_applied_guided_attn = hp.num_heads_applied_guided_attn
self.modules_applied_guided_attn = hp.modules_applied_guided_attn
# use idx 0 as padding idx
padding_idx = 0
# get positional encoding class
pos_enc_class = ScaledPositionalEncoding if self.use_scaled_pos_enc else PositionalEncoding
encoder_input_layer = torch.nn.Embedding(
num_embeddings=idim,
embedding_dim=hp.adim,
padding_idx=padding_idx
)
self.encoder = Encoder(
idim=idim,
attention_dim=hp.adim,
attention_heads=hp.aheads,
linear_units=hp.eunits,
input_layer=encoder_input_layer,
dropout_rate=hp.transformer_enc_dropout_rate,
positional_dropout_rate=hp.transformer_enc_positional_dropout_rate,
attention_dropout_rate=hp.transformer_enc_attn_dropout_rate,
pos_enc_class=pos_enc_class,
normalize_before=hp.encoder_normalize_before,
concat_after=hp.encoder_concat_after
)
# define core decoder
if hp.dprenet_layers != 0:
# decoder prenet
decoder_input_layer = torch.nn.Sequential(
DecoderPrenet(
idim=odim,
n_layers=hp.dprenet_layers,
n_units=hp.dprenet_units,
dropout_rate=hp.dprenet_dropout_rate
),
torch.nn.Linear(hp.dprenet_units, hp.adim)
)
else:
decoder_input_layer = "linear"
self.decoder = Decoder(
odim=-1,
attention_dim=hp.adim,
attention_heads=hp.aheads,
linear_units=hp.dunits,
dropout_rate=hp.transformer_dec_dropout_rate,
positional_dropout_rate=hp.transformer_dec_positional_dropout_rate,
self_attention_dropout_rate=hp.transformer_dec_attn_dropout_rate,
src_attention_dropout_rate=hp.transformer_enc_dec_attn_dropout_rate,
input_layer=decoder_input_layer,
use_output_layer=False,
pos_enc_class=pos_enc_class,
normalize_before=hp.decoder_normalize_before,
concat_after=hp.decoder_concat_after
)
# define final projection
self.feat_out = torch.nn.Linear(hp.adim, odim * hp.reduction_factor)
self.prob_out = torch.nn.Linear(hp.adim, hp.reduction_factor)
# define postnet
self.postnet = None if hp.postnet_layers == 0 else Postnet(
idim=idim,
odim=odim,
n_layers=hp.postnet_layers,
n_chans=hp.postnet_chans,
n_filts=hp.postnet_filts,
use_batch_norm=hp.use_batch_norm,
dropout_rate=hp.postnet_dropout_rate
)
# define loss function
self.criterion = TransformerLoss(use_masking=hp.use_masking,
bce_pos_weight=hp.bce_pos_weight)
if self.use_guided_attn_loss:
self.attn_criterion = GuidedMultiHeadAttentionLoss(
sigma=0.4,
alpha=1.0,
)
# initialize parameters
self._reset_parameters(init_type=hp.transformer_init,
init_enc_alpha=hp.initial_encoder_alpha,
init_dec_alpha=hp.initial_decoder_alpha)
def getCatalogMenu(params=[]):
page = 1
if type(params) is list:
logger.debug("page")
for element in params:
logger.debug("page %s" % str(element))
if "page" in element:
page = element["page"]
#/bbs/cposts/sc/
http = urllib3.PoolManager()
url = "https://www.lexaloffle.com/bbs/lister.php?use_hurl=1&cat=7&sub=2&page=%s&mode=carts&orderby=featured" % page
logger.debug("lexaoffle url is: %s" % url)
r = http.request('GET', url, preload_content=False)
html = r.data.decode()
bruteCards = Decoder.extract('pdat=[', '];', html)
cards = []
if page > 1:
element = {}
element["title"] = "next p. %s" % (int(page) - 1)
element["action"] = "function"
element["external"] = 'getCatalogMenu'
element["params"] = [{'page': int(page) - 1}]
cards.append(element)
for bruteCardLine in bruteCards.split(",``],"):
logger.debug("line is %s" % bruteCardLine)
if '[' in bruteCardLine:
element = {}
bruteCardLine = bruteCardLine[:bruteCardLine.find('],')]
name = Decoder.extract(', `', '`,', bruteCardLine)
id = Decoder.extract("['", "'", bruteCardLine)
link = 'https://www.lexaloffle.com/bbs/get_cart.php?cat=7&play_src=0&lid=%s' % id
out = re.sub('[^A-Za-z0-9.\-,\ ]+', '', name) + ".p8.png"
out = ROMS_PATH + "/pico8/" + out
command = 'curl "%s" --create-dirs --output "%s" \n' % (link, out)
element["external"] = command
element["action"] = 'command-exit'
element["title"] = name
cards.append(element)
#next page
if len(cards) > 0:
element = {}
element["title"] = "next p. %s" % (int(page) + 1)
element["action"] = "function"
element["external"] = 'getCatalogMenu'
element["params"] = [{'page': int(page) + 1}]
cards.append(element)
element = {}
element["title"] = "Search"
element["action"] = 'function-text'
element["external"] = 'getCatalogSearchMenu'
element["params"] = [{'webpage': link, 'final': False}]
#TODO append getCatalogSearchMenu search
cards.append(element)
#back
element = {}
element["title"] = "Back"
if int(page) != 1:
element["action"] = 'function'
element["external"] = 'getCatalogMenu'
else:
element["action"] = 'menu'
element["external"] = 'webpages'
cards.append(element)
return cards
def init():
params = get_params()
url = ""
mode = None
page = ""
cookie = ""
try:
page = urllib.unquote_plus(params["page"])
except:
pass
try:
url = urllib.unquote_plus(params["url"])
except:
pass
try:
mode = int(params["mode"])
except:
pass
try:
provider = urllib.unquote_plus(params["provider"])
except:
pass
try:
logger.info("cookie was filled with: " + params["cookie"])
cookie = urllib.unquote_plus(params["cookie"])
except:
pass
#print "Mode: "+str(mode)
print "URL: " + str(url)
print "cookie: " + str(cookie)
if mode == None: #init
get_main_dirs()
elif mode == 1: #get channels
get_dirs(url, '', page)
elif mode == 2: #open multimedia file in player
open(url, page)
elif mode == 3:
browse_channels(url, page)
elif mode == 4:
browse_channel(url, page, provider, cookie) #main logic
elif mode == 5:
open_channel(url, page)
elif mode == 0: #update
if xbmcgui.Dialog().yesno(addon.getLocalizedString(10011),
updater.getUpdateInfo(), "", "",
addon.getLocalizedString(11013),
addon.getLocalizedString(11014)):
updater.update()
get_main_dirs()
elif mode == 100: #decode provider link
logger.info("decoding: " + url)
link = Decoder.decodeLink(url)
logger.info("decoded: " + link)
open(link, page)
elif mode == 101:
jsonChannels = R977Musiccom.getChannels(page, cookie)
url = jsonChannels[0]["link"]
logger.info("found link: " + url + ", launching...")
open(url, page) #same that 2, but reserved for rtmp
elif mode == 102:
jsonChannels = Radionet.getChannels(page, cookie)
url = jsonChannels[0]["link"]
logger.info("found link: " + url + ", launching...")
open(url, page) #same that 2, but reserved for rtmp
elif mode == 103:
jsonChannels = Redmp3cc.getChannels(url, cookie)
url = jsonChannels[0]["link"]
logger.info("found link: " + url + ", launching...")
open(url, page) #same that 2, but reserved for rtmp
logger.info("remove temp: " + url)
#os.remove(url)
xbmcplugin.endOfDirectory(int(sys.argv[1]))
