def run(self):
log.debug("checkRSS: Starting round of RSS checking")
if autosub.WANTEDQUEUELOCK:
log.debug("checkRSS: Exiting, another threat is using the queues")
return False
else:
autosub.WANTEDQUEUELOCK = True
toDelete_wantedQueue = []
langs = ["nl"]
# default is to only check for Dutch subs
# but if English should be downloaden, check them too
# It is very important that the dutch language is run first!
if autosub.FALLBACKTOENG or autosub.DOWNLOADENG:
langs.append("en")
log.debug("checkRSS: We also want to check the English RSS feed")
for lang in langs:
if lang == "en":
RSSURL = autosub.ENRSSURL
log.debug("checkRSS: Now using the English RSS feed")
else:
RSSURL = autosub.NLRSSURL
log.debug("checkRSS: Now using the Dutch RSS feed")
try:
bierdopjeapi = API(RSSURL,RSS=True)
soup = BeautifulStoneSoup(bierdopjeapi.resp.read())
log.debug("checkRss: Succussfully connected to %s" %RSSURL)
bierdopjeapi.close()
except:
log.error("checkRss: The server returned an error for request %s" % RSSURL)
autosub.WANTEDQUEUELOCK = False
continue
# Parse all the item-tags from the RSSFeed
# The information that is parsed is: title, link and show_id
# The show_id is later used to match with the wanted items
# The title is used the determine the quality / source / releasegrp
rssItemList = []
items = soup.findAll('item')
if not len(items) > 0:
log.error("checkRss: invalid RssFeed")
log.debug("checkRss: dumping Rssfeed %s" %(soup.prettify()))
else:
log.debug("checkRss: Valid RssFeed")
for x in items:
soupx = BeautifulStoneSoup(str(x))
title = soupx.find('title').string
show_id = soupx.find('show_id').string
link = soupx.find('enclosure')['url']
item = {}
item['title'] = title
item['link'] = link
item['show_id'] = show_id
log.debug("checkRSS: Parsed from RSSFEED: %s %s %s" %(title,link,show_id))
rssItemList.append(item)
normalizedRssTitleList = []
# Now we create a new rsslist, containing information like: episode, season, etc
for item in rssItemList:
title = item['title']
link = item['link']
show_id = item['show_id']
log.debug("checkRSS: Normalizing the following entry in the RSS results: %s" % title)
normalizedRssTitle = ProcessFilename(title, '')
normalizedRssTitle['rssfile'] = title
normalizedRssTitle['link'] = link
normalizedRssTitle['show_id'] = str(show_id)
if 'title' in normalizedRssTitle.keys():
if 'season' in normalizedRssTitle.keys():
if 'episode' in normalizedRssTitle.keys():
normalizedRssTitleList.append(normalizedRssTitle)
#check versus wantedItem list
for index, wantedItem in enumerate(autosub.WANTEDQUEUE):
wantedItemquality = None
wantedItemreleasegrp = None
wantedItemsource = None
wantedItemcodec = None
wantedItemtitle = wantedItem['title']
wantedItemseason = wantedItem['season']
wantedItemepisode = wantedItem['episode']
originalfile = wantedItem['originalFileLocationOnDisk']
if lang not in wantedItem['lang']:
continue
if 'quality' in wantedItem.keys(): wantedItemquality = wantedItem['quality']
if 'releasegrp' in wantedItem.keys(): wantedItemreleasegrp = wantedItem['releasegrp']
if 'source' in wantedItem.keys(): wantedItemsource = wantedItem['source']
if 'codec' in wantedItem.keys(): wantedItemcodec = wantedItem['codec']
#lets try to find a showid
showid = autosub.Helpers.getShowid(wantedItemtitle)
#no showid? skip this item
if not showid:
continue
for normalizedRssTitle in normalizedRssTitleList:
downloadLink = None
normalizedRssTitleseason = normalizedRssTitle['season']
normalizedRssTitleepisode = normalizedRssTitle['episode']
normalizedRssTitlerssfile = normalizedRssTitle['rssfile']
normalizedRssTitleshowid = int(normalizedRssTitle['show_id'])
normalizedRssTitlelink = normalizedRssTitle['link']
log.debug("checkRSS: Trying to match ID:%r S:%r E:%r (wantedlist) with ID:%r S:%r E:%r (rss)" %(showid, wantedItemseason, wantedItemepisode, normalizedRssTitleshowid, normalizedRssTitleseason, normalizedRssTitleepisode))
if showid == normalizedRssTitleshowid and wantedItemseason == normalizedRssTitleseason and wantedItemepisode == normalizedRssTitleepisode:
log.debug("checkRSS: The episode %s - Season %s Episode %s was found in the RSS list, attempting to match a proper match" % (wantedItemtitle, wantedItemseason, wantedItemepisode))
score = autosub.Helpers.scoreMatch(normalizedRssTitle, normalizedRssTitlerssfile, wantedItemquality, wantedItemreleasegrp, wantedItemsource, wantedItemcodec)
if score >= autosub.MINMATCHSCORERSS:
log.debug ("checkRss: A match got a high enough score. MinMatchscore is %s " % autosub.MINMATCHSCORERSS)
downloadLink = normalizedRssTitlelink + autosub.APIRSS
log.info ("checkRss: Got a match, matching file is: %s" %normalizedRssTitlerssfile)
log.debug("checkRss: Dumping downloadlink for debug purpose: %s" %downloadLink)
if downloadLink:
originalfile = wantedItem['originalFileLocationOnDisk']
# Dutch subs
if autosub.SUBNL != "" and lang == "nl":
srtfile = os.path.splitext(originalfile)[0] + u"." + autosub.SUBNL + u".srt"
elif lang == "nl":
srtfile = os.path.splitext(originalfile)[0] + u".srt"
# English subs
if autosub.SUBENG != "" and lang == "en":
srtfile = os.path.splitext(originalfile)[0] + u"." + autosub.SUBENG + u".srt"
elif lang == "en":
srtfile = os.path.splitext(originalfile)[0] + u".srt"
wantedItem['downloadLink'] = downloadLink
wantedItem['destinationFileLocationOnDisk'] = srtfile
log.info("checkRSS: The episode %s - Season %s Episode %s has a matching subtitle on the RSSFeed, downloading it!" % (wantedItemtitle, wantedItemseason, wantedItemepisode))
downloadItem = wantedItem.copy()
downloadItem['downlang'] = lang
downloadItem['subtitle'] = normalizedRssTitlerssfile
if not DownloadSub(downloadItem):
break
if lang == 'nl' and (autosub.FALLBACKTOENG and not autosub.DOWNLOADENG) and 'en' in wantedItem['lang']:
log.debug('checkRss: We found a dutch subtitle and fallback is true. Removing the english subtitle from the wantedlist.')
wantedItem['lang'].remove('en')
wantedItem['lang'].remove(lang)
if len(wantedItem['lang']) == 0:
toDelete_wantedQueue.append(index)
break
else:
log.debug("checkRss: Matching score is not high enough. Score is %s should be %s" %(str(score),autosub.MINMATCHSCORERSS))
i = len(toDelete_wantedQueue)-1
while i >= 0:
log.debug("checkRSS: Removed item from the wantedQueue at index %s" % toDelete_wantedQueue[i])
autosub.WANTEDQUEUE.pop(toDelete_wantedQueue[i])
i = i-1
# Resetting the toDelete queue for the next run (if need)
toDelete_wantedQueue =[]
log.debug("checkRSS: Finished round of RSS checking")
autosub.WANTEDQUEUELOCK = False
return True
def DownloadSub(downloadDict):
#Before we download, lest check if there are enough APICalls left
if not autosub.Helpers.checkAPICalls():
log.error("downloadSubs: out of api calls")
return False
log.debug("downloadSubs: Starting DownloadSub function")
if 'destinationFileLocationOnDisk' in downloadDict.keys() and 'downloadLink' in downloadDict.keys():
log.debug("downloadSubs: Download dict seems ook. Dumping it for debug: %r" %downloadDict)
destsrt = downloadDict['destinationFileLocationOnDisk']
downloadLink = downloadDict['downloadLink']
try:
bierdopjeapi = API(downloadLink)
log.debug("downloadSubs: Trying to download the following subtitle %s" %downloadLink)
except:
log.error("downloadSubs: The server returned an error for request %s" % downloadLink)
return False
destdir = os.path.split(destsrt)[0] #make sure the download dest is there
if not os.path.exists(destdir):
log.debug("checkSubs: no destination directory %s" %destdir)
return False
elif not os.path.lexists(destdir):
log.debug("checkSubs: no destination directory %s" %destdir)
return False
#Lets first download the subtitle to a tempfile and then write it to the destination
tmpfile = tempfile.TemporaryFile('w+b')
try:
if bierdopjeapi.resp:
tmpfile.write(bierdopjeapi.resp.read())
tmpfile.write('\n') #If subtitle is exclusive for bierdopje, they add some footer which doesn't have a line feed >.>
bierdopjeapi.close()
except:
log.error("downloadSubs: Error while downloading subtitle %s. Common cases: bierdopje.com not reachable or the subtitle is corrupt on bierdopje.com. " % destsrt)
return False
tmpfile.seek(0) #Return to the start of the file
try:
log.debug("downloadSubs: Trying to save the subtitle to the filesystem")
open(destsrt, 'wb').write(tmpfile.read())
tmpfile.close()
except IOError:
log.error("downloadSubs: Could not write subtitle file. Permission denied? Enough diskspace?")
tmpfile.close()
return False
log.info("downloadSubs: DOWNLOADED: %s" % destsrt)
downloadDict['timestamp'] = time.strftime('%Y-%m-%d %H:%M:%S')
lastDown().setlastDown(dict = downloadDict)
notify.notify(downloadDict['downlang'], destsrt, downloadDict["originalFileLocationOnDisk"])
if autosub.POSTPROCESSCMD:
postprocesscmdconstructed = autosub.POSTPROCESSCMD + ' "' + downloadDict["destinationFileLocationOnDisk"] + '" "' + downloadDict["originalFileLocationOnDisk"] + '"'
log.debug("downloadSubs: Postprocess: running %s" % postprocesscmdconstructed)
log.info("downloadSubs: Running PostProcess")
postprocessoutput, postprocesserr = autosub.Helpers.RunCmd(postprocesscmdconstructed)
if postprocesserr:
log.error("downloadSubs: PostProcess: %s" % postprocesserr)
log.debug("downloadSubs: PostProcess Output:% s" % postprocessoutput)
log.debug('downloadSubs: ')
return True
else:
log.error("downloadSub: No downloadLink or locationOnDisk found at downloadItem, skipping")
return False
def run(self):
log.debug("checkRSS: Starting round of RSS checking")
if autosub.WANTEDQUEUELOCK:
log.debug("checkRSS: Exiting, another threat is using the queues")
return False
else:
autosub.WANTEDQUEUELOCK = True
toDelete_wantedQueue = []
langs = ["nl"]
# default is to only check for Dutch subs
# but if English should be downloaden, check them too
# It is very important that the dutch language is run first!
if autosub.FALLBACKTOENG or autosub.DOWNLOADENG:
langs.append("en")
log.debug("checkRSS: We also want to check the English RSS feed")
for lang in langs:
if lang == "en":
RSSURL = autosub.ENRSSURL
log.debug("checkRSS: Now using the English RSS feed")
else:
RSSURL = autosub.NLRSSURL
log.debug("checkRSS: Now using the Dutch RSS feed")
try:
bierdopjeapi = API(RSSURL, RSS=True)
soup = BeautifulStoneSoup(bierdopjeapi.resp.read())
log.debug("checkRss: Succussfully connected to %s" % RSSURL)
bierdopjeapi.close()
except:
log.error(
"checkRss: The server returned an error for request %s" %
RSSURL)
autosub.WANTEDQUEUELOCK = False
continue
# Parse all the item-tags from the RSSFeed
# The information that is parsed is: title, link and show_id
# The show_id is later used to match with the wanted items
# The title is used the determine the quality / source / releasegrp
rssItemList = []
items = soup.findAll('item')
if not len(items) > 0:
log.error("checkRss: invalid RssFeed")
log.debug("checkRss: dumping Rssfeed %s" % (soup.prettify()))
else:
log.debug("checkRss: Valid RssFeed")
for x in items:
soupx = BeautifulStoneSoup(str(x))
title = soupx.find('title').string
show_id = soupx.find('show_id').string
link = soupx.find('enclosure')['url']
item = {}
item['title'] = title
item['link'] = link
item['show_id'] = show_id
log.debug("checkRSS: Parsed from RSSFEED: %s %s %s" %
(title, link, show_id))
rssItemList.append(item)
normalizedRssTitleList = []
# Now we create a new rsslist, containing information like: episode, season, etc
for item in rssItemList:
title = item['title']
link = item['link']
show_id = item['show_id']
log.debug(
"checkRSS: Normalizing the following entry in the RSS results: %s"
% title)
normalizedRssTitle = ProcessFilename(title, '')
normalizedRssTitle['rssfile'] = title
normalizedRssTitle['link'] = link
normalizedRssTitle['show_id'] = str(show_id)
if 'title' in normalizedRssTitle.keys():
if 'season' in normalizedRssTitle.keys():
if 'episode' in normalizedRssTitle.keys():
normalizedRssTitleList.append(normalizedRssTitle)
#check versus wantedItem list
for index, wantedItem in enumerate(autosub.WANTEDQUEUE):
wantedItemquality = None
wantedItemreleasegrp = None
wantedItemsource = None
wantedItemcodec = None
wantedItemtitle = wantedItem['title']
wantedItemseason = wantedItem['season']
wantedItemepisode = wantedItem['episode']
originalfile = wantedItem['originalFileLocationOnDisk']
if lang not in wantedItem['lang']:
continue
if 'quality' in wantedItem.keys():
wantedItemquality = wantedItem['quality']
if 'releasegrp' in wantedItem.keys():
wantedItemreleasegrp = wantedItem['releasegrp']
if 'source' in wantedItem.keys():
wantedItemsource = wantedItem['source']
if 'codec' in wantedItem.keys():
wantedItemcodec = wantedItem['codec']
#lets try to find a showid
showid = autosub.Helpers.getShowid(wantedItemtitle)
#no showid? skip this item
if not showid:
continue
for normalizedRssTitle in normalizedRssTitleList:
downloadLink = None
normalizedRssTitleseason = normalizedRssTitle['season']
normalizedRssTitleepisode = normalizedRssTitle['episode']
normalizedRssTitlerssfile = normalizedRssTitle['rssfile']
normalizedRssTitleshowid = int(
normalizedRssTitle['show_id'])
normalizedRssTitlelink = normalizedRssTitle['link']
log.debug(
"checkRSS: Trying to match ID:%r S:%r E:%r (wantedlist) with ID:%r S:%r E:%r (rss)"
% (showid, wantedItemseason, wantedItemepisode,
normalizedRssTitleshowid, normalizedRssTitleseason,
normalizedRssTitleepisode))
if showid == normalizedRssTitleshowid and wantedItemseason == normalizedRssTitleseason and wantedItemepisode == normalizedRssTitleepisode:
log.debug(
"checkRSS: The episode %s - Season %s Episode %s was found in the RSS list, attempting to match a proper match"
% (wantedItemtitle, wantedItemseason,
wantedItemepisode))
score = autosub.Helpers.scoreMatch(
normalizedRssTitle, normalizedRssTitlerssfile,
wantedItemquality, wantedItemreleasegrp,
wantedItemsource, wantedItemcodec)
if score >= autosub.MINMATCHSCORERSS:
log.debug(
"checkRss: A match got a high enough score. MinMatchscore is %s "
% autosub.MINMATCHSCORERSS)
downloadLink = normalizedRssTitlelink + autosub.APIRSS
log.info(
"checkRss: Got a match, matching file is: %s" %
normalizedRssTitlerssfile)
log.debug(
"checkRss: Dumping downloadlink for debug purpose: %s"
% downloadLink)
if downloadLink:
originalfile = wantedItem[
'originalFileLocationOnDisk']
# Dutch subs
if autosub.SUBNL != "" and lang == "nl":
srtfile = os.path.splitext(originalfile)[
0] + u"." + autosub.SUBNL + u".srt"
elif lang == "nl":
srtfile = os.path.splitext(
originalfile)[0] + u".srt"
# English subs
if autosub.SUBENG != "" and lang == "en":
srtfile = os.path.splitext(originalfile)[
0] + u"." + autosub.SUBENG + u".srt"
elif lang == "en":
srtfile = os.path.splitext(
originalfile)[0] + u".srt"
wantedItem['downloadLink'] = downloadLink
wantedItem[
'destinationFileLocationOnDisk'] = srtfile
log.info(
"checkRSS: The episode %s - Season %s Episode %s has a matching subtitle on the RSSFeed, downloading it!"
% (wantedItemtitle, wantedItemseason,
wantedItemepisode))
downloadItem = wantedItem.copy()
downloadItem['downlang'] = lang
downloadItem[
'subtitle'] = normalizedRssTitlerssfile
if not DownloadSub(downloadItem):
break
if lang == 'nl' and (
autosub.FALLBACKTOENG
and not autosub.DOWNLOADENG
) and 'en' in wantedItem['lang']:
log.debug(
'checkRss: We found a dutch subtitle and fallback is true. Removing the english subtitle from the wantedlist.'
)
wantedItem['lang'].remove('en')
wantedItem['lang'].remove(lang)
if len(wantedItem['lang']) == 0:
toDelete_wantedQueue.append(index)
break
else:
log.debug(
"checkRss: Matching score is not high enough. Score is %s should be %s"
% (str(score), autosub.MINMATCHSCORERSS))
i = len(toDelete_wantedQueue) - 1
while i >= 0:
log.debug(
"checkRSS: Removed item from the wantedQueue at index %s" %
toDelete_wantedQueue[i])
autosub.WANTEDQUEUE.pop(toDelete_wantedQueue[i])
i = i - 1
# Resetting the toDelete queue for the next run (if need)
toDelete_wantedQueue = []
log.debug("checkRSS: Finished round of RSS checking")
autosub.WANTEDQUEUELOCK = False
return True