def SubtitleSeeker(Wanted, sourceWebsites):
# Get the scored list for all SubtitleSeeker hits
ScoreListNL, ScoreListEN = [],[]
log.debug('getSubLinks: SubtitlesSeeker search started for %s on sites: %s ' % (Wanted['ImdbId'],sourceWebsites))
# Compose the search URL for the subtitle and language we want.
langs = Wanted['langs'][0] + ',' + Wanted['langs'][1] if len(Wanted['langs']) == 2 else Wanted['langs'][0]
SearchUrl = "%s&imdb=%s&season=%s&episode=%s&language=%s&return_type=json" % (autosub.API, Wanted['ImdbId'], Wanted['season'], Wanted['episode'], langs)
# Let Subtitle seeker do a search voor the wanted sub
if autosub.Helpers.checkAPICallsSubSeeker(use=True):
try:
SubseekerSession = requests.session()
Result = SubseekerSession.get(SearchUrl).json()
SubseekerSession.close()
except Exception as error:
log.error("getSubLink: The SubtitleSeeker server returned an error. Message is %s" % error)
return ScoreListNL,ScoreListEN
else:
log.error("getSubLink: out of api calls for SubtitleSeeker.com")
return ScoreListNL,ScoreListEN
# Check to see whether we have results or not
try:
if not 'total_matches' in Result['results'].keys():
return ScoreListNL,ScoreListEN
except Exception as error:
log.info('getSublink: No subtitle found on Subtitleseeker for this video : %s' % Wanted['file'])
return ScoreListNL,ScoreListEN
if int(Result['results']['total_matches']) == 0:
return ScoreListNL,ScoreListEN
# Split the result in the two languages(if needed) and score the subs
NameDict = {}
for Item in Result['results']['items']:
if Item['site'].lower() in sourceWebsites:
Item['release'] = Item['release'][:-4] if Item['release'].lower().endswith(".srt") else Item['release']
NameDict.clear()
NameDict = ProcessFilename(Item['release'],Wanted['container'])
if not NameDict:
continue
score = autosub.Helpers.scoreMatch(NameDict,Wanted)
if score == 0:
continue
log.debug('SubtitleSeeker: Score = %s of %s for %s sub of %s on %s.' % (score, autosub.MINMATCHSCORE, Item['language'], Item['release'], Item['site']))
if score >= autosub.MINMATCHSCORE:
if Item['language'] == autosub.ENGLISH:
ScoreListEN.append({'score':score, 'url':Item['url'] , 'website':Item['site'].lower(),'Lang':Item['language'], 'releaseName':Item['release'],'SubCodec':u''})
if Item['language'] == autosub.DUTCH:
ScoreListNL.append({'score':score, 'url':Item['url'] , 'website':Item['site'].lower(),'Lang':Item['language'], 'releaseName':Item['release'],'SubCodec':u''})
return ScoreListNL,ScoreListEN
def Opensubtitles(Wanted):
ScoreListNL, ScoreListEN = [],[]
# Format a dict for the opensubtitles API call
Data = {}
Data['sublanguageid'] = Wanted['langs'][0][:3] if len(Wanted['langs']) == 1 else Wanted['langs'][0][:3] + ',' + Wanted['langs'][1][:3]
Data['imdbid' ] = Wanted['ImdbId']
Data['season'] = Wanted['season']
Data['episode'] = Wanted['episode']
log.debug('getSubLinks: Opensubtitles search started for %s.' % Wanted['ImdbId'])
time.sleep(3)
if not OpenSubtitlesNoOp():
return ScoreListNL,ScoreListEN
try:
Subs = autosub.OPENSUBTITLESSERVER.SearchSubtitles(autosub.OPENSUBTITLESTOKEN, [Data])
except:
autosub.OPENSUBTITLESTOKEN = None
log.error('Opensubtitles: Error from Opensubtitles search API')
return ScoreListNL,ScoreListEN
if Subs['status'] != '200 OK':
log.debug('Opensubtitles: No subs found for %s on Opensubtitles.' % Wanted['releaseName'])
return ScoreListNL,ScoreListEN
NameDict = {}
for Sub in Subs['data']:
try:
if int(Sub['SubBad']) > 0 or not Sub['MovieReleaseName'] or not Sub['IDSubtitleFile'] or (Sub['SubHearingImpaired'] != '0' and not autosub.HI):
continue
except:
continue
NameDict.clear()
NameDict = ProcessFilename(Sub['MovieReleaseName'],Wanted['container'])
if not NameDict:
continue
# here we score the subtitle and if it's high enough we add it to the list
score = autosub.Helpers.scoreMatch(NameDict,Wanted)
if score == 0:
continue
log.debug('Opensubtitles: Score = %s of %s for %s sub of %s.' % (score, autosub.MINMATCHSCORE, Sub['LanguageName'], Sub['MovieReleaseName']))
if score >= autosub.MINMATCHSCORE:
if Sub['LanguageName'] == autosub.DUTCH:
ScoreListNL.append({'score':score, 'url':Sub['IDSubtitleFile'] , 'website':'opensubtitles.org','releaseName':Sub['MovieReleaseName'], 'SubCodec':Sub['SubEncoding'],'Lang':Sub['LanguageName']})
if Sub['LanguageName'] == autosub.ENGLISH:
ScoreListEN.append({'score':score, 'url':Sub['IDSubtitleFile'] , 'website':'opensubtitles.org','releaseName':Sub['MovieReleaseName'], 'SubCodec':Sub['SubEncoding'],'Lang':Sub['LanguageName']})
return ScoreListNL,ScoreListEN
def walkDir(path):
SkipListNL = autosub.SKIPSTRINGNL.split(",") if len(autosub.SKIPSTRINGNL) > 0 else []
SkipListEN = autosub.SKIPSTRINGEN.split(",") if len(autosub.SKIPSTRINGEN) > 0 else []
if len(autosub.SKIPFOLDERSNL) == 0:
SkipFoldersNL = []
else:
SkipFoldersNL = autosub.SKIPFOLDERSNL.split(",") if len(autosub.SKIPFOLDERSNL) > 0 else []
for idx,folder in enumerate(SkipFoldersNL):
SkipFoldersNL[idx] = os.path.normpath(os.path.join(path,folder.rstrip("\/").lstrip("\/")))
if len(autosub.SKIPFOLDERSNL) == 0:
SkipFoldersNL = []
else:
SkipFoldersNL = autosub.SKIPFOLDERSNL.split(",") if len(autosub.SKIPFOLDERSNL) > 0 else []
for idx,folder in enumerate(SkipFoldersNL):
SkipFoldersNL[idx] = os.path.normpath(path + folder)
if len(autosub.SKIPFOLDERSEN) == 0:
SkipFoldersEN = []
else:
SkipFoldersEN = autosub.SKIPFOLDERSEN.split(",") if len(autosub.SKIPFOLDERSEN) > 0 else []
for idx,folder in enumerate(SkipFoldersEN):
SkipFoldersEN[idx] = os.path.normpath(path + folder)
for dirname, dirnames, filenames in os.walk(path, True, WalkError):
SkipThisFolderNL = False
for skip in SkipFoldersNL:
if dirname.startswith(skip):
SkipThisFolderNL = True
break
SkipThisFolderEN = False
for skip in SkipFoldersEN:
if dirname.startswith(skip):
SkipThisFolderEN = True
break
log.debug("scanDisk: directory name: %s" %dirname)
if re.search('_unpack_', dirname, re.IGNORECASE):
log.debug("scanDisk: found a unpack directory, skipping.")
continue
if autosub.SKIPHIDDENDIRS and os.path.split(dirname)[1].startswith(u'.'):
continue
if re.search('_failed_', dirname, re.IGNORECASE):
log.debug("scanDisk: found a failed directory, skipping.")
continue
if re.search('@eaDir', dirname, re.IGNORECASE):
log.debug("scanDisk: found a Synology indexing directory, skipping.")
tmpdirs = dirnames[:]
for dir in tmpdirs:
dirnames.remove(dir)
continue
if re.search("@.*thumb", dirname, re.IGNORECASE):
log.debug("scanDisk: found a Qnap multimedia thumbnail folder, skipping.")
continue
langs = []
FileDict = {}
for filename in filenames:
root,ext = os.path.splitext(filename)
if ext[1:] in ('avi', 'mkv', 'wmv', 'ts', 'mp4'):
if re.search('sample', filename):
continue
if not platform.system() == 'Windows':
# Get best ascii compatible character for special characters
try:
if not isinstance(filename, unicode):
coding = detect(filename)['encoding']
filename = unicode(filename.decode(coding),errors='replace')
correctedFilename = ''.join((c for c in unicodedata.normalize('NFD', filename) if unicodedata.category(c) != 'Mn'))
if filename != correctedFilename:
os.rename(os.path.join(dirname, filename), os.path.join(dirname, correctedFilename))
log.info("scanDir: Renamed file %s" % correctedFilename)
filename = correctedFilename
except:
log.error("scanDir: Skipping directory, file %s, %s" % (dirname,filename))
continue
# What subtitle files should we expect?
langs = []
NLext = u'.' + autosub.SUBNL + u'.srt' if autosub.SUBNL else u'.srt'
ENext = u'.' + autosub.SUBENG + u'.srt' if autosub.SUBENG else u'.srt'
ENext = u'.en.srt'if NLext == ENext and autosub.DOWNLOADDUTCH else ENext
if not os.access(dirname, os.W_OK):
log.error('scandisk: No write access to folder: %s' % dirname)
continue
# Check which languages we want to download based on user settings.
if autosub.DOWNLOADDUTCH and not SkipThisFolderNL:
Skipped = False
for SkipItem in SkipListNL:
if not SkipItem: break
if re.search(SkipItem.lower(), filename.lower()):
Skipped = True
break
if Skipped:
log.info("scanDir: %s found in %s so skipped for Dutch subs" % (SkipItem, filename))
elif os.path.exists(os.path.join(dirname, root + NLext)):
Skipped = True
log.debug("scanDir: %s skipped because the Dutch subtitle already exists" % filename)
else:
# If the Dutch subtitle not skipped and doesn't exist, then add it to the wanted list
langs.append(autosub.DUTCH)
if (autosub.DOWNLOADENG or (autosub.FALLBACKTOENG and autosub.DOWNLOADDUTCH and not Skipped)) and not SkipThisFolderEN:
Skipped = False
for SkipItem in SkipListEN:
if not SkipItem: break
if re.search(SkipItem.lower(), filename.lower()):
Skipped = True
break
if Skipped:
log.info("scanDir: %s found in %s so skipped for English subs" % (SkipItem, filename))
elif os.path.exists(os.path.join(dirname, root + ENext)):
log.debug("scanDir: %s skipped because the English subtitle already exists" % filename)
else:
# If the English subtitle not skipped and doesn't exist, then add it to the wanted list
if not os.path.exists(os.path.join(dirname, root + ENext)):
langs.append(autosub.ENGLISH)
if not langs:
# nothing to do for this file
continue
FileDict = ProcessFilename(os.path.splitext(filename)[0].strip(), ext)
time.sleep(0)
if not FileDict:
continue
if not 'title' in FileDict.keys() or not 'season' in FileDict.keys() or not 'episode' in FileDict.keys():
continue
if not FileDict['releasegrp'] and not FileDict['source'] and not FileDict['quality'] and not FileDict['source']:
log.error("scanDir: Not enough info in filename: %s" % filename)
continue
FileDict['timestamp'] = unicode(time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(os.path.getctime(os.path.join(dirname, filename)))))
FileDict['langs'] = langs
FileDict['NLext'] = NLext
FileDict['ENext'] = ENext
FileDict['file'] = root
FileDict['container'] = ext
FileDict['folder'] = dirname
FileDict['ImdbId'],FileDict['A7Id'], FileDict['TvdbId'], FileDict['title'] = Helpers.getShowid(FileDict['title'],autosub.ADDIC7EDLOGGED_IN)
if autosub.Helpers.SkipShow(FileDict['ImdbId'],FileDict['title'], FileDict['season'], FileDict['episode']):
continue
log.info("scanDir: %s WANTED FOR: %s" % (langs, filename))
autosub.WANTEDQUEUE.append(FileDict)
return
def run(self):
log.debug("checkRSS: Starting round of RSS checking")
if autosub.WANTEDQUEUELOCK:
log.debug("checkRSS: Exiting, another threat is using the queues")
return False
else:
autosub.WANTEDQUEUELOCK = True
toDelete_wantedQueue = []
langs = ["nl"]
# default is to only check for Dutch subs
# but if English should be downloaden, check them too
# It is very important that the dutch language is run first!
if autosub.FALLBACKTOENG or autosub.DOWNLOADENG:
langs.append("en")
log.debug("checkRSS: We also want to check the English RSS feed")
for lang in langs:
if lang == "en":
RSSURL = autosub.ENRSSURL
log.debug("checkRSS: Now using the English RSS feed")
else:
RSSURL = autosub.NLRSSURL
log.debug("checkRSS: Now using the Dutch RSS feed")
try:
bierdopjeapi = API(RSSURL,RSS=True)
soup = BeautifulStoneSoup(bierdopjeapi.resp.read())
log.debug("checkRss: Succussfully connected to %s" %RSSURL)
bierdopjeapi.close()
except:
log.error("checkRss: The server returned an error for request %s" % RSSURL)
autosub.WANTEDQUEUELOCK = False
continue
# Parse all the item-tags from the RSSFeed
# The information that is parsed is: title, link and show_id
# The show_id is later used to match with the wanted items
# The title is used the determine the quality / source / releasegrp
rssItemList = []
items = soup.findAll('item')
if not len(items) > 0:
log.error("checkRss: invalid RssFeed")
log.debug("checkRss: dumping Rssfeed %s" %(soup.prettify()))
else:
log.debug("checkRss: Valid RssFeed")
for x in items:
soupx = BeautifulStoneSoup(str(x))
title = soupx.find('title').string
show_id = soupx.find('show_id').string
link = soupx.find('enclosure')['url']
item = {}
item['title'] = title
item['link'] = link
item['show_id'] = show_id
log.debug("checkRSS: Parsed from RSSFEED: %s %s %s" %(title,link,show_id))
rssItemList.append(item)
normalizedRssTitleList = []
# Now we create a new rsslist, containing information like: episode, season, etc
for item in rssItemList:
title = item['title']
link = item['link']
show_id = item['show_id']
log.debug("checkRSS: Normalizing the following entry in the RSS results: %s" % title)
normalizedRssTitle = ProcessFilename(title, '')
normalizedRssTitle['rssfile'] = title
normalizedRssTitle['link'] = link
normalizedRssTitle['show_id'] = str(show_id)
if 'title' in normalizedRssTitle.keys():
if 'season' in normalizedRssTitle.keys():
if 'episode' in normalizedRssTitle.keys():
normalizedRssTitleList.append(normalizedRssTitle)
#check versus wantedItem list
for index, wantedItem in enumerate(autosub.WANTEDQUEUE):
wantedItemquality = None
wantedItemreleasegrp = None
wantedItemsource = None
wantedItemcodec = None
wantedItemtitle = wantedItem['title']
wantedItemseason = wantedItem['season']
wantedItemepisode = wantedItem['episode']
originalfile = wantedItem['originalFileLocationOnDisk']
if lang not in wantedItem['lang']:
continue
if 'quality' in wantedItem.keys(): wantedItemquality = wantedItem['quality']
if 'releasegrp' in wantedItem.keys(): wantedItemreleasegrp = wantedItem['releasegrp']
if 'source' in wantedItem.keys(): wantedItemsource = wantedItem['source']
if 'codec' in wantedItem.keys(): wantedItemcodec = wantedItem['codec']
#lets try to find a showid
showid = autosub.Helpers.getShowid(wantedItemtitle)
#no showid? skip this item
if not showid:
continue
for normalizedRssTitle in normalizedRssTitleList:
downloadLink = None
normalizedRssTitleseason = normalizedRssTitle['season']
normalizedRssTitleepisode = normalizedRssTitle['episode']
normalizedRssTitlerssfile = normalizedRssTitle['rssfile']
normalizedRssTitleshowid = int(normalizedRssTitle['show_id'])
normalizedRssTitlelink = normalizedRssTitle['link']
log.debug("checkRSS: Trying to match ID:%r S:%r E:%r (wantedlist) with ID:%r S:%r E:%r (rss)" %(showid, wantedItemseason, wantedItemepisode, normalizedRssTitleshowid, normalizedRssTitleseason, normalizedRssTitleepisode))
if showid == normalizedRssTitleshowid and wantedItemseason == normalizedRssTitleseason and wantedItemepisode == normalizedRssTitleepisode:
log.debug("checkRSS: The episode %s - Season %s Episode %s was found in the RSS list, attempting to match a proper match" % (wantedItemtitle, wantedItemseason, wantedItemepisode))
score = autosub.Helpers.scoreMatch(normalizedRssTitle, normalizedRssTitlerssfile, wantedItemquality, wantedItemreleasegrp, wantedItemsource, wantedItemcodec)
if score >= autosub.MINMATCHSCORERSS:
log.debug ("checkRss: A match got a high enough score. MinMatchscore is %s " % autosub.MINMATCHSCORERSS)
downloadLink = normalizedRssTitlelink + autosub.APIRSS
log.info ("checkRss: Got a match, matching file is: %s" %normalizedRssTitlerssfile)
log.debug("checkRss: Dumping downloadlink for debug purpose: %s" %downloadLink)
if downloadLink:
originalfile = wantedItem['originalFileLocationOnDisk']
# Dutch subs
if autosub.SUBNL != "" and lang == "nl":
srtfile = os.path.splitext(originalfile)[0] + u"." + autosub.SUBNL + u".srt"
elif lang == "nl":
srtfile = os.path.splitext(originalfile)[0] + u".srt"
# English subs
if autosub.SUBENG != "" and lang == "en":
srtfile = os.path.splitext(originalfile)[0] + u"." + autosub.SUBENG + u".srt"
elif lang == "en":
srtfile = os.path.splitext(originalfile)[0] + u".srt"
wantedItem['downloadLink'] = downloadLink
wantedItem['destinationFileLocationOnDisk'] = srtfile
log.info("checkRSS: The episode %s - Season %s Episode %s has a matching subtitle on the RSSFeed, downloading it!" % (wantedItemtitle, wantedItemseason, wantedItemepisode))
downloadItem = wantedItem.copy()
downloadItem['downlang'] = lang
downloadItem['subtitle'] = normalizedRssTitlerssfile
if not DownloadSub(downloadItem):
break
if lang == 'nl' and (autosub.FALLBACKTOENG and not autosub.DOWNLOADENG) and 'en' in wantedItem['lang']:
log.debug('checkRss: We found a dutch subtitle and fallback is true. Removing the english subtitle from the wantedlist.')
wantedItem['lang'].remove('en')
wantedItem['lang'].remove(lang)
if len(wantedItem['lang']) == 0:
toDelete_wantedQueue.append(index)
break
else:
log.debug("checkRss: Matching score is not high enough. Score is %s should be %s" %(str(score),autosub.MINMATCHSCORERSS))
i = len(toDelete_wantedQueue)-1
while i >= 0:
log.debug("checkRSS: Removed item from the wantedQueue at index %s" % toDelete_wantedQueue[i])
autosub.WANTEDQUEUE.pop(toDelete_wantedQueue[i])
i = i-1
# Resetting the toDelete queue for the next run (if need)
toDelete_wantedQueue =[]
log.debug("checkRSS: Finished round of RSS checking")
autosub.WANTEDQUEUELOCK = False
return True
def getSubLink(showid, lang, releaseDetails):
"""
Return the link to download the best matching subtitle.
Matching is based on the provided release details.
Keyword arguments:
showid -- The id of the show on Bierdopje
lang -- Language of the wanted subtitle, Dutch or English
releaseDetails -- Dict containing the quality, releasegrp, source season and episode.
"""
api = autosub.API
if showid == -1:
return (None, None)
quality = None
releasegrp = None
source = None
season = releaseDetails['season']
episode = releaseDetails['episode']
getSubLinkUrl = "%sGetAllSubsFor/%s/%s/%s/%s" % (api, showid, season,
episode, lang)
try:
bierdopjeapi = API(getSubLinkUrl)
dom = minidom.parse(bierdopjeapi.resp)
bierdopjeapi.resp.close()
except:
log.error("getSubLink: The server returned an error for request %s" %
getSubLinkUrl)
return (None, None)
if 'quality' in releaseDetails.keys(): quality = releaseDetails['quality']
if 'releasegrp' in releaseDetails.keys():
releasegrp = releaseDetails['releasegrp']
if 'source' in releaseDetails.keys(): source = releaseDetails['source']
if 'codec' in releaseDetails.keys(): codec = releaseDetails['codec']
if not dom or len(dom.getElementsByTagName('result')) == 0:
return (None, None)
scoredict = {}
releasedict = {}
for sub in dom.getElementsByTagName('result'):
release = sub.getElementsByTagName('filename')[0].firstChild.data
release = release.lower()
# Remove the .srt extension some of the uploaders leave on the file
if release.endswith(".srt"):
release = release[:-4]
# Scoredict is a dictionary with a download link and its match score. This will be used to determine the best match (the highest matchscore)
scoredict[sub.getElementsByTagName(
'downloadlink')[0].firstChild.data] = autosub.Helpers.scoreMatch(
ProcessFilename(release, ''), release, quality, releasegrp,
source, codec)
# Releasedict is a dictionary with the release name, used for the lastdownload database
releasedict[sub.getElementsByTagName('downloadlink')
[0].firstChild.data] = release
if scoredict[sub.getElementsByTagName('downloadlink')
[0].firstChild.data] == 15:
# Sometimes you just find a perfect match, why should we continue to search if we got a perfect match?
log.debug(
'getSubLink: A perfect match found, returning the download link'
)
return (
sub.getElementsByTagName('downloadlink')[0].firstChild.data,
release)
# Done comparing all the results, lets sort them and return the highest result
# If there are results with the same score, the download links which comes first (alphabetically) will be returned
# Also check if the result match the minimal score
sortedscoredict = sorted(scoredict.items(),
key=itemgetter(1),
reverse=True)
toDelete = []
for index, item in enumerate(sortedscoredict):
log.debug(
'getSubLink: checking minimal match score for %s. Minimal match score is: %s'
% (item, autosub.MINMATCHSCORE))
score = item[1]
link = item[0]
if not score >= autosub.MINMATCHSCORE:
log.debug('getSubLink: %s does not match the minimal match score' %
link)
toDelete.append(index)
i = len(toDelete) - 1
while i >= 0:
log.debug("getSubLink: Removed item from the ScoreDict at index %s" %
toDelete[i])
sortedscoredict.pop(toDelete[i])
i = i - 1
if len(sortedscoredict) > 0:
release = releasedict[sortedscoredict[0][0]]
return (sortedscoredict[0][0], release)
return (None, None)
def walkDir(path):
SkipListNL = []
SkipListEN = []
SkipListNL = autosub.SKIPSTRINGNL.split(",")
SkipListEN = autosub.SKIPSTRINGEN.split(",")
# Here we use os.walk to find all files in the path.
for dirname, dirnames, filenames in os.walk(path):
log.debug("scanDisk: directory name: %s" % dirname)
if re.search('_unpack_', dirname, re.IGNORECASE):
log.debug("scanDisk: found a unpack directory, skipping")
continue
if autosub.SKIPHIDDENDIRS and os.path.split(dirname)[1].startswith(
u'.'):
continue
if re.search('_failed_', dirname, re.IGNORECASE):
log.debug("scanDisk: found a failed directory, skipping")
continue
if re.search('@eaDir', dirname, re.IGNORECASE):
log.debug(
"scanDisk: found a Synology indexing directory, skipping this folder and all subfolders."
)
tmpdirs = dirnames[:]
for dir in tmpdirs:
dirnames.remove(dir)
continue
if re.search("@.*thumb", dirname, re.IGNORECASE):
# Ingnore QNAP multimedia thumbnail folders
continue
for filename in filenames:
splitname = filename.split(".")
ext = splitname[len(splitname) - 1]
if ext in ('avi', 'mkv', 'wmv', 'ts', 'mp4'):
if re.search('sample', filename): continue
if not platform.system() == 'Windows':
# Get best ascii compatible character for special characters
try:
if not isinstance(filename, unicode):
coding = detect(filename)
filename = unicode(filename.decode(coding))
correctedFilename = ''.join(
(c for c in unicodedata.normalize('NFD', filename)
if unicodedata.category(c) != 'Mn'))
if filename != correctedFilename:
os.rename(os.path.join(dirname, filename),
os.path.join(dirname, correctedFilename))
log.info("scanDir: Renamed file %s" %
correctedFilename)
filename = correctedFilename
except:
log.error("scanDir: Skipping directory %s" % dirname)
log.error("scanDir: Skipping file %s" % filename)
continue
# What subtitle files should we expect?
lang = []
#Check what the Dutch subtitle would be.
if autosub.SUBNL != "":
srtfilenl = os.path.splitext(
filename)[0] + u"." + autosub.SUBNL + u".srt"
else:
srtfilenl = os.path.splitext(filename)[0] + u".srt"
#Check what the English subtitle would be.
if autosub.SUBENG == "":
# Check for overlapping names
if autosub.SUBNL != "" or not autosub.DOWNLOADDUTCH:
srtfileeng = os.path.splitext(filename)[0] + u".srt"
# Hardcoded fallback
else:
srtfileeng = os.path.splitext(filename)[0] + u".en.srt"
else:
srtfileeng = os.path.splitext(
filename)[0] + u"." + autosub.SUBENG + u".srt"
# Check which languages we want to download based on user settings and check the skipstring
if autosub.DOWNLOADDUTCH:
Skipped = False
for SkipItem in SkipListNL:
if re.search(SkipItem.lower(), filename.lower()):
Skipped = True
log.debug(
"scanDir: %s found in %s so skipped for Dutch subs"
% (SkipItem, filename))
break
if not Skipped and os.path.exists(
os.path.join(dirname, srtfilenl)):
Skipped = True
log.debug(
"scanDir: %s skipped because the subtitle already exists"
% filename)
# If the Dutch subtitle doesn't exist en has no skip item, then add it to the wanted list.
if not Skipped:
lang.append(autosub.DUTCH)
if autosub.DOWNLOADENG:
Skipped = False
for SkipItem in SkipListEN:
if re.search(SkipItem.lower(), filename.lower()):
Skipped = True
log.debug(
"scanDir: %s found in %s so skipped for English subs"
% (SkipItem, filename))
break
if not Skipped and os.path.exists(
os.path.join(dirname, srtfileeng)):
Skipped = True
log.debug(
"scanDir: %s skipped because the subtitle already exists"
% filename)
# If the English subtitle doesn't exist en has no skip item, then add it to the wanted list.
if not Skipped:
lang.append(autosub.ENGLISH)
if (autosub.FALLBACKTOENG and autosub.DOWNLOADDUTCH
) and not autosub.DOWNLOADENG and not Skipped:
# If the Dutch and English subtitles do not exist and not skipped, then add English to the wanted list.
if not os.path.exists(os.path.join(
dirname, srtfilenl)) and not os.path.exists(
os.path.join(dirname, srtfileeng)):
lang.append(autosub.ENGLISH)
if not lang:
# autosub.WANTEDQUEUE empty
continue
log.debug("scanDir: File %s is missing subtitle(s): %s" %
(filename, ', '.join(map(str, lang))))
filenameResults = ProcessFilename(
os.path.splitext(filename)[0],
os.path.splitext(filename)[1])
if 'title' in filenameResults.keys():
if 'season' in filenameResults.keys():
if 'episode' in filenameResults.keys():
title = filenameResults['title']
season = filenameResults['season']
episode = filenameResults['episode']
if not filenameResults[
'releasegrp'] and not filenameResults[
'source'] and not filenameResults[
'quality'] and not filenameResults[
'source']:
continue
if autosub.Helpers.SkipShow(
title, season, episode) == True:
log.debug("scanDir: SkipShow returned True")
log.info(
"scanDir: Skipping %s - Season %s Episode %s"
% (title, season, episode))
continue
if len(lang) == 1:
log.info(
"scanDir: %s subtitle wanted for %s and added to wantedQueue"
% (lang[0], filename))
else:
log.info(
"scanDir: %s subtitles wanted for %s and added to wantedQueue"
% (' and '.join(map(str, lang)), filename))
filenameResults[
'originalFileLocationOnDisk'] = os.path.join(
dirname, filename)
filenameResults['timestamp'] = unicode(
time.strftime(
'%Y-%m-%d %H:%M:%S',
time.localtime(
os.path.getctime(filenameResults[
'originalFileLocationOnDisk']))))
filenameResults['lang'] = lang
autosub.WANTEDQUEUE.append(filenameResults)
else:
log.error(
"scanDir: Could not process the filename properly filename: %s"
% filename)
continue
else:
log.error(
"scanDir: Could not process the filename properly filename: %s"
% filename)
continue
else:
log.error(
"scanDir: Could not process the filename properly filename: %s"
% filename)
continue
def walkDir(path):
SkipListNL = autosub.SKIPSTRINGNL.split(",") if len(
autosub.SKIPSTRINGNL) > 0 else []
SkipListEN = autosub.SKIPSTRINGEN.split(",") if len(
autosub.SKIPSTRINGEN) > 0 else []
# Check for dutch folder skip
if len(autosub.SKIPFOLDERSNL) == 0:
SkipFoldersNL = []
else:
SkipFoldersNL = autosub.SKIPFOLDERSNL.split(",") if len(
autosub.SKIPFOLDERSNL) > 0 else []
for idx, folder in enumerate(SkipFoldersNL):
SkipFoldersNL[idx] = os.path.normpath(
os.path.join(path, folder.strip(" \/")))
# Check for english folder skip
if len(autosub.SKIPFOLDERSEN) == 0:
SkipFoldersEN = []
else:
SkipFoldersEN = autosub.SKIPFOLDERSEN.split(",") if len(
autosub.SKIPFOLDERSEN) > 0 else []
for idx, folder in enumerate(SkipFoldersEN):
SkipFoldersEN[idx] = os.path.normpath(
os.path.join(path, folder.strip(" \/")))
for dirname, dirnames, filenames in os.walk(path, True, WalkError):
SkipThisFolderNL = False
for skip in SkipFoldersNL:
if dirname.startswith(skip):
SkipThisFolderNL = True
break
SkipThisFolderEN = False
for skip in SkipFoldersEN:
if dirname.startswith(skip):
SkipThisFolderEN = True
break
log.debug("scanDisk: directory name: %s" % dirname)
if re.search('_unpack_', dirname, re.IGNORECASE):
log.debug("scanDisk: found a unpack directory, skipping.")
continue
if autosub.SKIPHIDDENDIRS and os.path.split(dirname)[1].startswith(
u'.'):
continue
if re.search('_failed_', dirname, re.IGNORECASE):
log.debug("scanDisk: found a failed directory, skipping.")
continue
if re.search('@eaDir', dirname, re.IGNORECASE):
log.debug(
"scanDisk: found a Synology indexing directory, skipping.")
tmpdirs = dirnames[:]
for dir in tmpdirs:
dirnames.remove(dir)
continue
if re.search("@.*thumb", dirname, re.IGNORECASE):
log.debug(
"scanDisk: found a Qnap multimedia thumbnail folder, skipping."
)
continue
langs = []
FileDict = {}
for filename in filenames:
if autosub.SEARCHSTOP:
log.info('scanDisk: Forced Stop by user')
return
try:
root, ext = os.path.splitext(filename)
if ext[1:] in ('avi', 'mkv', 'wmv', 'ts', 'mp4'):
if re.search('sample', filename):
continue
if not platform.system() == 'Windows':
# Get best ascii compatible character for special characters
try:
if not isinstance(filename, unicode):
coding = detect(filename)['encoding']
filename = unicode(filename.decode(coding),
errors='replace')
correctedFilename = ''.join(
(c for c in unicodedata.normalize(
'NFD', filename)
if unicodedata.category(c) != 'Mn'))
if filename != correctedFilename:
os.rename(
os.path.join(dirname, filename),
os.path.join(dirname, correctedFilename))
log.info("scanDir: Renamed file %s" %
correctedFilename)
filename = correctedFilename
except:
log.error(
"scanDir: Skipping directory, file %s, %s" %
(dirname, filename))
continue
# What subtitle files should we expect?
langs = []
NLext = u'.' + autosub.SUBNL + u'.srt' if autosub.SUBNL else u'.srt'
ENext = u'.' + autosub.SUBENG + u'.srt' if autosub.SUBENG else u'.srt'
ENext = u'.en.srt' if NLext == ENext and autosub.DOWNLOADDUTCH else ENext
if not os.access(dirname, os.W_OK):
log.error('scandisk: No write access to folder: %s' %
dirname)
continue
# Check which languages we want to download based on user settings.
log.debug('scanDir: Processing file: %s' % filename)
if autosub.DOWNLOADDUTCH and not SkipThisFolderNL:
Skipped = False
for SkipItem in SkipListNL:
if not SkipItem: break
if re.search(SkipItem.lower(), filename.lower()):
Skipped = True
break
if Skipped:
log.info(
"scanDir: %s found in %s so skipped for Dutch subs"
% (SkipItem, filename))
elif os.path.exists(os.path.join(
dirname, root + NLext)):
Skipped = True
log.debug(
"scanDir: %s skipped because the Dutch subtitle already exists"
% filename)
else:
# If the Dutch subtitle not skipped and doesn't exist, then add it to the wanted list
langs.append(autosub.DUTCH)
if (autosub.DOWNLOADENG or
(autosub.FALLBACKTOENG and autosub.DOWNLOADDUTCH
and not Skipped)) and not SkipThisFolderEN:
Skipped = False
for SkipItem in SkipListEN:
if not SkipItem: break
if re.search(SkipItem.lower(), filename.lower()):
Skipped = True
break
if Skipped:
log.info(
"scanDir: %s found in %s so skipped for English subs"
% (SkipItem, filename))
elif os.path.exists(os.path.join(
dirname, root + ENext)):
log.debug(
"scanDir: %s skipped because the English subtitle already exists"
% filename)
else:
# If the English subtitle not skipped and doesn't exist, then add it to the wanted list
if not os.path.exists(
os.path.join(dirname, root + ENext)):
langs.append(autosub.ENGLISH)
if not langs:
# nothing to do for this file
continue
FileDict = ProcessFilename(
os.path.splitext(filename)[0].strip(), ext)
if not FileDict:
log.debug(
'scanDisk: not enough info in the filename: %s' %
filename)
continue
Skip = False
if autosub.MINMATCHSCORE & 8 and not FileDict['source']:
Skip = True
elif autosub.MINMATCHSCORE & 4 and not FileDict['quality']:
Skip = True
elif autosub.MINMATCHSCORE & 2 and not FileDict['codec']:
Skip = True
elif autosub.MINMATCHSCORE & 1 and not FileDict[
'releasegrp']:
Skip = True
if Skip:
log.debug(
'scanDisk: Filespec does not meet minmatchscore so skipping this one'
)
continue
FileDict['timestamp'] = unicode(
time.strftime(
'%Y-%m-%d %H:%M:%S',
time.localtime(
os.path.getmtime(
os.path.join(dirname, filename)))))
FileDict['langs'] = langs
FileDict['NLext'] = NLext
FileDict['ENext'] = ENext
FileDict['file'] = root
FileDict['container'] = ext
FileDict['folder'] = dirname
FileDict['ImdbId'], FileDict['A7Id'], FileDict[
'TvdbId'], FileDict['title'] = Helpers.getShowid(
FileDict['title'])
if autosub.Helpers.SkipShow(FileDict['ImdbId'],
FileDict['title'],
FileDict['season'],
FileDict['episode']):
log.debug("scanDir: SKIPPED %s by Skipshow rules." %
FileDict['file'])
continue
log.info("scanDir: %s WANTED FOR: %s" % (langs, filename))
autosub.WANTEDQUEUE.append(FileDict)
time.sleep(0)
except Exception as error:
log.error('scanDir: Problem scanning file %s. Error is: %s' %
(filename, error))
return
def walkDir(path):
SkipListNL = []
SkipListEN = []
SkipListNL = autosub.SKIPSTRINGNL.split(",")
SkipListEN = autosub.SKIPSTRINGEN.split(",")
# Here we use os.walk to find all files in the path.
for dirname, dirnames, filenames in os.walk(path):
log.debug("scanDisk: directory name: %s" %dirname)
if re.search('_unpack_', dirname, re.IGNORECASE):
log.debug("scanDisk: found a unpack directory, skipping")
continue
if autosub.SKIPHIDDENDIRS and os.path.split(dirname)[1].startswith(u'.'):
continue
if re.search('_failed_', dirname, re.IGNORECASE):
log.debug("scanDisk: found a failed directory, skipping")
continue
if re.search('@eaDir', dirname, re.IGNORECASE):
log.debug("scanDisk: found a Synology indexing directory, skipping this folder and all subfolders.")
tmpdirs = dirnames[:]
for dir in tmpdirs:
dirnames.remove(dir)
continue
if re.search("@.*thumb", dirname, re.IGNORECASE):
# Ingnore QNAP multimedia thumbnail folders
continue
for filename in filenames:
splitname = filename.split(".")
ext = splitname[len(splitname) - 1]
if ext in ('avi', 'mkv', 'wmv', 'ts', 'mp4'):
if re.search('sample', filename): continue
if not platform.system() == 'Windows':
# Get best ascii compatible character for special characters
try:
if not isinstance(filename, unicode):
coding = detect(filename)
filename = unicode(filename.decode(coding))
correctedFilename = ''.join((c for c in unicodedata.normalize('NFD', filename) if unicodedata.category(c) != 'Mn'))
if filename != correctedFilename:
os.rename(os.path.join(dirname, filename), os.path.join(dirname, correctedFilename))
log.info("scanDir: Renamed file %s" % correctedFilename)
filename = correctedFilename
except:
log.error("scanDir: Skipping directory %s" % dirname)
log.error("scanDir: Skipping file %s" % filename)
continue
# What subtitle files should we expect?
lang=[]
#Check what the Dutch subtitle would be.
if autosub.SUBNL != "":
srtfilenl = os.path.splitext(filename)[0] + u"." + autosub.SUBNL + u".srt"
else:
srtfilenl = os.path.splitext(filename)[0] + u".srt"
#Check what the English subtitle would be.
if autosub.SUBENG == "":
# Check for overlapping names
if autosub.SUBNL != "" or not autosub.DOWNLOADDUTCH:
srtfileeng = os.path.splitext(filename)[0] + u".srt"
# Hardcoded fallback
else:
srtfileeng = os.path.splitext(filename)[0] + u".en.srt"
else:
srtfileeng = os.path.splitext(filename)[0] + u"." + autosub.SUBENG + u".srt"
# Check which languages we want to download based on user settings and check the skipstring
if autosub.DOWNLOADDUTCH:
Skipped = False
for SkipItem in SkipListNL:
if re.search(SkipItem.lower(), filename.lower()):
Skipped = True
log.debug("scanDir: %s found in %s so skipped for Dutch subs" % (SkipItem, filename))
break
if not Skipped and os.path.exists(os.path.join(dirname, srtfilenl)):
Skipped = True
log.debug("scanDir: %s skipped because the subtitle already exists" % filename)
# If the Dutch subtitle doesn't exist en has no skip item, then add it to the wanted list.
if not Skipped:
lang.append(autosub.DUTCH)
if autosub.DOWNLOADENG:
Skipped = False
for SkipItem in SkipListEN:
if re.search(SkipItem.lower(), filename.lower()):
Skipped = True
log.debug("scanDir: %s found in %s so skipped for English subs" % (SkipItem, filename))
break
if not Skipped and os.path.exists(os.path.join(dirname, srtfileeng)):
Skipped = True
log.debug("scanDir: %s skipped because the subtitle already exists" % filename)
# If the English subtitle doesn't exist en has no skip item, then add it to the wanted list.
if not Skipped:
lang.append(autosub.ENGLISH)
if (autosub.FALLBACKTOENG and autosub.DOWNLOADDUTCH) and not autosub.DOWNLOADENG and not Skipped:
# If the Dutch and English subtitles do not exist and not skipped, then add English to the wanted list.
if not os.path.exists(os.path.join(dirname, srtfilenl)) and not os.path.exists(os.path.join(dirname, srtfileeng)):
lang.append(autosub.ENGLISH)
if not lang:
# autosub.WANTEDQUEUE empty
continue
log.debug("scanDir: File %s is missing subtitle(s): %s" % (filename, ', '.join(map(str,lang))))
filenameResults = ProcessFilename(os.path.splitext(filename)[0], os.path.splitext(filename)[1])
if 'title' in filenameResults.keys():
if 'season' in filenameResults.keys():
if 'episode' in filenameResults.keys():
title = filenameResults['title']
season = filenameResults['season']
episode = filenameResults['episode']
if not filenameResults['releasegrp'] and not filenameResults['source'] and not filenameResults['quality'] and not filenameResults['source']:
continue
if autosub.Helpers.SkipShow(title, season, episode) == True:
log.debug("scanDir: SkipShow returned True")
log.info("scanDir: Skipping %s - Season %s Episode %s" % (title, season, episode))
continue
if len(lang) == 1:
log.info("scanDir: %s subtitle wanted for %s and added to wantedQueue" % (lang[0], filename))
else:
log.info("scanDir: %s subtitles wanted for %s and added to wantedQueue" % (' and '.join(map(str,lang)), filename))
filenameResults['originalFileLocationOnDisk'] = os.path.join(dirname, filename)
filenameResults['timestamp'] = unicode(time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(os.path.getctime(filenameResults['originalFileLocationOnDisk']))))
filenameResults['lang'] = lang
autosub.WANTEDQUEUE.append(filenameResults)
else:
log.error("scanDir: Could not process the filename properly filename: %s" % filename)
continue
else:
log.error("scanDir: Could not process the filename properly filename: %s" % filename)
continue
else:
log.error("scanDir: Could not process the filename properly filename: %s" % filename)
continue
def walkDir(path):
for dirname, dirnames, filenames in os.walk(os.path.join(path)):
log.debug("scanDisk: directory name: %s" %dirname)
if re.search('_unpack_', dirname, re.IGNORECASE):
log.debug("scanDisk: found a unpack directory, skipping")
continue
if autosub.SKIPHIDDENDIRS and os.path.split(dirname)[1].startswith(u'.'):
continue
if re.search('_failed_', dirname, re.IGNORECASE):
log.debug("scanDisk: found a failed directory, skipping")
continue
for filename in filenames:
splitname = filename.split(".")
ext = splitname[len(splitname) - 1]
if ext in ('avi', 'mkv', 'wmv', 'ts', 'mp4'):
if re.search('sample', filename): continue
# What subtitle files should we expect?
if autosub.SUBNL != "":
srtfile = os.path.splitext(filename)[0] + u"." + autosub.SUBNL + u".srt"
else:
srtfile = os.path.splitext(filename)[0] + u".srt"
srtfileeng = os.path.splitext(filename)[0] + u"." + autosub.SUBENG + u".srt"
if not os.path.exists(os.path.join(dirname, srtfile)) or (not os.path.exists(os.path.join(dirname, srtfileeng)) and autosub.DOWNLOADENG):
log.debug("scanDir: File %s is missing a subtitle" % filename)
lang = []
filenameResults = ProcessFilename(os.path.splitext(filename)[0], os.path.splitext(filename)[1])
if 'title' in filenameResults.keys():
if 'season' in filenameResults.keys():
if 'episode' in filenameResults.keys():
title = filenameResults['title']
season = filenameResults['season']
episode = filenameResults['episode']
if autosub.Helpers.SkipShow(title, season, episode) == True:
log.debug("scanDir: SkipShow returned True")
log.info("scanDir: Skipping %s - Season %s Episode %s" % (title, season, episode))
continue
log.info("scanDir: Dutch subtitle wanted for %s and added to wantedQueue" % filename)
filenameResults['originalFileLocationOnDisk'] = os.path.join(dirname, filename)
filenameResults['timestamp'] = unicode(time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(os.path.getctime(filenameResults['originalFileLocationOnDisk']))))
if not os.path.exists(os.path.join(dirname, srtfile)):
lang.append('nl')
if not os.path.exists(os.path.join(dirname, srtfileeng)) and (autosub.FALLBACKTOENG or autosub.DOWNLOADENG):
lang.append('en')
filenameResults['lang'] = lang
autosub.WANTEDQUEUE.append(filenameResults)
else:
log.error("scanDir: Could not process the filename properly filename: %s" % filename)
continue
else:
log.error("scanDir: Could not process the filename properly filename: %s" % filename)
continue
else:
log.error("scanDir: Could not process the filename properly filename: %s" % filename)
continue
def run(self):
log.debug("checkRSS: Starting round of RSS checking")
if autosub.WANTEDQUEUELOCK:
log.debug("checkRSS: Exiting, another threat is using the queues")
return False
else:
autosub.WANTEDQUEUELOCK = True
toDelete_wantedQueue = []
langs = ["nl"]
# default is to only check for Dutch subs
# but if English should be downloaden, check them too
# It is very important that the dutch language is run first!
if autosub.FALLBACKTOENG or autosub.DOWNLOADENG:
langs.append("en")
log.debug("checkRSS: We also want to check the English RSS feed")
for lang in langs:
if lang == "en":
RSSURL = autosub.ENRSSURL
log.debug("checkRSS: Now using the English RSS feed")
else:
RSSURL = autosub.NLRSSURL
log.debug("checkRSS: Now using the Dutch RSS feed")
try:
bierdopjeapi = API(RSSURL, RSS=True)
soup = BeautifulStoneSoup(bierdopjeapi.resp.read())
log.debug("checkRss: Succussfully connected to %s" % RSSURL)
bierdopjeapi.close()
except:
log.error(
"checkRss: The server returned an error for request %s" %
RSSURL)
autosub.WANTEDQUEUELOCK = False
continue
# Parse all the item-tags from the RSSFeed
# The information that is parsed is: title, link and show_id
# The show_id is later used to match with the wanted items
# The title is used the determine the quality / source / releasegrp
rssItemList = []
items = soup.findAll('item')
if not len(items) > 0:
log.error("checkRss: invalid RssFeed")
log.debug("checkRss: dumping Rssfeed %s" % (soup.prettify()))
else:
log.debug("checkRss: Valid RssFeed")
for x in items:
soupx = BeautifulStoneSoup(str(x))
title = soupx.find('title').string
show_id = soupx.find('show_id').string
link = soupx.find('enclosure')['url']
item = {}
item['title'] = title
item['link'] = link
item['show_id'] = show_id
log.debug("checkRSS: Parsed from RSSFEED: %s %s %s" %
(title, link, show_id))
rssItemList.append(item)
normalizedRssTitleList = []
# Now we create a new rsslist, containing information like: episode, season, etc
for item in rssItemList:
title = item['title']
link = item['link']
show_id = item['show_id']
log.debug(
"checkRSS: Normalizing the following entry in the RSS results: %s"
% title)
normalizedRssTitle = ProcessFilename(title, '')
normalizedRssTitle['rssfile'] = title
normalizedRssTitle['link'] = link
normalizedRssTitle['show_id'] = str(show_id)
if 'title' in normalizedRssTitle.keys():
if 'season' in normalizedRssTitle.keys():
if 'episode' in normalizedRssTitle.keys():
normalizedRssTitleList.append(normalizedRssTitle)
#check versus wantedItem list
for index, wantedItem in enumerate(autosub.WANTEDQUEUE):
wantedItemquality = None
wantedItemreleasegrp = None
wantedItemsource = None
wantedItemcodec = None
wantedItemtitle = wantedItem['title']
wantedItemseason = wantedItem['season']
wantedItemepisode = wantedItem['episode']
originalfile = wantedItem['originalFileLocationOnDisk']
if lang not in wantedItem['lang']:
continue
if 'quality' in wantedItem.keys():
wantedItemquality = wantedItem['quality']
if 'releasegrp' in wantedItem.keys():
wantedItemreleasegrp = wantedItem['releasegrp']
if 'source' in wantedItem.keys():
wantedItemsource = wantedItem['source']
if 'codec' in wantedItem.keys():
wantedItemcodec = wantedItem['codec']
#lets try to find a showid
showid = autosub.Helpers.getShowid(wantedItemtitle)
#no showid? skip this item
if not showid:
continue
for normalizedRssTitle in normalizedRssTitleList:
downloadLink = None
normalizedRssTitleseason = normalizedRssTitle['season']
normalizedRssTitleepisode = normalizedRssTitle['episode']
normalizedRssTitlerssfile = normalizedRssTitle['rssfile']
normalizedRssTitleshowid = int(
normalizedRssTitle['show_id'])
normalizedRssTitlelink = normalizedRssTitle['link']
log.debug(
"checkRSS: Trying to match ID:%r S:%r E:%r (wantedlist) with ID:%r S:%r E:%r (rss)"
% (showid, wantedItemseason, wantedItemepisode,
normalizedRssTitleshowid, normalizedRssTitleseason,
normalizedRssTitleepisode))
if showid == normalizedRssTitleshowid and wantedItemseason == normalizedRssTitleseason and wantedItemepisode == normalizedRssTitleepisode:
log.debug(
"checkRSS: The episode %s - Season %s Episode %s was found in the RSS list, attempting to match a proper match"
% (wantedItemtitle, wantedItemseason,
wantedItemepisode))
score = autosub.Helpers.scoreMatch(
normalizedRssTitle, normalizedRssTitlerssfile,
wantedItemquality, wantedItemreleasegrp,
wantedItemsource, wantedItemcodec)
if score >= autosub.MINMATCHSCORERSS:
log.debug(
"checkRss: A match got a high enough score. MinMatchscore is %s "
% autosub.MINMATCHSCORERSS)
downloadLink = normalizedRssTitlelink + autosub.APIRSS
log.info(
"checkRss: Got a match, matching file is: %s" %
normalizedRssTitlerssfile)
log.debug(
"checkRss: Dumping downloadlink for debug purpose: %s"
% downloadLink)
if downloadLink:
originalfile = wantedItem[
'originalFileLocationOnDisk']
# Dutch subs
if autosub.SUBNL != "" and lang == "nl":
srtfile = os.path.splitext(originalfile)[
0] + u"." + autosub.SUBNL + u".srt"
elif lang == "nl":
srtfile = os.path.splitext(
originalfile)[0] + u".srt"
# English subs
if autosub.SUBENG != "" and lang == "en":
srtfile = os.path.splitext(originalfile)[
0] + u"." + autosub.SUBENG + u".srt"
elif lang == "en":
srtfile = os.path.splitext(
originalfile)[0] + u".srt"
wantedItem['downloadLink'] = downloadLink
wantedItem[
'destinationFileLocationOnDisk'] = srtfile
log.info(
"checkRSS: The episode %s - Season %s Episode %s has a matching subtitle on the RSSFeed, downloading it!"
% (wantedItemtitle, wantedItemseason,
wantedItemepisode))
downloadItem = wantedItem.copy()
downloadItem['downlang'] = lang
downloadItem[
'subtitle'] = normalizedRssTitlerssfile
if not DownloadSub(downloadItem):
break
if lang == 'nl' and (
autosub.FALLBACKTOENG
and not autosub.DOWNLOADENG
) and 'en' in wantedItem['lang']:
log.debug(
'checkRss: We found a dutch subtitle and fallback is true. Removing the english subtitle from the wantedlist.'
)
wantedItem['lang'].remove('en')
wantedItem['lang'].remove(lang)
if len(wantedItem['lang']) == 0:
toDelete_wantedQueue.append(index)
break
else:
log.debug(
"checkRss: Matching score is not high enough. Score is %s should be %s"
% (str(score), autosub.MINMATCHSCORERSS))
i = len(toDelete_wantedQueue) - 1
while i >= 0:
log.debug(
"checkRSS: Removed item from the wantedQueue at index %s" %
toDelete_wantedQueue[i])
autosub.WANTEDQUEUE.pop(toDelete_wantedQueue[i])
i = i - 1
# Resetting the toDelete queue for the next run (if need)
toDelete_wantedQueue = []
log.debug("checkRSS: Finished round of RSS checking")
autosub.WANTEDQUEUELOCK = False
return True
def SubtitleSeeker(showid, lang, releaseDetails, sourceWebsites):
# Get the scored list for all SubtitleSeeker hits
api = autosub.API
season = releaseDetails['season'] if 'season' in releaseDetails.keys(
) else None
episode = releaseDetails['episode'] if 'episode' in releaseDetails.keys(
) else None
quality = releaseDetails['quality'] if 'quality' in releaseDetails.keys(
) else None
releasegrp = releaseDetails[
'releasegrp'] if 'releasegrp' in releaseDetails.keys() else None
source = releaseDetails['source'] if 'source' in releaseDetails.keys(
) else None
codec = releaseDetails['codec'] if 'codec' in releaseDetails.keys(
) else None
# this is the API search
getSubLinkUrl = "%s&imdb=%s&season=%s&episode=%s&language=%s" % (
api, showid, season, episode, lang)
log.info('getSubLinks: SubtitleSeeker request URL: %s' % getSubLinkUrl)
if autosub.Helpers.checkAPICallsSubSeeker(use=True):
try:
subseekerapi = autosub.Helpers.API(getSubLinkUrl)
dom = minidom.parse(subseekerapi.resp)
subseekerapi.resp.close()
except:
log.error(
"getSubLink: The server returned an error for request %s" %
getSubLinkUrl)
return None
else:
log.error("API: out of api calls for SubtitleSeeker.com")
return None
if not len(dom.getElementsByTagName('error')) == 0:
for error in dom.getElementsByTagName('error'):
try:
errormsg = error.getElementsByTagName('msg')[0].firstChild.data
log.error("getSubLink: Error found in API response: %s" %
errormsg)
except AttributeError:
log.debug(
"getSubLink: Invalid msg tag in API response, unable to read error message."
)
return None
if not dom or len(dom.getElementsByTagName('item')) == 0:
return None
scoreList = []
for sub in dom.getElementsByTagName('item'):
try:
release = sub.getElementsByTagName('release')[0].firstChild.data
release = release.lower()
# Remove the .srt extension some of the uploaders leave on the file
if release.endswith(".srt"):
release = release[:-4]
except AttributeError:
log.debug(
"getSubLink: Invalid release tag in API response, skipping this item."
)
continue
try:
website = sub.getElementsByTagName('site')[0].firstChild.data
website = website.lower()
except AttributeError:
log.debug(
"getSubLink: Invalid website tag in API response, skipping this item."
)
continue
try:
url = sub.getElementsByTagName('url')[0].firstChild.data
except AttributeError:
log.debug(
"getSubLink: Invalid URL tag in API response, skipping this item."
)
continue
tmpDict = ProcessFilename(release, '')
if not website in sourceWebsites or not tmpDict:
continue
# ReleaseDict is a dictionary with the score, releasename and source website for the subtitle release
releaseDict = {
'score': None,
'releasename': release,
'url': url,
'website': website
}
releaseDict['score'] = autosub.Helpers.scoreMatch(
tmpDict, release, quality, releasegrp, source, codec)
scoreList.append(releaseDict)
def Opensubtitles(EpisodeId, language, releaseDetails):
title = releaseDetails['title'] if 'title' in releaseDetails.keys(
) else None
season = releaseDetails['season'] if 'season' in releaseDetails.keys(
) else None
episode = releaseDetails['episode'] if 'episode' in releaseDetails.keys(
) else None
quality = releaseDetails['quality'] if 'quality' in releaseDetails.keys(
) else None
releasegrp = releaseDetails[
'releasegrp'] if 'releasegrp' in releaseDetails.keys() else None
source = releaseDetails['source'] if 'source' in releaseDetails.keys(
) else None
codec = releaseDetails['codec'] if 'codec' in releaseDetails.keys(
) else None
LangId = 'dut' if language == 'Dutch' else 'eng'
SearchUrl = '/xml/search/sublanguageid-' + str(LangId) + '/imdbid-' + str(
EpisodeId)
try:
TimeOut()
RequestResult = autosub.OPENSUBTTITLESSESSION.get(
autosub.OPENSUBTITLESURL + SearchUrl, timeout=10)
Referer = SearchUrl.replace('/xml', '')
autosub.OPENSUBTTITLESSESSION.headers.update({'referer': Referer})
except:
log.debug('getSubLinks: Could not connect to OpenSubtitles.')
return None
if 'text/xml' not in RequestResult.headers['Content-Type']:
log.error('getSubLinks: OpenSubtitles responded with an error')
return None
try:
root = ET.fromstring(RequestResult.content)
except:
log.debug(
'getSubLinks: Serie with IMDB ID %s could not be found on OpenSubtitles.'
% SerieImdb)
return None
try:
SubTitles = root.find('.//search/results')
except:
log.debug(
'getSubLinks: Serie with IMDB ID %s could not be found on OpenSubtitles.'
% SerieImdb)
return None
# We fetch the show overview page and search voor the Id's of the epiode we want
# Because as we have this whole page, we put the other Episode Id's in the cache
scoreList = []
for Sub in SubTitles:
try:
if Sub.tag != 'subtitle':
continue
try:
SubBad = int(Sub.find('SubBad').text)
if SubBad > 0:
log.debug(
"getSubLinks: OpenSubtitles has %d bad reports for this subtitle, skipping."
% SubBad)
continue
else:
log.debug(
"getSubLinks: OpenSubtitles has %d bad reports for this subtitle."
% SubBad)
except:
pass
Link = Sub.find('IDSubtitle').attrib['Link']
release = Sub.find('MovieReleaseName').text.split('[]')[0].lower()
except:
continue
url = autosub.OPENSUBTITLESURL[:-3] + Link + '/xml'
tmpDict = ProcessFilename(release, '')
if not tmpDict:
continue
# ReleaseDict is a dictionary with the score, releasename and source website for the subtitle release
releaseDict = {
'score': None,
'releasename': release,
'url': url,
'website': 'opensubtitles.org'
}
releaseDict['score'] = autosub.Helpers.scoreMatch(
tmpDict, release, quality, releasegrp, source, codec)
scoreList.append(releaseDict)
return scoreList
def walkDir(path):
for dirname, dirnames, filenames in os.walk(path):
log.debug("scanDisk: directory name: %s" % dirname)
if re.search("_unpack_", dirname, re.IGNORECASE):
log.debug("scanDisk: found a unpack directory, skipping")
continue
if autosub.SKIPHIDDENDIRS and os.path.split(dirname)[1].startswith(u"."):
continue
if re.search("_failed_", dirname, re.IGNORECASE):
log.debug("scanDisk: found a failed directory, skipping")
continue
if re.search("@eaDir", dirname, re.IGNORECASE):
log.debug("scanDisk: found a Synology indexing directory, skipping this folder and all subfolders.")
tmpdirs = dirnames[:]
for dir in tmpdirs:
dirnames.remove(dir)
continue
if re.search("@.*thumb", dirname, re.IGNORECASE):
# Ingnore QNAP multimedia thumbnail folders
continue
for filename in filenames:
splitname = filename.split(".")
ext = splitname[len(splitname) - 1]
if ext in ("avi", "mkv", "wmv", "ts", "mp4"):
if re.search("sample", filename):
continue
if autosub.WEBDL == "None":
if re.search("web-dl", filename.lower()):
log.debug("scanDisk: WEB-DL is set to 'None', skipping %s" % filename)
continue
if not platform.system() == "Windows":
# Get best ascii compatible character for special characters
try:
if not isinstance(filename, unicode):
coding = detect(filename)
filename = unicode(filename.decode(coding))
correctedFilename = "".join(
(c for c in unicodedata.normalize("NFD", filename) if unicodedata.category(c) != "Mn")
)
if filename != correctedFilename:
os.rename(os.path.join(dirname, filename), os.path.join(dirname, correctedFilename))
log.info("scanDir: Renamed file %s" % correctedFilename)
filename = correctedFilename
except:
log.error("scanDir: Skipping directory %s" % dirname)
log.error("scanDir: Skipping file %s" % filename)
continue
# What subtitle files should we expect?
lang = []
# Check what the Dutch subtitle would be.
if autosub.SUBNL != "":
srtfilenl = os.path.splitext(filename)[0] + u"." + autosub.SUBNL + u".srt"
else:
srtfilenl = os.path.splitext(filename)[0] + u".srt"
# Check what the English subtitle would be.
if autosub.SUBENG == "":
# Check for overlapping names
if autosub.SUBNL != "" or not autosub.DOWNLOADDUTCH:
srtfileeng = os.path.splitext(filename)[0] + u".srt"
# Hardcoded fallback
else:
srtfileeng = os.path.splitext(filename)[0] + u".en.srt"
else:
srtfileeng = os.path.splitext(filename)[0] + u"." + autosub.SUBENG + u".srt"
# Check which languages we want to download based on user settings.
if autosub.DOWNLOADDUTCH:
# If the Dutch subtitle doesn't exist, then add it to the wanted list.
if not os.path.exists(os.path.join(dirname, srtfilenl)):
lang.append(autosub.DUTCH)
if autosub.DOWNLOADENG:
# If the English subtitle doesn't exist, then add it to the wanted list.
if not os.path.exists(os.path.join(dirname, srtfileeng)):
if autosub.WEBDL == "DutchOnly" and re.search("web-dl", filename.lower()):
log.debug(
"scanDisk: WEB-DL is set to 'Dutch Only', not adding English as wanted for %s"
% filename
)
else:
lang.append(autosub.ENGLISH)
if (autosub.FALLBACKTOENG and autosub.DOWNLOADDUTCH) and not autosub.DOWNLOADENG:
# If the Dutch and English subtitles do not exist, then add English to the wanted list.
if not os.path.exists(os.path.join(dirname, srtfilenl)) and not os.path.exists(
os.path.join(dirname, srtfileeng)
):
if autosub.WEBDL == "DutchOnly" and re.search("web-dl", filename.lower()):
log.debug(
"scanDisk: WEB-DL is set to 'Dutch Only', not adding English as wanted for %s"
% filename
)
else:
lang.append(autosub.ENGLISH)
if not lang:
# autosub.WANTEDQUEUE empty
continue
log.debug("scanDir: File %s is missing subtitle(s): %s" % (filename, ", ".join(map(str, lang))))
filenameResults = ProcessFilename(os.path.splitext(filename)[0], os.path.splitext(filename)[1])
if "title" in filenameResults.keys():
if "season" in filenameResults.keys():
if "episode" in filenameResults.keys():
title = filenameResults["title"]
season = filenameResults["season"]
episode = filenameResults["episode"]
if (
not filenameResults["releasegrp"]
and not filenameResults["source"]
and not filenameResults["quality"]
and not filenameResults["source"]
):
continue
if autosub.Helpers.SkipShow(title, season, episode) == True:
log.debug("scanDir: SkipShow returned True")
log.info("scanDir: Skipping %s - Season %s Episode %s" % (title, season, episode))
continue
if len(lang) == 1:
log.info(
"scanDir: %s subtitle wanted for %s and added to wantedQueue" % (lang[0], filename)
)
else:
log.info(
"scanDir: %s subtitles wanted for %s and added to wantedQueue"
% (" and ".join(map(str, lang)), filename)
)
filenameResults["originalFileLocationOnDisk"] = os.path.join(dirname, filename)
filenameResults["timestamp"] = unicode(
time.strftime(
"%Y-%m-%d %H:%M:%S",
time.localtime(os.path.getctime(filenameResults["originalFileLocationOnDisk"])),
)
)
filenameResults["lang"] = lang
autosub.WANTEDQUEUE.append(filenameResults)
else:
log.error("scanDir: Could not process the filename properly filename: %s" % filename)
continue
else:
log.error("scanDir: Could not process the filename properly filename: %s" % filename)
continue
else:
log.error("scanDir: Could not process the filename properly filename: %s" % filename)
continue
