def walkDir(path):
SkipListNL = autosub.SKIPSTRINGNL.split(",") if len(autosub.SKIPSTRINGNL) > 0 else []
SkipListEN = autosub.SKIPSTRINGEN.split(",") if len(autosub.SKIPSTRINGEN) > 0 else []
if len(autosub.SKIPFOLDERSNL) == 0:
SkipFoldersNL = []
else:
SkipFoldersNL = autosub.SKIPFOLDERSNL.split(",") if len(autosub.SKIPFOLDERSNL) > 0 else []
for idx,folder in enumerate(SkipFoldersNL):
SkipFoldersNL[idx] = os.path.normpath(os.path.join(path,folder.rstrip("\/").lstrip("\/")))
if len(autosub.SKIPFOLDERSNL) == 0:
SkipFoldersNL = []
else:
SkipFoldersNL = autosub.SKIPFOLDERSNL.split(",") if len(autosub.SKIPFOLDERSNL) > 0 else []
for idx,folder in enumerate(SkipFoldersNL):
SkipFoldersNL[idx] = os.path.normpath(path + folder)
if len(autosub.SKIPFOLDERSEN) == 0:
SkipFoldersEN = []
else:
SkipFoldersEN = autosub.SKIPFOLDERSEN.split(",") if len(autosub.SKIPFOLDERSEN) > 0 else []
for idx,folder in enumerate(SkipFoldersEN):
SkipFoldersEN[idx] = os.path.normpath(path + folder)
for dirname, dirnames, filenames in os.walk(path, True, WalkError):
SkipThisFolderNL = False
for skip in SkipFoldersNL:
if dirname.startswith(skip):
SkipThisFolderNL = True
break
SkipThisFolderEN = False
for skip in SkipFoldersEN:
if dirname.startswith(skip):
SkipThisFolderEN = True
break
log.debug("scanDisk: directory name: %s" %dirname)
if re.search('_unpack_', dirname, re.IGNORECASE):
log.debug("scanDisk: found a unpack directory, skipping.")
continue
if autosub.SKIPHIDDENDIRS and os.path.split(dirname)[1].startswith(u'.'):
continue
if re.search('_failed_', dirname, re.IGNORECASE):
log.debug("scanDisk: found a failed directory, skipping.")
continue
if re.search('@eaDir', dirname, re.IGNORECASE):
log.debug("scanDisk: found a Synology indexing directory, skipping.")
tmpdirs = dirnames[:]
for dir in tmpdirs:
dirnames.remove(dir)
continue
if re.search("@.*thumb", dirname, re.IGNORECASE):
log.debug("scanDisk: found a Qnap multimedia thumbnail folder, skipping.")
continue
langs = []
FileDict = {}
for filename in filenames:
root,ext = os.path.splitext(filename)
if ext[1:] in ('avi', 'mkv', 'wmv', 'ts', 'mp4'):
if re.search('sample', filename):
continue
if not platform.system() == 'Windows':
# Get best ascii compatible character for special characters
try:
if not isinstance(filename, unicode):
coding = detect(filename)['encoding']
filename = unicode(filename.decode(coding),errors='replace')
correctedFilename = ''.join((c for c in unicodedata.normalize('NFD', filename) if unicodedata.category(c) != 'Mn'))
if filename != correctedFilename:
os.rename(os.path.join(dirname, filename), os.path.join(dirname, correctedFilename))
log.info("scanDir: Renamed file %s" % correctedFilename)
filename = correctedFilename
except:
log.error("scanDir: Skipping directory, file %s, %s" % (dirname,filename))
continue
# What subtitle files should we expect?
langs = []
NLext = u'.' + autosub.SUBNL + u'.srt' if autosub.SUBNL else u'.srt'
ENext = u'.' + autosub.SUBENG + u'.srt' if autosub.SUBENG else u'.srt'
ENext = u'.en.srt'if NLext == ENext and autosub.DOWNLOADDUTCH else ENext
if not os.access(dirname, os.W_OK):
log.error('scandisk: No write access to folder: %s' % dirname)
continue
# Check which languages we want to download based on user settings.
if autosub.DOWNLOADDUTCH and not SkipThisFolderNL:
Skipped = False
for SkipItem in SkipListNL:
if not SkipItem: break
if re.search(SkipItem.lower(), filename.lower()):
Skipped = True
break
if Skipped:
log.info("scanDir: %s found in %s so skipped for Dutch subs" % (SkipItem, filename))
elif os.path.exists(os.path.join(dirname, root + NLext)):
Skipped = True
log.debug("scanDir: %s skipped because the Dutch subtitle already exists" % filename)
else:
# If the Dutch subtitle not skipped and doesn't exist, then add it to the wanted list
langs.append(autosub.DUTCH)
if (autosub.DOWNLOADENG or (autosub.FALLBACKTOENG and autosub.DOWNLOADDUTCH and not Skipped)) and not SkipThisFolderEN:
Skipped = False
for SkipItem in SkipListEN:
if not SkipItem: break
if re.search(SkipItem.lower(), filename.lower()):
Skipped = True
break
if Skipped:
log.info("scanDir: %s found in %s so skipped for English subs" % (SkipItem, filename))
elif os.path.exists(os.path.join(dirname, root + ENext)):
log.debug("scanDir: %s skipped because the English subtitle already exists" % filename)
else:
# If the English subtitle not skipped and doesn't exist, then add it to the wanted list
if not os.path.exists(os.path.join(dirname, root + ENext)):
langs.append(autosub.ENGLISH)
if not langs:
# nothing to do for this file
continue
FileDict = ProcessFilename(os.path.splitext(filename)[0].strip(), ext)
time.sleep(0)
if not FileDict:
continue
if not 'title' in FileDict.keys() or not 'season' in FileDict.keys() or not 'episode' in FileDict.keys():
continue
if not FileDict['releasegrp'] and not FileDict['source'] and not FileDict['quality'] and not FileDict['source']:
log.error("scanDir: Not enough info in filename: %s" % filename)
continue
FileDict['timestamp'] = unicode(time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(os.path.getctime(os.path.join(dirname, filename)))))
FileDict['langs'] = langs
FileDict['NLext'] = NLext
FileDict['ENext'] = ENext
FileDict['file'] = root
FileDict['container'] = ext
FileDict['folder'] = dirname
FileDict['ImdbId'],FileDict['A7Id'], FileDict['TvdbId'], FileDict['title'] = Helpers.getShowid(FileDict['title'],autosub.ADDIC7EDLOGGED_IN)
if autosub.Helpers.SkipShow(FileDict['ImdbId'],FileDict['title'], FileDict['season'], FileDict['episode']):
continue
log.info("scanDir: %s WANTED FOR: %s" % (langs, filename))
autosub.WANTEDQUEUE.append(FileDict)
return
def walkDir(path):
SkipListNL = []
SkipListEN = []
SkipListNL = autosub.SKIPSTRINGNL.split(",")
SkipListEN = autosub.SKIPSTRINGEN.split(",")
# Here we use os.walk to find all files in the path.
for dirname, dirnames, filenames in os.walk(path):
log.debug("scanDisk: directory name: %s" % dirname)
if re.search('_unpack_', dirname, re.IGNORECASE):
log.debug("scanDisk: found a unpack directory, skipping")
continue
if autosub.SKIPHIDDENDIRS and os.path.split(dirname)[1].startswith(
u'.'):
continue
if re.search('_failed_', dirname, re.IGNORECASE):
log.debug("scanDisk: found a failed directory, skipping")
continue
if re.search('@eaDir', dirname, re.IGNORECASE):
log.debug(
"scanDisk: found a Synology indexing directory, skipping this folder and all subfolders."
)
tmpdirs = dirnames[:]
for dir in tmpdirs:
dirnames.remove(dir)
continue
if re.search("@.*thumb", dirname, re.IGNORECASE):
# Ingnore QNAP multimedia thumbnail folders
continue
for filename in filenames:
splitname = filename.split(".")
ext = splitname[len(splitname) - 1]
if ext in ('avi', 'mkv', 'wmv', 'ts', 'mp4'):
if re.search('sample', filename): continue
if not platform.system() == 'Windows':
# Get best ascii compatible character for special characters
try:
if not isinstance(filename, unicode):
coding = detect(filename)
filename = unicode(filename.decode(coding))
correctedFilename = ''.join(
(c for c in unicodedata.normalize('NFD', filename)
if unicodedata.category(c) != 'Mn'))
if filename != correctedFilename:
os.rename(os.path.join(dirname, filename),
os.path.join(dirname, correctedFilename))
log.info("scanDir: Renamed file %s" %
correctedFilename)
filename = correctedFilename
except:
log.error("scanDir: Skipping directory %s" % dirname)
log.error("scanDir: Skipping file %s" % filename)
continue
# What subtitle files should we expect?
lang = []
#Check what the Dutch subtitle would be.
if autosub.SUBNL != "":
srtfilenl = os.path.splitext(
filename)[0] + u"." + autosub.SUBNL + u".srt"
else:
srtfilenl = os.path.splitext(filename)[0] + u".srt"
#Check what the English subtitle would be.
if autosub.SUBENG == "":
# Check for overlapping names
if autosub.SUBNL != "" or not autosub.DOWNLOADDUTCH:
srtfileeng = os.path.splitext(filename)[0] + u".srt"
# Hardcoded fallback
else:
srtfileeng = os.path.splitext(filename)[0] + u".en.srt"
else:
srtfileeng = os.path.splitext(
filename)[0] + u"." + autosub.SUBENG + u".srt"
# Check which languages we want to download based on user settings and check the skipstring
if autosub.DOWNLOADDUTCH:
Skipped = False
for SkipItem in SkipListNL:
if re.search(SkipItem.lower(), filename.lower()):
Skipped = True
log.debug(
"scanDir: %s found in %s so skipped for Dutch subs"
% (SkipItem, filename))
break
if not Skipped and os.path.exists(
os.path.join(dirname, srtfilenl)):
Skipped = True
log.debug(
"scanDir: %s skipped because the subtitle already exists"
% filename)
# If the Dutch subtitle doesn't exist en has no skip item, then add it to the wanted list.
if not Skipped:
lang.append(autosub.DUTCH)
if autosub.DOWNLOADENG:
Skipped = False
for SkipItem in SkipListEN:
if re.search(SkipItem.lower(), filename.lower()):
Skipped = True
log.debug(
"scanDir: %s found in %s so skipped for English subs"
% (SkipItem, filename))
break
if not Skipped and os.path.exists(
os.path.join(dirname, srtfileeng)):
Skipped = True
log.debug(
"scanDir: %s skipped because the subtitle already exists"
% filename)
# If the English subtitle doesn't exist en has no skip item, then add it to the wanted list.
if not Skipped:
lang.append(autosub.ENGLISH)
if (autosub.FALLBACKTOENG and autosub.DOWNLOADDUTCH
) and not autosub.DOWNLOADENG and not Skipped:
# If the Dutch and English subtitles do not exist and not skipped, then add English to the wanted list.
if not os.path.exists(os.path.join(
dirname, srtfilenl)) and not os.path.exists(
os.path.join(dirname, srtfileeng)):
lang.append(autosub.ENGLISH)
if not lang:
# autosub.WANTEDQUEUE empty
continue
log.debug("scanDir: File %s is missing subtitle(s): %s" %
(filename, ', '.join(map(str, lang))))
filenameResults = ProcessFilename(
os.path.splitext(filename)[0],
os.path.splitext(filename)[1])
if 'title' in filenameResults.keys():
if 'season' in filenameResults.keys():
if 'episode' in filenameResults.keys():
title = filenameResults['title']
season = filenameResults['season']
episode = filenameResults['episode']
if not filenameResults[
'releasegrp'] and not filenameResults[
'source'] and not filenameResults[
'quality'] and not filenameResults[
'source']:
continue
if autosub.Helpers.SkipShow(
title, season, episode) == True:
log.debug("scanDir: SkipShow returned True")
log.info(
"scanDir: Skipping %s - Season %s Episode %s"
% (title, season, episode))
continue
if len(lang) == 1:
log.info(
"scanDir: %s subtitle wanted for %s and added to wantedQueue"
% (lang[0], filename))
else:
log.info(
"scanDir: %s subtitles wanted for %s and added to wantedQueue"
% (' and '.join(map(str, lang)), filename))
filenameResults[
'originalFileLocationOnDisk'] = os.path.join(
dirname, filename)
filenameResults['timestamp'] = unicode(
time.strftime(
'%Y-%m-%d %H:%M:%S',
time.localtime(
os.path.getctime(filenameResults[
'originalFileLocationOnDisk']))))
filenameResults['lang'] = lang
autosub.WANTEDQUEUE.append(filenameResults)
else:
log.error(
"scanDir: Could not process the filename properly filename: %s"
% filename)
continue
else:
log.error(
"scanDir: Could not process the filename properly filename: %s"
% filename)
continue
else:
log.error(
"scanDir: Could not process the filename properly filename: %s"
% filename)
continue
def run(self):
log.debug("checkRSS: Starting round of RSS checking")
if autosub.WANTEDQUEUELOCK:
log.debug("checkRSS: Exiting, another threat is using the queues")
return False
else:
autosub.WANTEDQUEUELOCK = True
toDelete_wantedQueue = []
langs = ["nl"]
# default is to only check for Dutch subs
# but if English should be downloaden, check them too
# It is very important that the dutch language is run first!
if autosub.FALLBACKTOENG or autosub.DOWNLOADENG:
langs.append("en")
log.debug("checkRSS: We also want to check the English RSS feed")
for lang in langs:
if lang == "en":
RSSURL = autosub.ENRSSURL
log.debug("checkRSS: Now using the English RSS feed")
else:
RSSURL = autosub.NLRSSURL
log.debug("checkRSS: Now using the Dutch RSS feed")
try:
bierdopjeapi = API(RSSURL,RSS=True)
soup = BeautifulStoneSoup(bierdopjeapi.resp.read())
log.debug("checkRss: Succussfully connected to %s" %RSSURL)
bierdopjeapi.close()
except:
log.error("checkRss: The server returned an error for request %s" % RSSURL)
autosub.WANTEDQUEUELOCK = False
continue
# Parse all the item-tags from the RSSFeed
# The information that is parsed is: title, link and show_id
# The show_id is later used to match with the wanted items
# The title is used the determine the quality / source / releasegrp
rssItemList = []
items = soup.findAll('item')
if not len(items) > 0:
log.error("checkRss: invalid RssFeed")
log.debug("checkRss: dumping Rssfeed %s" %(soup.prettify()))
else:
log.debug("checkRss: Valid RssFeed")
for x in items:
soupx = BeautifulStoneSoup(str(x))
title = soupx.find('title').string
show_id = soupx.find('show_id').string
link = soupx.find('enclosure')['url']
item = {}
item['title'] = title
item['link'] = link
item['show_id'] = show_id
log.debug("checkRSS: Parsed from RSSFEED: %s %s %s" %(title,link,show_id))
rssItemList.append(item)
normalizedRssTitleList = []
# Now we create a new rsslist, containing information like: episode, season, etc
for item in rssItemList:
title = item['title']
link = item['link']
show_id = item['show_id']
log.debug("checkRSS: Normalizing the following entry in the RSS results: %s" % title)
normalizedRssTitle = ProcessFilename(title, '')
normalizedRssTitle['rssfile'] = title
normalizedRssTitle['link'] = link
normalizedRssTitle['show_id'] = str(show_id)
if 'title' in normalizedRssTitle.keys():
if 'season' in normalizedRssTitle.keys():
if 'episode' in normalizedRssTitle.keys():
normalizedRssTitleList.append(normalizedRssTitle)
#check versus wantedItem list
for index, wantedItem in enumerate(autosub.WANTEDQUEUE):
wantedItemquality = None
wantedItemreleasegrp = None
wantedItemsource = None
wantedItemcodec = None
wantedItemtitle = wantedItem['title']
wantedItemseason = wantedItem['season']
wantedItemepisode = wantedItem['episode']
originalfile = wantedItem['originalFileLocationOnDisk']
if lang not in wantedItem['lang']:
continue
if 'quality' in wantedItem.keys(): wantedItemquality = wantedItem['quality']
if 'releasegrp' in wantedItem.keys(): wantedItemreleasegrp = wantedItem['releasegrp']
if 'source' in wantedItem.keys(): wantedItemsource = wantedItem['source']
if 'codec' in wantedItem.keys(): wantedItemcodec = wantedItem['codec']
#lets try to find a showid
showid = autosub.Helpers.getShowid(wantedItemtitle)
#no showid? skip this item
if not showid:
continue
for normalizedRssTitle in normalizedRssTitleList:
downloadLink = None
normalizedRssTitleseason = normalizedRssTitle['season']
normalizedRssTitleepisode = normalizedRssTitle['episode']
normalizedRssTitlerssfile = normalizedRssTitle['rssfile']
normalizedRssTitleshowid = int(normalizedRssTitle['show_id'])
normalizedRssTitlelink = normalizedRssTitle['link']
log.debug("checkRSS: Trying to match ID:%r S:%r E:%r (wantedlist) with ID:%r S:%r E:%r (rss)" %(showid, wantedItemseason, wantedItemepisode, normalizedRssTitleshowid, normalizedRssTitleseason, normalizedRssTitleepisode))
if showid == normalizedRssTitleshowid and wantedItemseason == normalizedRssTitleseason and wantedItemepisode == normalizedRssTitleepisode:
log.debug("checkRSS: The episode %s - Season %s Episode %s was found in the RSS list, attempting to match a proper match" % (wantedItemtitle, wantedItemseason, wantedItemepisode))
score = autosub.Helpers.scoreMatch(normalizedRssTitle, normalizedRssTitlerssfile, wantedItemquality, wantedItemreleasegrp, wantedItemsource, wantedItemcodec)
if score >= autosub.MINMATCHSCORERSS:
log.debug ("checkRss: A match got a high enough score. MinMatchscore is %s " % autosub.MINMATCHSCORERSS)
downloadLink = normalizedRssTitlelink + autosub.APIRSS
log.info ("checkRss: Got a match, matching file is: %s" %normalizedRssTitlerssfile)
log.debug("checkRss: Dumping downloadlink for debug purpose: %s" %downloadLink)
if downloadLink:
originalfile = wantedItem['originalFileLocationOnDisk']
# Dutch subs
if autosub.SUBNL != "" and lang == "nl":
srtfile = os.path.splitext(originalfile)[0] + u"." + autosub.SUBNL + u".srt"
elif lang == "nl":
srtfile = os.path.splitext(originalfile)[0] + u".srt"
# English subs
if autosub.SUBENG != "" and lang == "en":
srtfile = os.path.splitext(originalfile)[0] + u"." + autosub.SUBENG + u".srt"
elif lang == "en":
srtfile = os.path.splitext(originalfile)[0] + u".srt"
wantedItem['downloadLink'] = downloadLink
wantedItem['destinationFileLocationOnDisk'] = srtfile
log.info("checkRSS: The episode %s - Season %s Episode %s has a matching subtitle on the RSSFeed, downloading it!" % (wantedItemtitle, wantedItemseason, wantedItemepisode))
downloadItem = wantedItem.copy()
downloadItem['downlang'] = lang
downloadItem['subtitle'] = normalizedRssTitlerssfile
if not DownloadSub(downloadItem):
break
if lang == 'nl' and (autosub.FALLBACKTOENG and not autosub.DOWNLOADENG) and 'en' in wantedItem['lang']:
log.debug('checkRss: We found a dutch subtitle and fallback is true. Removing the english subtitle from the wantedlist.')
wantedItem['lang'].remove('en')
wantedItem['lang'].remove(lang)
if len(wantedItem['lang']) == 0:
toDelete_wantedQueue.append(index)
break
else:
log.debug("checkRss: Matching score is not high enough. Score is %s should be %s" %(str(score),autosub.MINMATCHSCORERSS))
i = len(toDelete_wantedQueue)-1
while i >= 0:
log.debug("checkRSS: Removed item from the wantedQueue at index %s" % toDelete_wantedQueue[i])
autosub.WANTEDQUEUE.pop(toDelete_wantedQueue[i])
i = i-1
# Resetting the toDelete queue for the next run (if need)
toDelete_wantedQueue =[]
log.debug("checkRSS: Finished round of RSS checking")
autosub.WANTEDQUEUELOCK = False
return True
def walkDir(path):
SkipListNL = []
SkipListEN = []
SkipListNL = autosub.SKIPSTRINGNL.split(",")
SkipListEN = autosub.SKIPSTRINGEN.split(",")
# Here we use os.walk to find all files in the path.
for dirname, dirnames, filenames in os.walk(path):
log.debug("scanDisk: directory name: %s" %dirname)
if re.search('_unpack_', dirname, re.IGNORECASE):
log.debug("scanDisk: found a unpack directory, skipping")
continue
if autosub.SKIPHIDDENDIRS and os.path.split(dirname)[1].startswith(u'.'):
continue
if re.search('_failed_', dirname, re.IGNORECASE):
log.debug("scanDisk: found a failed directory, skipping")
continue
if re.search('@eaDir', dirname, re.IGNORECASE):
log.debug("scanDisk: found a Synology indexing directory, skipping this folder and all subfolders.")
tmpdirs = dirnames[:]
for dir in tmpdirs:
dirnames.remove(dir)
continue
if re.search("@.*thumb", dirname, re.IGNORECASE):
# Ingnore QNAP multimedia thumbnail folders
continue
for filename in filenames:
splitname = filename.split(".")
ext = splitname[len(splitname) - 1]
if ext in ('avi', 'mkv', 'wmv', 'ts', 'mp4'):
if re.search('sample', filename): continue
if not platform.system() == 'Windows':
# Get best ascii compatible character for special characters
try:
if not isinstance(filename, unicode):
coding = detect(filename)
filename = unicode(filename.decode(coding))
correctedFilename = ''.join((c for c in unicodedata.normalize('NFD', filename) if unicodedata.category(c) != 'Mn'))
if filename != correctedFilename:
os.rename(os.path.join(dirname, filename), os.path.join(dirname, correctedFilename))
log.info("scanDir: Renamed file %s" % correctedFilename)
filename = correctedFilename
except:
log.error("scanDir: Skipping directory %s" % dirname)
log.error("scanDir: Skipping file %s" % filename)
continue
# What subtitle files should we expect?
lang=[]
#Check what the Dutch subtitle would be.
if autosub.SUBNL != "":
srtfilenl = os.path.splitext(filename)[0] + u"." + autosub.SUBNL + u".srt"
else:
srtfilenl = os.path.splitext(filename)[0] + u".srt"
#Check what the English subtitle would be.
if autosub.SUBENG == "":
# Check for overlapping names
if autosub.SUBNL != "" or not autosub.DOWNLOADDUTCH:
srtfileeng = os.path.splitext(filename)[0] + u".srt"
# Hardcoded fallback
else:
srtfileeng = os.path.splitext(filename)[0] + u".en.srt"
else:
srtfileeng = os.path.splitext(filename)[0] + u"." + autosub.SUBENG + u".srt"
# Check which languages we want to download based on user settings and check the skipstring
if autosub.DOWNLOADDUTCH:
Skipped = False
for SkipItem in SkipListNL:
if re.search(SkipItem.lower(), filename.lower()):
Skipped = True
log.debug("scanDir: %s found in %s so skipped for Dutch subs" % (SkipItem, filename))
break
if not Skipped and os.path.exists(os.path.join(dirname, srtfilenl)):
Skipped = True
log.debug("scanDir: %s skipped because the subtitle already exists" % filename)
# If the Dutch subtitle doesn't exist en has no skip item, then add it to the wanted list.
if not Skipped:
lang.append(autosub.DUTCH)
if autosub.DOWNLOADENG:
Skipped = False
for SkipItem in SkipListEN:
if re.search(SkipItem.lower(), filename.lower()):
Skipped = True
log.debug("scanDir: %s found in %s so skipped for English subs" % (SkipItem, filename))
break
if not Skipped and os.path.exists(os.path.join(dirname, srtfileeng)):
Skipped = True
log.debug("scanDir: %s skipped because the subtitle already exists" % filename)
# If the English subtitle doesn't exist en has no skip item, then add it to the wanted list.
if not Skipped:
lang.append(autosub.ENGLISH)
if (autosub.FALLBACKTOENG and autosub.DOWNLOADDUTCH) and not autosub.DOWNLOADENG and not Skipped:
# If the Dutch and English subtitles do not exist and not skipped, then add English to the wanted list.
if not os.path.exists(os.path.join(dirname, srtfilenl)) and not os.path.exists(os.path.join(dirname, srtfileeng)):
lang.append(autosub.ENGLISH)
if not lang:
# autosub.WANTEDQUEUE empty
continue
log.debug("scanDir: File %s is missing subtitle(s): %s" % (filename, ', '.join(map(str,lang))))
filenameResults = ProcessFilename(os.path.splitext(filename)[0], os.path.splitext(filename)[1])
if 'title' in filenameResults.keys():
if 'season' in filenameResults.keys():
if 'episode' in filenameResults.keys():
title = filenameResults['title']
season = filenameResults['season']
episode = filenameResults['episode']
if not filenameResults['releasegrp'] and not filenameResults['source'] and not filenameResults['quality'] and not filenameResults['source']:
continue
if autosub.Helpers.SkipShow(title, season, episode) == True:
log.debug("scanDir: SkipShow returned True")
log.info("scanDir: Skipping %s - Season %s Episode %s" % (title, season, episode))
continue
if len(lang) == 1:
log.info("scanDir: %s subtitle wanted for %s and added to wantedQueue" % (lang[0], filename))
else:
log.info("scanDir: %s subtitles wanted for %s and added to wantedQueue" % (' and '.join(map(str,lang)), filename))
filenameResults['originalFileLocationOnDisk'] = os.path.join(dirname, filename)
filenameResults['timestamp'] = unicode(time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(os.path.getctime(filenameResults['originalFileLocationOnDisk']))))
filenameResults['lang'] = lang
autosub.WANTEDQUEUE.append(filenameResults)
else:
log.error("scanDir: Could not process the filename properly filename: %s" % filename)
continue
else:
log.error("scanDir: Could not process the filename properly filename: %s" % filename)
continue
else:
log.error("scanDir: Could not process the filename properly filename: %s" % filename)
continue
def walkDir(path):
for dirname, dirnames, filenames in os.walk(os.path.join(path)):
log.debug("scanDisk: directory name: %s" %dirname)
if re.search('_unpack_', dirname, re.IGNORECASE):
log.debug("scanDisk: found a unpack directory, skipping")
continue
if autosub.SKIPHIDDENDIRS and os.path.split(dirname)[1].startswith(u'.'):
continue
if re.search('_failed_', dirname, re.IGNORECASE):
log.debug("scanDisk: found a failed directory, skipping")
continue
for filename in filenames:
splitname = filename.split(".")
ext = splitname[len(splitname) - 1]
if ext in ('avi', 'mkv', 'wmv', 'ts', 'mp4'):
if re.search('sample', filename): continue
# What subtitle files should we expect?
if autosub.SUBNL != "":
srtfile = os.path.splitext(filename)[0] + u"." + autosub.SUBNL + u".srt"
else:
srtfile = os.path.splitext(filename)[0] + u".srt"
srtfileeng = os.path.splitext(filename)[0] + u"." + autosub.SUBENG + u".srt"
if not os.path.exists(os.path.join(dirname, srtfile)) or (not os.path.exists(os.path.join(dirname, srtfileeng)) and autosub.DOWNLOADENG):
log.debug("scanDir: File %s is missing a subtitle" % filename)
lang = []
filenameResults = ProcessFilename(os.path.splitext(filename)[0], os.path.splitext(filename)[1])
if 'title' in filenameResults.keys():
if 'season' in filenameResults.keys():
if 'episode' in filenameResults.keys():
title = filenameResults['title']
season = filenameResults['season']
episode = filenameResults['episode']
if autosub.Helpers.SkipShow(title, season, episode) == True:
log.debug("scanDir: SkipShow returned True")
log.info("scanDir: Skipping %s - Season %s Episode %s" % (title, season, episode))
continue
log.info("scanDir: Dutch subtitle wanted for %s and added to wantedQueue" % filename)
filenameResults['originalFileLocationOnDisk'] = os.path.join(dirname, filename)
filenameResults['timestamp'] = unicode(time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(os.path.getctime(filenameResults['originalFileLocationOnDisk']))))
if not os.path.exists(os.path.join(dirname, srtfile)):
lang.append('nl')
if not os.path.exists(os.path.join(dirname, srtfileeng)) and (autosub.FALLBACKTOENG or autosub.DOWNLOADENG):
lang.append('en')
filenameResults['lang'] = lang
autosub.WANTEDQUEUE.append(filenameResults)
else:
log.error("scanDir: Could not process the filename properly filename: %s" % filename)
continue
else:
log.error("scanDir: Could not process the filename properly filename: %s" % filename)
continue
else:
log.error("scanDir: Could not process the filename properly filename: %s" % filename)
continue
def run(self):
log.debug("checkRSS: Starting round of RSS checking")
if autosub.WANTEDQUEUELOCK:
log.debug("checkRSS: Exiting, another threat is using the queues")
return False
else:
autosub.WANTEDQUEUELOCK = True
toDelete_wantedQueue = []
langs = ["nl"]
# default is to only check for Dutch subs
# but if English should be downloaden, check them too
# It is very important that the dutch language is run first!
if autosub.FALLBACKTOENG or autosub.DOWNLOADENG:
langs.append("en")
log.debug("checkRSS: We also want to check the English RSS feed")
for lang in langs:
if lang == "en":
RSSURL = autosub.ENRSSURL
log.debug("checkRSS: Now using the English RSS feed")
else:
RSSURL = autosub.NLRSSURL
log.debug("checkRSS: Now using the Dutch RSS feed")
try:
bierdopjeapi = API(RSSURL, RSS=True)
soup = BeautifulStoneSoup(bierdopjeapi.resp.read())
log.debug("checkRss: Succussfully connected to %s" % RSSURL)
bierdopjeapi.close()
except:
log.error(
"checkRss: The server returned an error for request %s" %
RSSURL)
autosub.WANTEDQUEUELOCK = False
continue
# Parse all the item-tags from the RSSFeed
# The information that is parsed is: title, link and show_id
# The show_id is later used to match with the wanted items
# The title is used the determine the quality / source / releasegrp
rssItemList = []
items = soup.findAll('item')
if not len(items) > 0:
log.error("checkRss: invalid RssFeed")
log.debug("checkRss: dumping Rssfeed %s" % (soup.prettify()))
else:
log.debug("checkRss: Valid RssFeed")
for x in items:
soupx = BeautifulStoneSoup(str(x))
title = soupx.find('title').string
show_id = soupx.find('show_id').string
link = soupx.find('enclosure')['url']
item = {}
item['title'] = title
item['link'] = link
item['show_id'] = show_id
log.debug("checkRSS: Parsed from RSSFEED: %s %s %s" %
(title, link, show_id))
rssItemList.append(item)
normalizedRssTitleList = []
# Now we create a new rsslist, containing information like: episode, season, etc
for item in rssItemList:
title = item['title']
link = item['link']
show_id = item['show_id']
log.debug(
"checkRSS: Normalizing the following entry in the RSS results: %s"
% title)
normalizedRssTitle = ProcessFilename(title, '')
normalizedRssTitle['rssfile'] = title
normalizedRssTitle['link'] = link
normalizedRssTitle['show_id'] = str(show_id)
if 'title' in normalizedRssTitle.keys():
if 'season' in normalizedRssTitle.keys():
if 'episode' in normalizedRssTitle.keys():
normalizedRssTitleList.append(normalizedRssTitle)
#check versus wantedItem list
for index, wantedItem in enumerate(autosub.WANTEDQUEUE):
wantedItemquality = None
wantedItemreleasegrp = None
wantedItemsource = None
wantedItemcodec = None
wantedItemtitle = wantedItem['title']
wantedItemseason = wantedItem['season']
wantedItemepisode = wantedItem['episode']
originalfile = wantedItem['originalFileLocationOnDisk']
if lang not in wantedItem['lang']:
continue
if 'quality' in wantedItem.keys():
wantedItemquality = wantedItem['quality']
if 'releasegrp' in wantedItem.keys():
wantedItemreleasegrp = wantedItem['releasegrp']
if 'source' in wantedItem.keys():
wantedItemsource = wantedItem['source']
if 'codec' in wantedItem.keys():
wantedItemcodec = wantedItem['codec']
#lets try to find a showid
showid = autosub.Helpers.getShowid(wantedItemtitle)
#no showid? skip this item
if not showid:
continue
for normalizedRssTitle in normalizedRssTitleList:
downloadLink = None
normalizedRssTitleseason = normalizedRssTitle['season']
normalizedRssTitleepisode = normalizedRssTitle['episode']
normalizedRssTitlerssfile = normalizedRssTitle['rssfile']
normalizedRssTitleshowid = int(
normalizedRssTitle['show_id'])
normalizedRssTitlelink = normalizedRssTitle['link']
log.debug(
"checkRSS: Trying to match ID:%r S:%r E:%r (wantedlist) with ID:%r S:%r E:%r (rss)"
% (showid, wantedItemseason, wantedItemepisode,
normalizedRssTitleshowid, normalizedRssTitleseason,
normalizedRssTitleepisode))
if showid == normalizedRssTitleshowid and wantedItemseason == normalizedRssTitleseason and wantedItemepisode == normalizedRssTitleepisode:
log.debug(
"checkRSS: The episode %s - Season %s Episode %s was found in the RSS list, attempting to match a proper match"
% (wantedItemtitle, wantedItemseason,
wantedItemepisode))
score = autosub.Helpers.scoreMatch(
normalizedRssTitle, normalizedRssTitlerssfile,
wantedItemquality, wantedItemreleasegrp,
wantedItemsource, wantedItemcodec)
if score >= autosub.MINMATCHSCORERSS:
log.debug(
"checkRss: A match got a high enough score. MinMatchscore is %s "
% autosub.MINMATCHSCORERSS)
downloadLink = normalizedRssTitlelink + autosub.APIRSS
log.info(
"checkRss: Got a match, matching file is: %s" %
normalizedRssTitlerssfile)
log.debug(
"checkRss: Dumping downloadlink for debug purpose: %s"
% downloadLink)
if downloadLink:
originalfile = wantedItem[
'originalFileLocationOnDisk']
# Dutch subs
if autosub.SUBNL != "" and lang == "nl":
srtfile = os.path.splitext(originalfile)[
0] + u"." + autosub.SUBNL + u".srt"
elif lang == "nl":
srtfile = os.path.splitext(
originalfile)[0] + u".srt"
# English subs
if autosub.SUBENG != "" and lang == "en":
srtfile = os.path.splitext(originalfile)[
0] + u"." + autosub.SUBENG + u".srt"
elif lang == "en":
srtfile = os.path.splitext(
originalfile)[0] + u".srt"
wantedItem['downloadLink'] = downloadLink
wantedItem[
'destinationFileLocationOnDisk'] = srtfile
log.info(
"checkRSS: The episode %s - Season %s Episode %s has a matching subtitle on the RSSFeed, downloading it!"
% (wantedItemtitle, wantedItemseason,
wantedItemepisode))
downloadItem = wantedItem.copy()
downloadItem['downlang'] = lang
downloadItem[
'subtitle'] = normalizedRssTitlerssfile
if not DownloadSub(downloadItem):
break
if lang == 'nl' and (
autosub.FALLBACKTOENG
and not autosub.DOWNLOADENG
) and 'en' in wantedItem['lang']:
log.debug(
'checkRss: We found a dutch subtitle and fallback is true. Removing the english subtitle from the wantedlist.'
)
wantedItem['lang'].remove('en')
wantedItem['lang'].remove(lang)
if len(wantedItem['lang']) == 0:
toDelete_wantedQueue.append(index)
break
else:
log.debug(
"checkRss: Matching score is not high enough. Score is %s should be %s"
% (str(score), autosub.MINMATCHSCORERSS))
i = len(toDelete_wantedQueue) - 1
while i >= 0:
log.debug(
"checkRSS: Removed item from the wantedQueue at index %s" %
toDelete_wantedQueue[i])
autosub.WANTEDQUEUE.pop(toDelete_wantedQueue[i])
i = i - 1
# Resetting the toDelete queue for the next run (if need)
toDelete_wantedQueue = []
log.debug("checkRSS: Finished round of RSS checking")
autosub.WANTEDQUEUELOCK = False
return True
def walkDir(path):
SkipListNL = autosub.SKIPSTRINGNL.split(",") if len(
autosub.SKIPSTRINGNL) > 0 else []
SkipListEN = autosub.SKIPSTRINGEN.split(",") if len(
autosub.SKIPSTRINGEN) > 0 else []
if len(autosub.SKIPFOLDERSNL) == 0:
SkipFoldersNL = []
else:
SkipFoldersNL = autosub.SKIPFOLDERSNL.split(",") if len(
autosub.SKIPFOLDERSNL) > 0 else []
for idx, folder in enumerate(SkipFoldersNL):
SkipFoldersNL[idx] = os.path.normpath(
os.path.join(path,
folder.rstrip("\/").lstrip("\/")))
if len(autosub.SKIPFOLDERSNL) == 0:
SkipFoldersNL = []
else:
SkipFoldersNL = autosub.SKIPFOLDERSNL.split(",") if len(
autosub.SKIPFOLDERSNL) > 0 else []
for idx, folder in enumerate(SkipFoldersNL):
SkipFoldersNL[idx] = os.path.normpath(path + folder)
if len(autosub.SKIPFOLDERSEN) == 0:
SkipFoldersEN = []
else:
SkipFoldersEN = autosub.SKIPFOLDERSEN.split(",") if len(
autosub.SKIPFOLDERSEN) > 0 else []
for idx, folder in enumerate(SkipFoldersEN):
SkipFoldersEN[idx] = os.path.normpath(path + folder)
for dirname, dirnames, filenames in os.walk(path, True, WalkError):
SkipThisFolderNL = False
for skip in SkipFoldersNL:
if dirname.startswith(skip):
SkipThisFolderNL = True
break
SkipThisFolderEN = False
for skip in SkipFoldersEN:
if dirname.startswith(skip):
SkipThisFolderEN = True
break
log.debug("scanDisk: directory name: %s" % dirname)
if re.search('_unpack_', dirname, re.IGNORECASE):
log.debug("scanDisk: found a unpack directory, skipping.")
continue
if autosub.SKIPHIDDENDIRS and os.path.split(dirname)[1].startswith(
u'.'):
continue
if re.search('_failed_', dirname, re.IGNORECASE):
log.debug("scanDisk: found a failed directory, skipping.")
continue
if re.search('@eaDir', dirname, re.IGNORECASE):
log.debug(
"scanDisk: found a Synology indexing directory, skipping.")
tmpdirs = dirnames[:]
for dir in tmpdirs:
dirnames.remove(dir)
continue
if re.search("@.*thumb", dirname, re.IGNORECASE):
log.debug(
"scanDisk: found a Qnap multimedia thumbnail folder, skipping."
)
continue
langs = []
FileDict = {}
for filename in filenames:
root, ext = os.path.splitext(filename)
if ext[1:] in ('avi', 'mkv', 'wmv', 'ts', 'mp4'):
if re.search('sample', filename):
continue
if not platform.system() == 'Windows':
# Get best ascii compatible character for special characters
try:
if not isinstance(filename, unicode):
coding = detect(filename)['encoding']
filename = unicode(filename.decode(coding),
errors='replace')
correctedFilename = ''.join(
(c for c in unicodedata.normalize('NFD', filename)
if unicodedata.category(c) != 'Mn'))
if filename != correctedFilename:
os.rename(os.path.join(dirname, filename),
os.path.join(dirname, correctedFilename))
log.info("scanDir: Renamed file %s" %
correctedFilename)
filename = correctedFilename
except:
log.error("scanDir: Skipping directory, file %s, %s" %
(dirname, filename))
continue
# What subtitle files should we expect?
langs = []
NLext = u'.' + autosub.SUBNL + u'.srt' if autosub.SUBNL else u'.srt'
ENext = u'.' + autosub.SUBENG + u'.srt' if autosub.SUBENG else u'.srt'
ENext = u'.en.srt' if NLext == ENext and autosub.DOWNLOADDUTCH else ENext
if not os.access(dirname, os.W_OK):
log.error('scandisk: No write access to folder: %s' %
dirname)
continue
# Check which languages we want to download based on user settings.
if autosub.DOWNLOADDUTCH and not SkipThisFolderNL:
Skipped = False
for SkipItem in SkipListNL:
if not SkipItem: break
if re.search(SkipItem.lower(), filename.lower()):
Skipped = True
break
if Skipped:
log.info(
"scanDir: %s found in %s so skipped for Dutch subs"
% (SkipItem, filename))
elif os.path.exists(os.path.join(dirname, root + NLext)):
Skipped = True
log.debug(
"scanDir: %s skipped because the Dutch subtitle already exists"
% filename)
else:
# If the Dutch subtitle not skipped and doesn't exist, then add it to the wanted list
langs.append(autosub.DUTCH)
if (autosub.DOWNLOADENG or
(autosub.FALLBACKTOENG and autosub.DOWNLOADDUTCH
and not Skipped)) and not SkipThisFolderEN:
Skipped = False
for SkipItem in SkipListEN:
if not SkipItem: break
if re.search(SkipItem.lower(), filename.lower()):
Skipped = True
break
if Skipped:
log.info(
"scanDir: %s found in %s so skipped for English subs"
% (SkipItem, filename))
elif os.path.exists(os.path.join(dirname, root + ENext)):
log.debug(
"scanDir: %s skipped because the English subtitle already exists"
% filename)
else:
# If the English subtitle not skipped and doesn't exist, then add it to the wanted list
if not os.path.exists(
os.path.join(dirname, root + ENext)):
langs.append(autosub.ENGLISH)
if not langs:
# nothing to do for this file
continue
FileDict = ProcessFilename(
os.path.splitext(filename)[0].strip(), ext)
time.sleep(0)
if not FileDict:
continue
if not 'title' in FileDict.keys(
) or not 'season' in FileDict.keys(
) or not 'episode' in FileDict.keys():
continue
if not FileDict['releasegrp'] and not FileDict[
'source'] and not FileDict['quality'] and not FileDict[
'source']:
log.error("scanDir: Not enough info in filename: %s" %
filename)
continue
FileDict['timestamp'] = unicode(
time.strftime(
'%Y-%m-%d %H:%M:%S',
time.localtime(
os.path.getctime(os.path.join(dirname,
filename)))))
FileDict['langs'] = langs
FileDict['NLext'] = NLext
FileDict['ENext'] = ENext
FileDict['file'] = root
FileDict['container'] = ext
FileDict['folder'] = dirname
FileDict['ImdbId'], FileDict['A7Id'], FileDict[
'TvdbId'], FileDict['title'] = Helpers.getShowid(
FileDict['title'], autosub.ADDIC7EDLOGGED_IN)
if autosub.Helpers.SkipShow(FileDict['ImdbId'],
FileDict['title'],
FileDict['season'],
FileDict['episode']):
continue
log.info("scanDir: %s WANTED FOR: %s" % (langs, filename))
autosub.WANTEDQUEUE.append(FileDict)
return
def walkDir(path):
for dirname, dirnames, filenames in os.walk(path):
log.debug("scanDisk: directory name: %s" % dirname)
if re.search("_unpack_", dirname, re.IGNORECASE):
log.debug("scanDisk: found a unpack directory, skipping")
continue
if autosub.SKIPHIDDENDIRS and os.path.split(dirname)[1].startswith(u"."):
continue
if re.search("_failed_", dirname, re.IGNORECASE):
log.debug("scanDisk: found a failed directory, skipping")
continue
if re.search("@eaDir", dirname, re.IGNORECASE):
log.debug("scanDisk: found a Synology indexing directory, skipping this folder and all subfolders.")
tmpdirs = dirnames[:]
for dir in tmpdirs:
dirnames.remove(dir)
continue
if re.search("@.*thumb", dirname, re.IGNORECASE):
# Ingnore QNAP multimedia thumbnail folders
continue
for filename in filenames:
splitname = filename.split(".")
ext = splitname[len(splitname) - 1]
if ext in ("avi", "mkv", "wmv", "ts", "mp4"):
if re.search("sample", filename):
continue
if autosub.WEBDL == "None":
if re.search("web-dl", filename.lower()):
log.debug("scanDisk: WEB-DL is set to 'None', skipping %s" % filename)
continue
if not platform.system() == "Windows":
# Get best ascii compatible character for special characters
try:
if not isinstance(filename, unicode):
coding = detect(filename)
filename = unicode(filename.decode(coding))
correctedFilename = "".join(
(c for c in unicodedata.normalize("NFD", filename) if unicodedata.category(c) != "Mn")
)
if filename != correctedFilename:
os.rename(os.path.join(dirname, filename), os.path.join(dirname, correctedFilename))
log.info("scanDir: Renamed file %s" % correctedFilename)
filename = correctedFilename
except:
log.error("scanDir: Skipping directory %s" % dirname)
log.error("scanDir: Skipping file %s" % filename)
continue
# What subtitle files should we expect?
lang = []
# Check what the Dutch subtitle would be.
if autosub.SUBNL != "":
srtfilenl = os.path.splitext(filename)[0] + u"." + autosub.SUBNL + u".srt"
else:
srtfilenl = os.path.splitext(filename)[0] + u".srt"
# Check what the English subtitle would be.
if autosub.SUBENG == "":
# Check for overlapping names
if autosub.SUBNL != "" or not autosub.DOWNLOADDUTCH:
srtfileeng = os.path.splitext(filename)[0] + u".srt"
# Hardcoded fallback
else:
srtfileeng = os.path.splitext(filename)[0] + u".en.srt"
else:
srtfileeng = os.path.splitext(filename)[0] + u"." + autosub.SUBENG + u".srt"
# Check which languages we want to download based on user settings.
if autosub.DOWNLOADDUTCH:
# If the Dutch subtitle doesn't exist, then add it to the wanted list.
if not os.path.exists(os.path.join(dirname, srtfilenl)):
lang.append(autosub.DUTCH)
if autosub.DOWNLOADENG:
# If the English subtitle doesn't exist, then add it to the wanted list.
if not os.path.exists(os.path.join(dirname, srtfileeng)):
if autosub.WEBDL == "DutchOnly" and re.search("web-dl", filename.lower()):
log.debug(
"scanDisk: WEB-DL is set to 'Dutch Only', not adding English as wanted for %s"
% filename
)
else:
lang.append(autosub.ENGLISH)
if (autosub.FALLBACKTOENG and autosub.DOWNLOADDUTCH) and not autosub.DOWNLOADENG:
# If the Dutch and English subtitles do not exist, then add English to the wanted list.
if not os.path.exists(os.path.join(dirname, srtfilenl)) and not os.path.exists(
os.path.join(dirname, srtfileeng)
):
if autosub.WEBDL == "DutchOnly" and re.search("web-dl", filename.lower()):
log.debug(
"scanDisk: WEB-DL is set to 'Dutch Only', not adding English as wanted for %s"
% filename
)
else:
lang.append(autosub.ENGLISH)
if not lang:
# autosub.WANTEDQUEUE empty
continue
log.debug("scanDir: File %s is missing subtitle(s): %s" % (filename, ", ".join(map(str, lang))))
filenameResults = ProcessFilename(os.path.splitext(filename)[0], os.path.splitext(filename)[1])
if "title" in filenameResults.keys():
if "season" in filenameResults.keys():
if "episode" in filenameResults.keys():
title = filenameResults["title"]
season = filenameResults["season"]
episode = filenameResults["episode"]
if (
not filenameResults["releasegrp"]
and not filenameResults["source"]
and not filenameResults["quality"]
and not filenameResults["source"]
):
continue
if autosub.Helpers.SkipShow(title, season, episode) == True:
log.debug("scanDir: SkipShow returned True")
log.info("scanDir: Skipping %s - Season %s Episode %s" % (title, season, episode))
continue
if len(lang) == 1:
log.info(
"scanDir: %s subtitle wanted for %s and added to wantedQueue" % (lang[0], filename)
)
else:
log.info(
"scanDir: %s subtitles wanted for %s and added to wantedQueue"
% (" and ".join(map(str, lang)), filename)
)
filenameResults["originalFileLocationOnDisk"] = os.path.join(dirname, filename)
filenameResults["timestamp"] = unicode(
time.strftime(
"%Y-%m-%d %H:%M:%S",
time.localtime(os.path.getctime(filenameResults["originalFileLocationOnDisk"])),
)
)
filenameResults["lang"] = lang
autosub.WANTEDQUEUE.append(filenameResults)
else:
log.error("scanDir: Could not process the filename properly filename: %s" % filename)
continue
else:
log.error("scanDir: Could not process the filename properly filename: %s" % filename)
continue
else:
log.error("scanDir: Could not process the filename properly filename: %s" % filename)
continue
