def walkDir(path): SkipListNL = autosub.SKIPSTRINGNL.split(",") if len(autosub.SKIPSTRINGNL) > 0 else [] SkipListEN = autosub.SKIPSTRINGEN.split(",") if len(autosub.SKIPSTRINGEN) > 0 else [] if len(autosub.SKIPFOLDERSNL) == 0: SkipFoldersNL = [] else: SkipFoldersNL = autosub.SKIPFOLDERSNL.split(",") if len(autosub.SKIPFOLDERSNL) > 0 else [] for idx,folder in enumerate(SkipFoldersNL): SkipFoldersNL[idx] = os.path.normpath(os.path.join(path,folder.rstrip("\/").lstrip("\/"))) if len(autosub.SKIPFOLDERSNL) == 0: SkipFoldersNL = [] else: SkipFoldersNL = autosub.SKIPFOLDERSNL.split(",") if len(autosub.SKIPFOLDERSNL) > 0 else [] for idx,folder in enumerate(SkipFoldersNL): SkipFoldersNL[idx] = os.path.normpath(path + folder) if len(autosub.SKIPFOLDERSEN) == 0: SkipFoldersEN = [] else: SkipFoldersEN = autosub.SKIPFOLDERSEN.split(",") if len(autosub.SKIPFOLDERSEN) > 0 else [] for idx,folder in enumerate(SkipFoldersEN): SkipFoldersEN[idx] = os.path.normpath(path + folder) for dirname, dirnames, filenames in os.walk(path, True, WalkError): SkipThisFolderNL = False for skip in SkipFoldersNL: if dirname.startswith(skip): SkipThisFolderNL = True break SkipThisFolderEN = False for skip in SkipFoldersEN: if dirname.startswith(skip): SkipThisFolderEN = True break log.debug("scanDisk: directory name: %s" %dirname) if re.search('_unpack_', dirname, re.IGNORECASE): log.debug("scanDisk: found a unpack directory, skipping.") continue if autosub.SKIPHIDDENDIRS and os.path.split(dirname)[1].startswith(u'.'): continue if re.search('_failed_', dirname, re.IGNORECASE): log.debug("scanDisk: found a failed directory, skipping.") continue if re.search('@eaDir', dirname, re.IGNORECASE): log.debug("scanDisk: found a Synology indexing directory, skipping.") tmpdirs = dirnames[:] for dir in tmpdirs: dirnames.remove(dir) continue if re.search("@.*thumb", dirname, re.IGNORECASE): log.debug("scanDisk: found a Qnap multimedia thumbnail folder, skipping.") continue langs = [] FileDict = {} for filename in filenames: root,ext = os.path.splitext(filename) if ext[1:] in ('avi', 'mkv', 'wmv', 'ts', 'mp4'): if re.search('sample', filename): continue if not platform.system() == 'Windows': # Get best ascii compatible character for special characters try: if not isinstance(filename, unicode): coding = detect(filename)['encoding'] filename = unicode(filename.decode(coding),errors='replace') correctedFilename = ''.join((c for c in unicodedata.normalize('NFD', filename) if unicodedata.category(c) != 'Mn')) if filename != correctedFilename: os.rename(os.path.join(dirname, filename), os.path.join(dirname, correctedFilename)) log.info("scanDir: Renamed file %s" % correctedFilename) filename = correctedFilename except: log.error("scanDir: Skipping directory, file %s, %s" % (dirname,filename)) continue # What subtitle files should we expect? langs = [] NLext = u'.' + autosub.SUBNL + u'.srt' if autosub.SUBNL else u'.srt' ENext = u'.' + autosub.SUBENG + u'.srt' if autosub.SUBENG else u'.srt' ENext = u'.en.srt'if NLext == ENext and autosub.DOWNLOADDUTCH else ENext if not os.access(dirname, os.W_OK): log.error('scandisk: No write access to folder: %s' % dirname) continue # Check which languages we want to download based on user settings. if autosub.DOWNLOADDUTCH and not SkipThisFolderNL: Skipped = False for SkipItem in SkipListNL: if not SkipItem: break if re.search(SkipItem.lower(), filename.lower()): Skipped = True break if Skipped: log.info("scanDir: %s found in %s so skipped for Dutch subs" % (SkipItem, filename)) elif os.path.exists(os.path.join(dirname, root + NLext)): Skipped = True log.debug("scanDir: %s skipped because the Dutch subtitle already exists" % filename) else: # If the Dutch subtitle not skipped and doesn't exist, then add it to the wanted list langs.append(autosub.DUTCH) if (autosub.DOWNLOADENG or (autosub.FALLBACKTOENG and autosub.DOWNLOADDUTCH and not Skipped)) and not SkipThisFolderEN: Skipped = False for SkipItem in SkipListEN: if not SkipItem: break if re.search(SkipItem.lower(), filename.lower()): Skipped = True break if Skipped: log.info("scanDir: %s found in %s so skipped for English subs" % (SkipItem, filename)) elif os.path.exists(os.path.join(dirname, root + ENext)): log.debug("scanDir: %s skipped because the English subtitle already exists" % filename) else: # If the English subtitle not skipped and doesn't exist, then add it to the wanted list if not os.path.exists(os.path.join(dirname, root + ENext)): langs.append(autosub.ENGLISH) if not langs: # nothing to do for this file continue FileDict = ProcessFilename(os.path.splitext(filename)[0].strip(), ext) time.sleep(0) if not FileDict: continue if not 'title' in FileDict.keys() or not 'season' in FileDict.keys() or not 'episode' in FileDict.keys(): continue if not FileDict['releasegrp'] and not FileDict['source'] and not FileDict['quality'] and not FileDict['source']: log.error("scanDir: Not enough info in filename: %s" % filename) continue FileDict['timestamp'] = unicode(time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(os.path.getctime(os.path.join(dirname, filename))))) FileDict['langs'] = langs FileDict['NLext'] = NLext FileDict['ENext'] = ENext FileDict['file'] = root FileDict['container'] = ext FileDict['folder'] = dirname FileDict['ImdbId'],FileDict['A7Id'], FileDict['TvdbId'], FileDict['title'] = Helpers.getShowid(FileDict['title'],autosub.ADDIC7EDLOGGED_IN) if autosub.Helpers.SkipShow(FileDict['ImdbId'],FileDict['title'], FileDict['season'], FileDict['episode']): continue log.info("scanDir: %s WANTED FOR: %s" % (langs, filename)) autosub.WANTEDQUEUE.append(FileDict) return
def run(self): log.debug("checkSub: Starting round of checkSub") # First we check if checksub in not still running if autosub.WANTEDQUEUELOCK: log.info("checkSub: Exiting, another threat is using the queues. Will try again in 60 seconds") time.sleep(60) return False else: autosub.WANTEDQUEUELOCK = True autosub.DBCONNECTION = sqlite3.connect(autosub.DBFILE) autosub.DBIDCACHE = idCache() autosub.DBEPISODECACHE = EpisodeIdCache() toDelete_wantedQueue = [] if not Helpers.checkAPICallsTvdb() or not Helpers.checkAPICallsSubSeeker(): log.warning("checkSub: out of api calls") return True # Initiate the Addic7ed API and check the current number of downloads UseAddic= False if autosub.ADDIC7EDUSER and autosub.ADDIC7EDPASSWD and autosub.ADDIC7EDLANG != 'None': try: # Sets autosub.DOWNLOADS_A7 and autosub.DOWNLOADS_A7MAX # and gives a True response if it's ok to download from a7 autosub.ADDIC7EDAPI = autosub.Addic7ed.Addic7edAPI() UseAddic= autosub.ADDIC7EDAPI.checkCurrentDownloads(logout=False) except: log.debug("checkSub: Couldn't connect with Addic7ed.com") # Initiate a session to OpenSubtitles and log in if OpenSubtitles is choosen if autosub.OPENSUBTITLESLANG != 'None' and autosub.OPENSUBTITLESUSER and autosub.OPENSUBTITLESPASSWD: UseOpensubtitles = OpenSubtitlesLogin() else: UseOpensubtitles = False for index, wantedItem in enumerate(autosub.WANTEDQUEUE): title = wantedItem['title'] season = wantedItem['season'] episode = wantedItem['episode'] originalfile = wantedItem['originalFileLocationOnDisk'] languages = wantedItem['lang'] if not Helpers.checkAPICallsTvdb() or not Helpers.checkAPICallsSubSeeker(): #Make sure that we are allow to connect to SubtitleSeeker and TvDB log.warning("checkSub: out of api calls") break if autosub.SUBNL != "": nlsrtfile = os.path.splitext(originalfile)[0] + u"." + autosub.SUBNL + u".srt" else: nlsrtfile = os.path.splitext(originalfile)[0] + u".srt" if autosub.SUBENG == "": # Check for overlapping names if autosub.SUBNL != "" or not autosub.DOWNLOADDUTCH: engsrtfile = os.path.splitext(originalfile)[0] + u".srt" # Hardcoded fallback else: engsrtfile = os.path.splitext(originalfile)[0] + u".en.srt" else: engsrtfile = os.path.splitext(originalfile)[0] + u"." + autosub.SUBENG + u".srt" #lets try to find a showid; no showid? skip this item showid,a7_id, OsId = Helpers.getShowid(title, UseAddic, UseOpensubtitles) if UseOpensubtitles and OsId: EpisodeId = GetEpisodeId(OsId, season, episode) else: EpisodeId = None log.debug("checkSub: ID's - IMDB: %s, Addic7ed: %s, OpenSubtitles: %s" %(showid,a7_id, OsId)) if not showid: continue for lang in languages[:]: downloadItem = wantedItem.copy() downloadItem['downlang'] = lang # Check if Addic7ed download limit has been reached if UseAddic and autosub.DOWNLOADS_A7 >= autosub.DOWNLOADS_A7MAX: UseAddic= False log.debug("checkSub: You have reached your 24h limit of %s Addic7ed downloads!" % autosub.DOWNLOADS_A7MAX) log.debug("checkSub: trying to get a downloadlink for %s, language is %s" % (originalfile, lang)) # get all links higher than the minmatch as input for downloadSub allResults = autosub.getSubLinks.getSubLinks(showid, a7_id, EpisodeId, lang, wantedItem) if not allResults: log.debug("checkSub: no suitable subtitles were found for %s based on your minmatchscore" % downloadItem['originalFileLocationOnDisk']) continue if lang == autosub.DUTCH: downloadItem['destinationFileLocationOnDisk'] = nlsrtfile elif lang == autosub.ENGLISH: downloadItem['destinationFileLocationOnDisk'] = engsrtfile if allResults: log.info("checkSub: The episode %s - Season %s Episode %s has 1 or more matching subtitles on SubtitleSeeker, downloading it!" % (title, season, episode)) log.debug("checkSub: destination filename %s" % downloadItem['destinationFileLocationOnDisk']) if not DownloadSub(allResults, UseAddic, downloadItem): continue #Remove downloaded language languages.remove(lang) if lang == autosub.DUTCH: if (autosub.FALLBACKTOENG and not autosub.DOWNLOADENG) and autosub.ENGLISH in languages: log.debug('checkSub: We found a Dutch subtitle and fallback is true. Removing the English subtitle from the wantedlist.') languages.remove(autosub.ENGLISH) if autosub.ENGLISHSUBDELETE: log.info("checkSub: Clean up English enabled") if os.path.exists(engsrtfile): log.debug("checkSub: Trying to delete English subtitle: %s" % engsrtfile) try: os.unlink(engsrtfile) log.info("checkSub: Removed English subtitle: %s" % engsrtfile) except: log.error("checkSub: Error while trying to remove subtitle %s." % engsrtfile) else: log.info("checkSub: English subtitle not found.") if len(languages) == 0: toDelete_wantedQueue.append(index) break autosub.DBCONNECTION.close() del autosub.DBCONNECTION del autosub.DBIDCACHE del autosub.DBEPISODECACHE if autosub.ADDIC7EDAPI: autosub.ADDIC7EDAPI.logout() if autosub.OPENSUBTITLESLOGGED_IN: OpenSubtitlesLogout() i = len(toDelete_wantedQueue) - 1 while i >= 0: log.debug("checkSub: Removed item from the wantedQueue at index %s" % toDelete_wantedQueue[i]) autosub.WANTEDQUEUE.pop(toDelete_wantedQueue[i]) i = i - 1 log.debug("checkSub: Finished round of checkSub") autosub.WANTEDQUEUELOCK = False return True
def walkDir(path): SkipListNL = autosub.SKIPSTRINGNL.split(",") if len( autosub.SKIPSTRINGNL) > 0 else [] SkipListEN = autosub.SKIPSTRINGEN.split(",") if len( autosub.SKIPSTRINGEN) > 0 else [] # Check for dutch folder skip if len(autosub.SKIPFOLDERSNL) == 0: SkipFoldersNL = [] else: SkipFoldersNL = autosub.SKIPFOLDERSNL.split(",") if len( autosub.SKIPFOLDERSNL) > 0 else [] for idx, folder in enumerate(SkipFoldersNL): SkipFoldersNL[idx] = os.path.normpath( os.path.join(path, folder.strip(" \/"))) # Check for english folder skip if len(autosub.SKIPFOLDERSEN) == 0: SkipFoldersEN = [] else: SkipFoldersEN = autosub.SKIPFOLDERSEN.split(",") if len( autosub.SKIPFOLDERSEN) > 0 else [] for idx, folder in enumerate(SkipFoldersEN): SkipFoldersEN[idx] = os.path.normpath( os.path.join(path, folder.strip(" \/"))) for dirname, dirnames, filenames in os.walk(path, True, WalkError): SkipThisFolderNL = False for skip in SkipFoldersNL: if dirname.startswith(skip): SkipThisFolderNL = True break SkipThisFolderEN = False for skip in SkipFoldersEN: if dirname.startswith(skip): SkipThisFolderEN = True break log.debug("scanDisk: directory name: %s" % dirname) if re.search('_unpack_', dirname, re.IGNORECASE): log.debug("scanDisk: found a unpack directory, skipping.") continue if autosub.SKIPHIDDENDIRS and os.path.split(dirname)[1].startswith( u'.'): continue if re.search('_failed_', dirname, re.IGNORECASE): log.debug("scanDisk: found a failed directory, skipping.") continue if re.search('@eaDir', dirname, re.IGNORECASE): log.debug( "scanDisk: found a Synology indexing directory, skipping.") tmpdirs = dirnames[:] for dir in tmpdirs: dirnames.remove(dir) continue if re.search("@.*thumb", dirname, re.IGNORECASE): log.debug( "scanDisk: found a Qnap multimedia thumbnail folder, skipping." ) continue langs = [] FileDict = {} for filename in filenames: if autosub.SEARCHSTOP: log.info('scanDisk: Forced Stop by user') return try: root, ext = os.path.splitext(filename) if ext[1:] in ('avi', 'mkv', 'wmv', 'ts', 'mp4'): if re.search('sample', filename): continue if not platform.system() == 'Windows': # Get best ascii compatible character for special characters try: if not isinstance(filename, unicode): coding = detect(filename)['encoding'] filename = unicode(filename.decode(coding), errors='replace') correctedFilename = ''.join( (c for c in unicodedata.normalize( 'NFD', filename) if unicodedata.category(c) != 'Mn')) if filename != correctedFilename: os.rename( os.path.join(dirname, filename), os.path.join(dirname, correctedFilename)) log.info("scanDir: Renamed file %s" % correctedFilename) filename = correctedFilename except: log.error( "scanDir: Skipping directory, file %s, %s" % (dirname, filename)) continue # What subtitle files should we expect? langs = [] NLext = u'.' + autosub.SUBNL + u'.srt' if autosub.SUBNL else u'.srt' ENext = u'.' + autosub.SUBENG + u'.srt' if autosub.SUBENG else u'.srt' ENext = u'.en.srt' if NLext == ENext and autosub.DOWNLOADDUTCH else ENext if not os.access(dirname, os.W_OK): log.error('scandisk: No write access to folder: %s' % dirname) continue # Check which languages we want to download based on user settings. log.debug('scanDir: Processing file: %s' % filename) if autosub.DOWNLOADDUTCH and not SkipThisFolderNL: Skipped = False for SkipItem in SkipListNL: if not SkipItem: break if re.search(SkipItem.lower(), filename.lower()): Skipped = True break if Skipped: log.info( "scanDir: %s found in %s so skipped for Dutch subs" % (SkipItem, filename)) elif os.path.exists(os.path.join( dirname, root + NLext)): Skipped = True log.debug( "scanDir: %s skipped because the Dutch subtitle already exists" % filename) else: # If the Dutch subtitle not skipped and doesn't exist, then add it to the wanted list langs.append(autosub.DUTCH) if (autosub.DOWNLOADENG or (autosub.FALLBACKTOENG and autosub.DOWNLOADDUTCH and not Skipped)) and not SkipThisFolderEN: Skipped = False for SkipItem in SkipListEN: if not SkipItem: break if re.search(SkipItem.lower(), filename.lower()): Skipped = True break if Skipped: log.info( "scanDir: %s found in %s so skipped for English subs" % (SkipItem, filename)) elif os.path.exists(os.path.join( dirname, root + ENext)): log.debug( "scanDir: %s skipped because the English subtitle already exists" % filename) else: # If the English subtitle not skipped and doesn't exist, then add it to the wanted list if not os.path.exists( os.path.join(dirname, root + ENext)): langs.append(autosub.ENGLISH) if not langs: # nothing to do for this file continue FileDict = ProcessFilename( os.path.splitext(filename)[0].strip(), ext) if not FileDict: log.debug( 'scanDisk: not enough info in the filename: %s' % filename) continue Skip = False if autosub.MINMATCHSCORE & 8 and not FileDict['source']: Skip = True elif autosub.MINMATCHSCORE & 4 and not FileDict['quality']: Skip = True elif autosub.MINMATCHSCORE & 2 and not FileDict['codec']: Skip = True elif autosub.MINMATCHSCORE & 1 and not FileDict[ 'releasegrp']: Skip = True if Skip: log.debug( 'scanDisk: Filespec does not meet minmatchscore so skipping this one' ) continue FileDict['timestamp'] = unicode( time.strftime( '%Y-%m-%d %H:%M:%S', time.localtime( os.path.getmtime( os.path.join(dirname, filename))))) FileDict['langs'] = langs FileDict['NLext'] = NLext FileDict['ENext'] = ENext FileDict['file'] = root FileDict['container'] = ext FileDict['folder'] = dirname FileDict['ImdbId'], FileDict['A7Id'], FileDict[ 'TvdbId'], FileDict['title'] = Helpers.getShowid( FileDict['title']) if autosub.Helpers.SkipShow(FileDict['ImdbId'], FileDict['title'], FileDict['season'], FileDict['episode']): log.debug("scanDir: SKIPPED %s by Skipshow rules." % FileDict['file']) continue log.info("scanDir: %s WANTED FOR: %s" % (langs, filename)) autosub.WANTEDQUEUE.append(FileDict) time.sleep(0) except Exception as error: log.error('scanDir: Problem scanning file %s. Error is: %s' % (filename, error)) return
def run(self): log.debug("checkSub: Starting round of checkSub") toDelete_wantedQueue = [] if not autosub.Helpers.checkAPICalls(): log.warning("checkSub: out of api calls") return True if autosub.WANTEDQUEUELOCK: log.debug("checkSub: Exiting, another threat is using the queues") return False else: autosub.WANTEDQUEUELOCK = True for index, wantedItem in enumerate(autosub.WANTEDQUEUE): title = wantedItem['title'] season = wantedItem['season'] episode = wantedItem['episode'] originalfile = wantedItem['originalFileLocationOnDisk'] languages = wantedItem['lang'] if not autosub.Helpers.checkAPICalls(): #Make sure that we are allow to connect to bierdopje log.warning("checkSub: out of api calls") break if autosub.SUBNL != "": srtfile = os.path.splitext( originalfile)[0] + u"." + autosub.SUBNL + u".srt" else: srtfile = os.path.splitext(originalfile)[0] + u".srt" engsrtfile = os.path.splitext( originalfile)[0] + u"." + autosub.SUBENG + u".srt" #lets try to find a showid showid = Helpers.getShowid(title) #no showid? skip this item if not showid: continue langtmp = languages[:] for lang in langtmp: log.debug( "checkSub: trying to get a downloadlink for %s, language is %s" % (originalfile, lang)) downloadLink, release = autosub.Bierdopje.getSubLink( showid, lang, wantedItem) if downloadLink: if lang == 'nl': wantedItem['destinationFileLocationOnDisk'] = srtfile elif lang == 'en': wantedItem[ 'destinationFileLocationOnDisk'] = engsrtfile log.info( "checkSub: The episode %s - Season %s Episode %s has a matching subtitle on bierdopje, downloading it!" % (title, season, episode)) log.debug( 'checkSub: Dumping downloadlink for debug perpuse %s' % downloadLink) log.debug("checkSub: destination filename %s" % wantedItem['destinationFileLocationOnDisk']) wantedItem['downloadLink'] = downloadLink downloadItem = wantedItem.copy() downloadItem['downlang'] = lang downloadItem['subtitle'] = release if not DownloadSub(downloadItem): break if lang == 'nl' and ( autosub.FALLBACKTOENG and not autosub.DOWNLOADENG) and 'en' in languages: log.debug( 'checkSub: We found a dutch subtitle and fallback is true. Removing the english subtitle from the wantedlist.' ) languages.remove('en') languages.remove(lang) if len(languages) == 0: toDelete_wantedQueue.append(index) break languages.remove(lang) if len(languages) == 0: toDelete_wantedQueue.append(index) i = len(toDelete_wantedQueue) - 1 while i >= 0: log.debug( "checkSub: Removed item from the wantedQueue at index %s" % toDelete_wantedQueue[i]) autosub.WANTEDQUEUE.pop(toDelete_wantedQueue[i]) i = i - 1 log.debug("checkSub: Finished round of checkSub") autosub.WANTEDQUEUELOCK = False return True
def run(self): log.debug("checkSub: Starting round of checkSub") toDelete_wantedQueue = [] if not autosub.Helpers.checkAPICalls(): log.warning("checkSub: out of api calls") return True if autosub.WANTEDQUEUELOCK: log.debug("checkSub: Exiting, another threat is using the queues") return False else: autosub.WANTEDQUEUELOCK = True for index, wantedItem in enumerate(autosub.WANTEDQUEUE): title = wantedItem['title'] season = wantedItem['season'] episode = wantedItem['episode'] originalfile = wantedItem['originalFileLocationOnDisk'] languages = wantedItem['lang'] if not autosub.Helpers.checkAPICalls(): #Make sure that we are allow to connect to bierdopje log.warning("checkSub: out of api calls") break if autosub.SUBNL != "": srtfile = os.path.splitext(originalfile)[0] + u"." + autosub.SUBNL + u".srt" else: srtfile = os.path.splitext(originalfile)[0] + u".srt" engsrtfile = os.path.splitext(originalfile)[0] + u"." + autosub.SUBENG + u".srt" #lets try to find a showid showid = Helpers.getShowid(title) #no showid? skip this item if not showid: continue langtmp = languages[:] for lang in langtmp: log.debug("checkSub: trying to get a downloadlink for %s, language is %s" % (originalfile, lang)) downloadLink, release = autosub.Bierdopje.getSubLink(showid, lang, wantedItem) if downloadLink: if lang == 'nl': wantedItem['destinationFileLocationOnDisk'] = srtfile elif lang == 'en': wantedItem['destinationFileLocationOnDisk'] = engsrtfile log.info("checkSub: The episode %s - Season %s Episode %s has a matching subtitle on bierdopje, downloading it!" % (title, season, episode)) log.debug('checkSub: Dumping downloadlink for debug perpuse %s' %downloadLink) log.debug("checkSub: destination filename %s" % wantedItem['destinationFileLocationOnDisk']) wantedItem['downloadLink'] = downloadLink downloadItem = wantedItem.copy() downloadItem['downlang'] = lang downloadItem['subtitle'] = release if not DownloadSub(downloadItem): break if lang == 'nl' and (autosub.FALLBACKTOENG and not autosub.DOWNLOADENG) and 'en' in languages: log.debug('checkSub: We found a dutch subtitle and fallback is true. Removing the english subtitle from the wantedlist.') languages.remove('en') languages.remove(lang) if len(languages) == 0: toDelete_wantedQueue.append(index) break languages.remove(lang) if len(languages) == 0: toDelete_wantedQueue.append(index) i = len(toDelete_wantedQueue) - 1 while i >= 0: log.debug("checkSub: Removed item from the wantedQueue at index %s" % toDelete_wantedQueue[i]) autosub.WANTEDQUEUE.pop(toDelete_wantedQueue[i]) i = i - 1 log.debug("checkSub: Finished round of checkSub") autosub.WANTEDQUEUELOCK = False return True
def walkDir(path): SkipListFR = autosub.SKIPSTRINGFR.split(",") if len(autosub.SKIPSTRINGFR) > 0 else [] SkipListEN = autosub.SKIPSTRINGEN.split(",") if len(autosub.SKIPSTRINGEN) > 0 else [] # Check for dutch folder skip if len(autosub.SKIPFOLDERSFR) == 0: SkipFoldersFR = [] else: SkipFoldersFR = autosub.SKIPFOLDERSFR.split(",") if len(autosub.SKIPFOLDERSFR) > 0 else [] for idx,folder in enumerate(SkipFoldersFR): SkipFoldersFR[idx] = os.path.normpath(os.path.join(path,folder.strip(" \/"))) # Check for english folder skip if len(autosub.SKIPFOLDERSEN) == 0: SkipFoldersEN = [] else: SkipFoldersEN = autosub.SKIPFOLDERSEN.split(",") if len(autosub.SKIPFOLDERSEN) > 0 else [] for idx,folder in enumerate(SkipFoldersEN): SkipFoldersEN[idx] = os.path.normpath(os.path.join(path,folder.strip(" \/"))) for dirname, dirnames, filenames in os.walk(path, True, WalkError): #filenames = [decodeName(f) for f in filenames] SkipThisFolderFR = False for skip in SkipFoldersFR: if dirname.startswith(skip): SkipThisFolderFR = True break SkipThisFolderEN = False for skip in SkipFoldersEN: if dirname.startswith(skip): SkipThisFolderEN = True break log.debug("scanDisk: directory name: %s" %dirname) if re.search('_unpack_', dirname, re.IGNORECASE): log.debug("scanDisk: found a unpack directory, skipping.") continue if autosub.SKIPHIDDENDIRS and os.path.split(dirname)[1].startswith(u'.'): continue if re.search('_failed_', dirname, re.IGNORECASE): log.debug("scanDisk: found a failed directory, skipping.") continue if re.search('@eaDir', dirname, re.IGNORECASE): log.debug("scanDisk: found a Synology indexing directory, skipping.") tmpdirs = dirnames[:] for dir in tmpdirs: dirnames.remove(dir) continue if re.search("@.*thumb", dirname, re.IGNORECASE): log.debug("scanDisk: found a Qnap multimedia thumbnail folder, skipping.") continue langs = [] FileDict = {} for filename in filenames: if autosub.SEARCHSTOP: log.info('scanDisk: Forced Stop by user') return try: root,ext = os.path.splitext(filename) if ext[1:] in ('avi', 'mkv', 'wmv', 'ts', 'mp4'): if re.search('sample', filename): continue if not platform.system() == 'Windows': # Get best ascii compatible character for special characters try: if not isinstance(filename, unicode): coding = detect(filename)['encoding'] filename = unicode(filename.decode(coding),errors='replace') correctedFilename = ''.join((c for c in unicodedata.normalize('NFD', filename) if unicodedata.category(c) != 'Mn')) if filename != correctedFilename: os.rename(os.path.join(dirname, filename), os.path.join(dirname, correctedFilename)) log.info("scanDir: Renamed file %s" % correctedFilename) filename = correctedFilename except: log.error("scanDir: Skipping directory, file %s, %s" % (dirname,filename)) continue # What subtitle files should we expect? langs = [] FRext = u'.' + autosub.SUBFR + u'.srt' if autosub.SUBFR else u'.srt' ENext = u'.' + autosub.SUBENG + u'.srt' if autosub.SUBENG else u'.srt' ENext = u'.en.srt'if FRext == ENext and autosub.DOWNLOADFRENCH else ENext if not os.access(dirname, os.W_OK): log.error('scandisk: No write access to folder: %s' % dirname) continue # Check which languages we want to download based on user settings. log.debug('scanDir: Processing file: %s' % filename) if autosub.DOWNLOADFRENCH and not SkipThisFolderFR: Skipped = False for SkipItem in SkipListFR: if not SkipItem: break if re.search(SkipItem.lower(), filename.lower()): Skipped = True break if Skipped: log.info("scanDir: %s found in %s so skipped for French subs" % (SkipItem, filename)) elif os.path.exists(os.path.join(dirname, root + FRext)): Skipped = True log.debug("scanDir: %s skipped because the French subtitle already exists" % filename) else: # If the French subtitle not skipped and doesn't exist, then add it to the wanted list langs.append(autosub.FRENCH) if (autosub.DOWNLOADENG or (autosub.FALLBACKTOENG and autosub.DOWNLOADFRENCH and not Skipped)) and not SkipThisFolderEN: Skipped = False for SkipItem in SkipListEN: if not SkipItem: break if re.search(SkipItem.lower(), filename.lower()): Skipped = True break if Skipped: log.info("scanDir: %s found in %s so skipped for English subs" % (SkipItem, filename)) elif os.path.exists(os.path.join(dirname, root + ENext)): log.debug("scanDir: %s skipped because the English subtitle already exists" % filename) else: # If the English subtitle not skipped and doesn't exist, then add it to the wanted list if not os.path.exists(os.path.join(dirname, root + ENext)): langs.append(autosub.ENGLISH) if not langs: # nothing to do for this file continue FileDict = ProcessFilename(os.path.splitext(filename)[0].strip(), ext) if not FileDict: log.debug('scanDisk: not enough info in the filename: %s' % filename) continue Skip = False if autosub.MINMATCHSCORE & 8 and not FileDict['source'] : Skip = True elif autosub.MINMATCHSCORE & 4 and not FileDict['quality'] : Skip = True elif autosub.MINMATCHSCORE & 2 and not FileDict['codec'] : Skip = True elif autosub.MINMATCHSCORE & 1 and not FileDict['releasegrp']: Skip = True if Skip: log.debug('scanDisk: Filespec does not meet minmatchscore so skipping this one') FileDict['timestamp'] = unicode(time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(os.path.getctime(os.path.join(dirname, filename))))) FileDict['langs'] = langs FileDict['FRext'] = FRext FileDict['ENext'] = ENext FileDict['file'] = root FileDict['container'] = ext FileDict['folder'] = dirname FileDict['ImdbId'],FileDict['A7Id'], FileDict['TvdbId'], FileDict['title'] = Helpers.getShowid(FileDict['title']) if autosub.Helpers.SkipShow(FileDict['ImdbId'],FileDict['title'], FileDict['season'], FileDict['episode']): log.debug("scanDir: SKIPPED %s by Skipshow rules." % FileDict['file']) continue log.info("scanDir: %s WANTED FOR: %s" % (langs, filename)) autosub.WANTEDQUEUE.append(FileDict) time.sleep(0) except Exception as error: log.error('scanDir: Problem scanning file %s. Error is: %s' %(filename, error)) return
def run(self): log.debug("checkSub: Starting round of checkSub") # First we check if checksub in not still running if autosub.WANTEDQUEUELOCK: log.info( "checkSub: Exiting, another threat is using the queues. Will try again in 60 seconds" ) time.sleep(60) return False else: autosub.WANTEDQUEUELOCK = True autosub.DBCONNECTION = sqlite3.connect(autosub.DBFILE) autosub.DBIDCACHE = idCache() autosub.DBEPISODECACHE = EpisodeIdCache() toDelete_wantedQueue = [] if not Helpers.checkAPICallsTvdb( ) or not Helpers.checkAPICallsSubSeeker(): log.warning("checkSub: out of api calls") return True # Initiate the Addic7ed API and check the current number of downloads UseAddic = False if autosub.ADDIC7EDUSER and autosub.ADDIC7EDPASSWD and autosub.ADDIC7EDLANG != 'None': try: # Sets autosub.DOWNLOADS_A7 and autosub.DOWNLOADS_A7MAX # and gives a True response if it's ok to download from a7 autosub.ADDIC7EDAPI = autosub.Addic7ed.Addic7edAPI() UseAddic = autosub.ADDIC7EDAPI.checkCurrentDownloads( logout=False) except: log.debug("checkSub: Couldn't connect with Addic7ed.com") # Initiate a session to OpenSubtitles and log in if OpenSubtitles is choosen if autosub.OPENSUBTITLESLANG != 'None' and autosub.OPENSUBTITLESUSER and autosub.OPENSUBTITLESPASSWD: UseOpensubtitles = OpenSubtitlesLogin() else: UseOpensubtitles = False for index, wantedItem in enumerate(autosub.WANTEDQUEUE): title = wantedItem['title'] season = wantedItem['season'] episode = wantedItem['episode'] originalfile = wantedItem['originalFileLocationOnDisk'] languages = wantedItem['lang'] if not Helpers.checkAPICallsTvdb( ) or not Helpers.checkAPICallsSubSeeker(): #Make sure that we are allow to connect to SubtitleSeeker and TvDB log.warning("checkSub: out of api calls") break if autosub.SUBNL != "": nlsrtfile = os.path.splitext( originalfile)[0] + u"." + autosub.SUBNL + u".srt" else: nlsrtfile = os.path.splitext(originalfile)[0] + u".srt" if autosub.SUBENG == "": # Check for overlapping names if autosub.SUBNL != "" or not autosub.DOWNLOADDUTCH: engsrtfile = os.path.splitext(originalfile)[0] + u".srt" # Hardcoded fallback else: engsrtfile = os.path.splitext(originalfile)[0] + u".en.srt" else: engsrtfile = os.path.splitext( originalfile)[0] + u"." + autosub.SUBENG + u".srt" #lets try to find a showid; no showid? skip this item showid, a7_id, OsId = Helpers.getShowid(title, UseAddic, UseOpensubtitles) if UseOpensubtitles and OsId: EpisodeId = GetEpisodeId(OsId, season, episode) else: EpisodeId = None log.debug( "checkSub: ID's - IMDB: %s, Addic7ed: %s, OpenSubtitles: %s" % (showid, a7_id, OsId)) if not showid: continue for lang in languages[:]: downloadItem = wantedItem.copy() downloadItem['downlang'] = lang # Check if Addic7ed download limit has been reached if UseAddic and autosub.DOWNLOADS_A7 >= autosub.DOWNLOADS_A7MAX: UseAddic = False log.debug( "checkSub: You have reached your 24h limit of %s Addic7ed downloads!" % autosub.DOWNLOADS_A7MAX) log.debug( "checkSub: trying to get a downloadlink for %s, language is %s" % (originalfile, lang)) # get all links higher than the minmatch as input for downloadSub allResults = autosub.getSubLinks.getSubLinks( showid, a7_id, EpisodeId, lang, wantedItem) if not allResults: log.debug( "checkSub: no suitable subtitles were found for %s based on your minmatchscore" % downloadItem['originalFileLocationOnDisk']) continue if lang == autosub.DUTCH: downloadItem['destinationFileLocationOnDisk'] = nlsrtfile elif lang == autosub.ENGLISH: downloadItem['destinationFileLocationOnDisk'] = engsrtfile if allResults: log.info( "checkSub: The episode %s - Season %s Episode %s has 1 or more matching subtitles on SubtitleSeeker, downloading it!" % (title, season, episode)) log.debug("checkSub: destination filename %s" % downloadItem['destinationFileLocationOnDisk']) if not DownloadSub(allResults, UseAddic, downloadItem): continue #Remove downloaded language languages.remove(lang) if lang == autosub.DUTCH: if (autosub.FALLBACKTOENG and not autosub.DOWNLOADENG ) and autosub.ENGLISH in languages: log.debug( 'checkSub: We found a Dutch subtitle and fallback is true. Removing the English subtitle from the wantedlist.' ) languages.remove(autosub.ENGLISH) if autosub.ENGLISHSUBDELETE: log.info("checkSub: Clean up English enabled") if os.path.exists(engsrtfile): log.debug( "checkSub: Trying to delete English subtitle: %s" % engsrtfile) try: os.unlink(engsrtfile) log.info( "checkSub: Removed English subtitle: %s" % engsrtfile) except: log.error( "checkSub: Error while trying to remove subtitle %s." % engsrtfile) else: log.info("checkSub: English subtitle not found.") if len(languages) == 0: toDelete_wantedQueue.append(index) break autosub.DBCONNECTION.close() del autosub.DBCONNECTION del autosub.DBIDCACHE del autosub.DBEPISODECACHE if autosub.ADDIC7EDAPI: autosub.ADDIC7EDAPI.logout() if autosub.OPENSUBTITLESLOGGED_IN: OpenSubtitlesLogout() i = len(toDelete_wantedQueue) - 1 while i >= 0: log.debug( "checkSub: Removed item from the wantedQueue at index %s" % toDelete_wantedQueue[i]) autosub.WANTEDQUEUE.pop(toDelete_wantedQueue[i]) i = i - 1 log.debug("checkSub: Finished round of checkSub") autosub.WANTEDQUEUELOCK = False return True