def _OSdownload(SubId, SubCodec): log.debug("Download subtitle: %s" % SubId) time.sleep(1) if not OS_NoOp(): return None try: Result = autosub.OPENSUBTITLESSERVER.DownloadSubtitles( autosub.OPENSUBTITLESTOKEN, [SubId]) except: autosub.OPENSUBTITLESTOKEN = None log.error('Error from Opensubtitles download API. DownloadId is: %s' % SubId) return None if Result['status'] == '200 OK': try: CompressedData = Result['data'][0]['data'].decode('base64') except Exception as error: log.error( 'Error decompressing sub from opensubtitles. Message is: %s' % error) return None if not CompressedData: log.debug( 'No data returned from DownloadSubtitles API call. Skipping this one.' ) return None SubDataBytes = gzip.GzipFile(fileobj=io.BytesIO(CompressedData)).read() # Opensubtitles makes no difference in UTF-8 and UTF8-SIG so we check with chardet the correct encoding # if Opensubtile does not know the encoding we assume windows-1252 is used. if SubCodec: if 'UTF' in SubCodec.upper() or SubCodec == 'Unknown': SubCodec = chardet.detect(SubDataBytes)['encoding'] elif '1252' in SubCodec: SubCodec = u'cp1252' elif '850' in SubCodec: SubCodec = u'cp850' else: SubCodec = chardet.detect(SubDataBytes)['encoding'] if not 'UTF' in SubCodec.upper(): SubCodec = u'cp1252' try: SubData = SubDataBytes.decode(SubCodec, errors='replace') except Exception as error: log.error('Error decoding sub from opensubtitles. Message is: %s' % error) return None return (SubData) else: if Result['status'][:3] == '406': autosub.OPENSUBTITLESTOKEN = None log.error('Message : %s' % Result['status']) return None
def unzip(Session, url): # returns a file-like StringIO object try: Result = Session.get(url, verify=autosub.CERTIFICATEPATH) except: log.debug("unzip: Zip file at %s couldn't be retrieved" % url) return None try: zip = zipfile.ZipFile(io.BytesIO(Result.content)) except Exception as error: log.debug("unzip: Expected a zip file but got error for link %s" % url) log.debug("unzip: %s is likely a dead link" % url) return None nameList = zip.namelist() for name in nameList: # sometimes .nfo files are in the zip container if name.lower().endswith('srt'): try: Data = zip.read(name) Codec = detect(Data)['encoding'] fpr = io.TextIOWrapper(zip.open(name), errors='replace', encoding=Codec, newline='') SubData = fpr.read() fpr.close() if SubData: return SubData except Exception as error: pass log.debug("unzip: No subtitle files was found in the zip archive for %s" % url) return None
def _getzip(Session, url): # returns a file-like String object try: Result = Session.get(url, verify=autosub.CERT, timeout=15) except: log.debug("Zip file at %s couldn't be retrieved" % url) return None try: zip = zipfile.ZipFile(io.BytesIO(Result.content)) except Exception as error: log.debug("Expected a zip file but got error for link %s" % url) log.debug("%s is likely a dead link" % url) return None nameList = zip.namelist() for name in nameList: # sometimes .nfo files are in the zip container if name.lower().endswith('srt'): try: Data = zip.read(name) if Data.startswith(codecs.BOM_UTF8): SubData = unicode(Data[3:], 'UTF-8') else: Codec = chardet.detect(Data)['encoding'] SubData = unicode(Data, Codec) if SubData: return SubData except Exception as error: log.error(error.message) log.debug("No subtitle files was found in the zip archive for %s" % url) return None
def unzip(Session, url): # returns a file-like StringIO object try: Result = Session.get(url,verify=autosub.CERTIFICATEPATH) except: log.debug("unzip: Zip file at %s couldn't be retrieved" % url) return None try: zip = zipfile.ZipFile(io.BytesIO(Result.content)) except Exception as error: log.debug("unzip: Expected a zip file but got error for link %s" % url) log.debug("unzip: %s is likely a dead link" % url) return None nameList = zip.namelist() for name in nameList: # sometimes .nfo files are in the zip container if name.lower().endswith('srt'): try: Data = zip.read(name) Codec = detect(Data)['encoding'] fpr = io.TextIOWrapper(zip.open(name),errors='replace',encoding = Codec,newline='') SubData = fpr.read() fpr.close() if SubData: return SubData except Exception as error: pass log.debug("unzip: No subtitle files was found in the zip archive for %s" % url) return None
def openSubtitles(SubId, SubCodec): log.debug("OpenSubtitles: Download subtitle: %s" % SubId) Result = autosub.OPENSUBTITLESSERVER.DownloadSubtitles(autosub.OPENSUBTITLESTOKEN, [SubId]) if Result['status'] == '200 OK': compressed_data = Result['data'][0]['data'].decode('base64') SubDataBytes = gzip.GzipFile(fileobj=io.BytesIO(compressed_data)).read() # Opensubtitles sees not difference in UTF-8 and UTF8-SIG so we check with chardet the correct encoding if SubCodec == 'UTF-8': SubCodec = chardet.detect(SubDataBytes)['encoding'] SubData = SubDataBytes.decode(SubCodec) return(SubData) else: log.debug('OpenSubtitles: Download subtitle with ID: %s failed' %SubId) return None
def openSubtitles(SubId, SubCodec): log.debug("OpenSubtitles: Download subtitle: %s" % SubId) time.sleep(6) if not OpenSubtitlesNoOp(): return None try: Result = autosub.OPENSUBTITLESSERVER.DownloadSubtitles(autosub.OPENSUBTITLESTOKEN, [SubId]) except: autosub.OPENSUBTITLESTOKEN = None log.error('Opensubtitles: Error from Opensubtitles download API. DownloadId is: %s' %SubId) return None if Result['status'] == '200 OK': try: CompressedData = Result['data'][0]['data'].decode('base64') except Exception as error: log.error('downloadSubs: Error decompressing sub from opensubtitles. Message is: %s' % error) return None if not CompressedData: log.debug('DownloadSub: No data returned from DownloadSubtitles API call. Skipping this one.') return None SubDataBytes = gzip.GzipFile(fileobj=io.BytesIO(CompressedData)).read() # Opensubtitles makes no difference in UTF-8 and UTF8-SIG so we check with chardet the correct encoding # if Opensubtile does not know the encoding we assume windows-1252 is used. if SubCodec: if 'UTF' in SubCodec.upper() or SubCodec == 'Unknown': SubCodec = detect(SubDataBytes)['encoding'] if not SubCodec: SubCodec = u'windows-1252' else: SubCodec = u'windows-1252' try: SubData = SubDataBytes.decode(SubCodec,errors='replace') except Exception as error: log.error('downloadSubs: Error decoding sub from opensubtitles. Message is: %s' % error) return None return(SubData) else: if Result['status'][:3] == '407': autosub.OPENSUBTITLESTOKEN = 'limit' log.error('Opensubtitles: Error from Opensubtitles downloadsubs API. Message : %s' % Result['status']) return None
def walkDir(path): SkipListNL = autosub.SKIPSTRINGNL.split(",") if len(autosub.SKIPSTRINGNL) > 0 else [] SkipListEN = autosub.SKIPSTRINGEN.split(",") if len(autosub.SKIPSTRINGEN) > 0 else [] if len(autosub.SKIPFOLDERSNL) == 0: SkipFoldersNL = [] else: SkipFoldersNL = autosub.SKIPFOLDERSNL.split(",") if len(autosub.SKIPFOLDERSNL) > 0 else [] for idx,folder in enumerate(SkipFoldersNL): SkipFoldersNL[idx] = os.path.normpath(os.path.join(path,folder.rstrip("\/").lstrip("\/"))) if len(autosub.SKIPFOLDERSNL) == 0: SkipFoldersNL = [] else: SkipFoldersNL = autosub.SKIPFOLDERSNL.split(",") if len(autosub.SKIPFOLDERSNL) > 0 else [] for idx,folder in enumerate(SkipFoldersNL): SkipFoldersNL[idx] = os.path.normpath(path + folder) if len(autosub.SKIPFOLDERSEN) == 0: SkipFoldersEN = [] else: SkipFoldersEN = autosub.SKIPFOLDERSEN.split(",") if len(autosub.SKIPFOLDERSEN) > 0 else [] for idx,folder in enumerate(SkipFoldersEN): SkipFoldersEN[idx] = os.path.normpath(path + folder) for dirname, dirnames, filenames in os.walk(path, True, WalkError): SkipThisFolderNL = False for skip in SkipFoldersNL: if dirname.startswith(skip): SkipThisFolderNL = True break SkipThisFolderEN = False for skip in SkipFoldersEN: if dirname.startswith(skip): SkipThisFolderEN = True break log.debug("scanDisk: directory name: %s" %dirname) if re.search('_unpack_', dirname, re.IGNORECASE): log.debug("scanDisk: found a unpack directory, skipping.") continue if autosub.SKIPHIDDENDIRS and os.path.split(dirname)[1].startswith(u'.'): continue if re.search('_failed_', dirname, re.IGNORECASE): log.debug("scanDisk: found a failed directory, skipping.") continue if re.search('@eaDir', dirname, re.IGNORECASE): log.debug("scanDisk: found a Synology indexing directory, skipping.") tmpdirs = dirnames[:] for dir in tmpdirs: dirnames.remove(dir) continue if re.search("@.*thumb", dirname, re.IGNORECASE): log.debug("scanDisk: found a Qnap multimedia thumbnail folder, skipping.") continue langs = [] FileDict = {} for filename in filenames: root,ext = os.path.splitext(filename) if ext[1:] in ('avi', 'mkv', 'wmv', 'ts', 'mp4'): if re.search('sample', filename): continue if not platform.system() == 'Windows': # Get best ascii compatible character for special characters try: if not isinstance(filename, unicode): coding = detect(filename)['encoding'] filename = unicode(filename.decode(coding),errors='replace') correctedFilename = ''.join((c for c in unicodedata.normalize('NFD', filename) if unicodedata.category(c) != 'Mn')) if filename != correctedFilename: os.rename(os.path.join(dirname, filename), os.path.join(dirname, correctedFilename)) log.info("scanDir: Renamed file %s" % correctedFilename) filename = correctedFilename except: log.error("scanDir: Skipping directory, file %s, %s" % (dirname,filename)) continue # What subtitle files should we expect? langs = [] NLext = u'.' + autosub.SUBNL + u'.srt' if autosub.SUBNL else u'.srt' ENext = u'.' + autosub.SUBENG + u'.srt' if autosub.SUBENG else u'.srt' ENext = u'.en.srt'if NLext == ENext and autosub.DOWNLOADDUTCH else ENext if not os.access(dirname, os.W_OK): log.error('scandisk: No write access to folder: %s' % dirname) continue # Check which languages we want to download based on user settings. if autosub.DOWNLOADDUTCH and not SkipThisFolderNL: Skipped = False for SkipItem in SkipListNL: if not SkipItem: break if re.search(SkipItem.lower(), filename.lower()): Skipped = True break if Skipped: log.info("scanDir: %s found in %s so skipped for Dutch subs" % (SkipItem, filename)) elif os.path.exists(os.path.join(dirname, root + NLext)): Skipped = True log.debug("scanDir: %s skipped because the Dutch subtitle already exists" % filename) else: # If the Dutch subtitle not skipped and doesn't exist, then add it to the wanted list langs.append(autosub.DUTCH) if (autosub.DOWNLOADENG or (autosub.FALLBACKTOENG and autosub.DOWNLOADDUTCH and not Skipped)) and not SkipThisFolderEN: Skipped = False for SkipItem in SkipListEN: if not SkipItem: break if re.search(SkipItem.lower(), filename.lower()): Skipped = True break if Skipped: log.info("scanDir: %s found in %s so skipped for English subs" % (SkipItem, filename)) elif os.path.exists(os.path.join(dirname, root + ENext)): log.debug("scanDir: %s skipped because the English subtitle already exists" % filename) else: # If the English subtitle not skipped and doesn't exist, then add it to the wanted list if not os.path.exists(os.path.join(dirname, root + ENext)): langs.append(autosub.ENGLISH) if not langs: # nothing to do for this file continue FileDict = ProcessFilename(os.path.splitext(filename)[0].strip(), ext) time.sleep(0) if not FileDict: continue if not 'title' in FileDict.keys() or not 'season' in FileDict.keys() or not 'episode' in FileDict.keys(): continue if not FileDict['releasegrp'] and not FileDict['source'] and not FileDict['quality'] and not FileDict['source']: log.error("scanDir: Not enough info in filename: %s" % filename) continue FileDict['timestamp'] = unicode(time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(os.path.getctime(os.path.join(dirname, filename))))) FileDict['langs'] = langs FileDict['NLext'] = NLext FileDict['ENext'] = ENext FileDict['file'] = root FileDict['container'] = ext FileDict['folder'] = dirname FileDict['ImdbId'],FileDict['A7Id'], FileDict['TvdbId'], FileDict['title'] = Helpers.getShowid(FileDict['title'],autosub.ADDIC7EDLOGGED_IN) if autosub.Helpers.SkipShow(FileDict['ImdbId'],FileDict['title'], FileDict['season'], FileDict['episode']): continue log.info("scanDir: %s WANTED FOR: %s" % (langs, filename)) autosub.WANTEDQUEUE.append(FileDict) return
def walkDir(path): SkipListNL = [] SkipListEN = [] SkipListNL = autosub.SKIPSTRINGNL.split(",") SkipListEN = autosub.SKIPSTRINGEN.split(",") # Here we use os.walk to find all files in the path. for dirname, dirnames, filenames in os.walk(path): log.debug("scanDisk: directory name: %s" % dirname) if re.search('_unpack_', dirname, re.IGNORECASE): log.debug("scanDisk: found a unpack directory, skipping") continue if autosub.SKIPHIDDENDIRS and os.path.split(dirname)[1].startswith( u'.'): continue if re.search('_failed_', dirname, re.IGNORECASE): log.debug("scanDisk: found a failed directory, skipping") continue if re.search('@eaDir', dirname, re.IGNORECASE): log.debug( "scanDisk: found a Synology indexing directory, skipping this folder and all subfolders." ) tmpdirs = dirnames[:] for dir in tmpdirs: dirnames.remove(dir) continue if re.search("@.*thumb", dirname, re.IGNORECASE): # Ingnore QNAP multimedia thumbnail folders continue for filename in filenames: splitname = filename.split(".") ext = splitname[len(splitname) - 1] if ext in ('avi', 'mkv', 'wmv', 'ts', 'mp4'): if re.search('sample', filename): continue if not platform.system() == 'Windows': # Get best ascii compatible character for special characters try: if not isinstance(filename, unicode): coding = detect(filename) filename = unicode(filename.decode(coding)) correctedFilename = ''.join( (c for c in unicodedata.normalize('NFD', filename) if unicodedata.category(c) != 'Mn')) if filename != correctedFilename: os.rename(os.path.join(dirname, filename), os.path.join(dirname, correctedFilename)) log.info("scanDir: Renamed file %s" % correctedFilename) filename = correctedFilename except: log.error("scanDir: Skipping directory %s" % dirname) log.error("scanDir: Skipping file %s" % filename) continue # What subtitle files should we expect? lang = [] #Check what the Dutch subtitle would be. if autosub.SUBNL != "": srtfilenl = os.path.splitext( filename)[0] + u"." + autosub.SUBNL + u".srt" else: srtfilenl = os.path.splitext(filename)[0] + u".srt" #Check what the English subtitle would be. if autosub.SUBENG == "": # Check for overlapping names if autosub.SUBNL != "" or not autosub.DOWNLOADDUTCH: srtfileeng = os.path.splitext(filename)[0] + u".srt" # Hardcoded fallback else: srtfileeng = os.path.splitext(filename)[0] + u".en.srt" else: srtfileeng = os.path.splitext( filename)[0] + u"." + autosub.SUBENG + u".srt" # Check which languages we want to download based on user settings and check the skipstring if autosub.DOWNLOADDUTCH: Skipped = False for SkipItem in SkipListNL: if re.search(SkipItem.lower(), filename.lower()): Skipped = True log.debug( "scanDir: %s found in %s so skipped for Dutch subs" % (SkipItem, filename)) break if not Skipped and os.path.exists( os.path.join(dirname, srtfilenl)): Skipped = True log.debug( "scanDir: %s skipped because the subtitle already exists" % filename) # If the Dutch subtitle doesn't exist en has no skip item, then add it to the wanted list. if not Skipped: lang.append(autosub.DUTCH) if autosub.DOWNLOADENG: Skipped = False for SkipItem in SkipListEN: if re.search(SkipItem.lower(), filename.lower()): Skipped = True log.debug( "scanDir: %s found in %s so skipped for English subs" % (SkipItem, filename)) break if not Skipped and os.path.exists( os.path.join(dirname, srtfileeng)): Skipped = True log.debug( "scanDir: %s skipped because the subtitle already exists" % filename) # If the English subtitle doesn't exist en has no skip item, then add it to the wanted list. if not Skipped: lang.append(autosub.ENGLISH) if (autosub.FALLBACKTOENG and autosub.DOWNLOADDUTCH ) and not autosub.DOWNLOADENG and not Skipped: # If the Dutch and English subtitles do not exist and not skipped, then add English to the wanted list. if not os.path.exists(os.path.join( dirname, srtfilenl)) and not os.path.exists( os.path.join(dirname, srtfileeng)): lang.append(autosub.ENGLISH) if not lang: # autosub.WANTEDQUEUE empty continue log.debug("scanDir: File %s is missing subtitle(s): %s" % (filename, ', '.join(map(str, lang)))) filenameResults = ProcessFilename( os.path.splitext(filename)[0], os.path.splitext(filename)[1]) if 'title' in filenameResults.keys(): if 'season' in filenameResults.keys(): if 'episode' in filenameResults.keys(): title = filenameResults['title'] season = filenameResults['season'] episode = filenameResults['episode'] if not filenameResults[ 'releasegrp'] and not filenameResults[ 'source'] and not filenameResults[ 'quality'] and not filenameResults[ 'source']: continue if autosub.Helpers.SkipShow( title, season, episode) == True: log.debug("scanDir: SkipShow returned True") log.info( "scanDir: Skipping %s - Season %s Episode %s" % (title, season, episode)) continue if len(lang) == 1: log.info( "scanDir: %s subtitle wanted for %s and added to wantedQueue" % (lang[0], filename)) else: log.info( "scanDir: %s subtitles wanted for %s and added to wantedQueue" % (' and '.join(map(str, lang)), filename)) filenameResults[ 'originalFileLocationOnDisk'] = os.path.join( dirname, filename) filenameResults['timestamp'] = unicode( time.strftime( '%Y-%m-%d %H:%M:%S', time.localtime( os.path.getctime(filenameResults[ 'originalFileLocationOnDisk'])))) filenameResults['lang'] = lang autosub.WANTEDQUEUE.append(filenameResults) else: log.error( "scanDir: Could not process the filename properly filename: %s" % filename) continue else: log.error( "scanDir: Could not process the filename properly filename: %s" % filename) continue else: log.error( "scanDir: Could not process the filename properly filename: %s" % filename) continue
def walkDir(path): SkipListNL = autosub.SKIPSTRINGNL.split(",") if len( autosub.SKIPSTRINGNL) > 0 else [] SkipListEN = autosub.SKIPSTRINGEN.split(",") if len( autosub.SKIPSTRINGEN) > 0 else [] # Check for dutch folder skip if len(autosub.SKIPFOLDERSNL) == 0: SkipFoldersNL = [] else: SkipFoldersNL = autosub.SKIPFOLDERSNL.split(",") if len( autosub.SKIPFOLDERSNL) > 0 else [] for idx, folder in enumerate(SkipFoldersNL): SkipFoldersNL[idx] = os.path.normpath( os.path.join(path, folder.strip(" \/"))) # Check for english folder skip if len(autosub.SKIPFOLDERSEN) == 0: SkipFoldersEN = [] else: SkipFoldersEN = autosub.SKIPFOLDERSEN.split(",") if len( autosub.SKIPFOLDERSEN) > 0 else [] for idx, folder in enumerate(SkipFoldersEN): SkipFoldersEN[idx] = os.path.normpath( os.path.join(path, folder.strip(" \/"))) for dirname, dirnames, filenames in os.walk(path, True, WalkError): SkipThisFolderNL = False for skip in SkipFoldersNL: if dirname.startswith(skip): SkipThisFolderNL = True break SkipThisFolderEN = False for skip in SkipFoldersEN: if dirname.startswith(skip): SkipThisFolderEN = True break log.debug("scanDisk: directory name: %s" % dirname) if re.search('_unpack_', dirname, re.IGNORECASE): log.debug("scanDisk: found a unpack directory, skipping.") continue if autosub.SKIPHIDDENDIRS and os.path.split(dirname)[1].startswith( u'.'): continue if re.search('_failed_', dirname, re.IGNORECASE): log.debug("scanDisk: found a failed directory, skipping.") continue if re.search('@eaDir', dirname, re.IGNORECASE): log.debug( "scanDisk: found a Synology indexing directory, skipping.") tmpdirs = dirnames[:] for dir in tmpdirs: dirnames.remove(dir) continue if re.search("@.*thumb", dirname, re.IGNORECASE): log.debug( "scanDisk: found a Qnap multimedia thumbnail folder, skipping." ) continue langs = [] FileDict = {} for filename in filenames: if autosub.SEARCHSTOP: log.info('scanDisk: Forced Stop by user') return try: root, ext = os.path.splitext(filename) if ext[1:] in ('avi', 'mkv', 'wmv', 'ts', 'mp4'): if re.search('sample', filename): continue if not platform.system() == 'Windows': # Get best ascii compatible character for special characters try: if not isinstance(filename, unicode): coding = detect(filename)['encoding'] filename = unicode(filename.decode(coding), errors='replace') correctedFilename = ''.join( (c for c in unicodedata.normalize( 'NFD', filename) if unicodedata.category(c) != 'Mn')) if filename != correctedFilename: os.rename( os.path.join(dirname, filename), os.path.join(dirname, correctedFilename)) log.info("scanDir: Renamed file %s" % correctedFilename) filename = correctedFilename except: log.error( "scanDir: Skipping directory, file %s, %s" % (dirname, filename)) continue # What subtitle files should we expect? langs = [] NLext = u'.' + autosub.SUBNL + u'.srt' if autosub.SUBNL else u'.srt' ENext = u'.' + autosub.SUBENG + u'.srt' if autosub.SUBENG else u'.srt' ENext = u'.en.srt' if NLext == ENext and autosub.DOWNLOADDUTCH else ENext if not os.access(dirname, os.W_OK): log.error('scandisk: No write access to folder: %s' % dirname) continue # Check which languages we want to download based on user settings. log.debug('scanDir: Processing file: %s' % filename) if autosub.DOWNLOADDUTCH and not SkipThisFolderNL: Skipped = False for SkipItem in SkipListNL: if not SkipItem: break if re.search(SkipItem.lower(), filename.lower()): Skipped = True break if Skipped: log.info( "scanDir: %s found in %s so skipped for Dutch subs" % (SkipItem, filename)) elif os.path.exists(os.path.join( dirname, root + NLext)): Skipped = True log.debug( "scanDir: %s skipped because the Dutch subtitle already exists" % filename) else: # If the Dutch subtitle not skipped and doesn't exist, then add it to the wanted list langs.append(autosub.DUTCH) if (autosub.DOWNLOADENG or (autosub.FALLBACKTOENG and autosub.DOWNLOADDUTCH and not Skipped)) and not SkipThisFolderEN: Skipped = False for SkipItem in SkipListEN: if not SkipItem: break if re.search(SkipItem.lower(), filename.lower()): Skipped = True break if Skipped: log.info( "scanDir: %s found in %s so skipped for English subs" % (SkipItem, filename)) elif os.path.exists(os.path.join( dirname, root + ENext)): log.debug( "scanDir: %s skipped because the English subtitle already exists" % filename) else: # If the English subtitle not skipped and doesn't exist, then add it to the wanted list if not os.path.exists( os.path.join(dirname, root + ENext)): langs.append(autosub.ENGLISH) if not langs: # nothing to do for this file continue FileDict = ProcessFilename( os.path.splitext(filename)[0].strip(), ext) if not FileDict: log.debug( 'scanDisk: not enough info in the filename: %s' % filename) continue Skip = False if autosub.MINMATCHSCORE & 8 and not FileDict['source']: Skip = True elif autosub.MINMATCHSCORE & 4 and not FileDict['quality']: Skip = True elif autosub.MINMATCHSCORE & 2 and not FileDict['codec']: Skip = True elif autosub.MINMATCHSCORE & 1 and not FileDict[ 'releasegrp']: Skip = True if Skip: log.debug( 'scanDisk: Filespec does not meet minmatchscore so skipping this one' ) continue FileDict['timestamp'] = unicode( time.strftime( '%Y-%m-%d %H:%M:%S', time.localtime( os.path.getmtime( os.path.join(dirname, filename))))) FileDict['langs'] = langs FileDict['NLext'] = NLext FileDict['ENext'] = ENext FileDict['file'] = root FileDict['container'] = ext FileDict['folder'] = dirname FileDict['ImdbId'], FileDict['A7Id'], FileDict[ 'TvdbId'], FileDict['title'] = Helpers.getShowid( FileDict['title']) if autosub.Helpers.SkipShow(FileDict['ImdbId'], FileDict['title'], FileDict['season'], FileDict['episode']): log.debug("scanDir: SKIPPED %s by Skipshow rules." % FileDict['file']) continue log.info("scanDir: %s WANTED FOR: %s" % (langs, filename)) autosub.WANTEDQUEUE.append(FileDict) time.sleep(0) except Exception as error: log.error('scanDir: Problem scanning file %s. Error is: %s' % (filename, error)) return
def walkDir(path): SkipListNL = [] SkipListEN = [] SkipListNL = autosub.SKIPSTRINGNL.split(",") SkipListEN = autosub.SKIPSTRINGEN.split(",") # Here we use os.walk to find all files in the path. for dirname, dirnames, filenames in os.walk(path): log.debug("scanDisk: directory name: %s" %dirname) if re.search('_unpack_', dirname, re.IGNORECASE): log.debug("scanDisk: found a unpack directory, skipping") continue if autosub.SKIPHIDDENDIRS and os.path.split(dirname)[1].startswith(u'.'): continue if re.search('_failed_', dirname, re.IGNORECASE): log.debug("scanDisk: found a failed directory, skipping") continue if re.search('@eaDir', dirname, re.IGNORECASE): log.debug("scanDisk: found a Synology indexing directory, skipping this folder and all subfolders.") tmpdirs = dirnames[:] for dir in tmpdirs: dirnames.remove(dir) continue if re.search("@.*thumb", dirname, re.IGNORECASE): # Ingnore QNAP multimedia thumbnail folders continue for filename in filenames: splitname = filename.split(".") ext = splitname[len(splitname) - 1] if ext in ('avi', 'mkv', 'wmv', 'ts', 'mp4'): if re.search('sample', filename): continue if not platform.system() == 'Windows': # Get best ascii compatible character for special characters try: if not isinstance(filename, unicode): coding = detect(filename) filename = unicode(filename.decode(coding)) correctedFilename = ''.join((c for c in unicodedata.normalize('NFD', filename) if unicodedata.category(c) != 'Mn')) if filename != correctedFilename: os.rename(os.path.join(dirname, filename), os.path.join(dirname, correctedFilename)) log.info("scanDir: Renamed file %s" % correctedFilename) filename = correctedFilename except: log.error("scanDir: Skipping directory %s" % dirname) log.error("scanDir: Skipping file %s" % filename) continue # What subtitle files should we expect? lang=[] #Check what the Dutch subtitle would be. if autosub.SUBNL != "": srtfilenl = os.path.splitext(filename)[0] + u"." + autosub.SUBNL + u".srt" else: srtfilenl = os.path.splitext(filename)[0] + u".srt" #Check what the English subtitle would be. if autosub.SUBENG == "": # Check for overlapping names if autosub.SUBNL != "" or not autosub.DOWNLOADDUTCH: srtfileeng = os.path.splitext(filename)[0] + u".srt" # Hardcoded fallback else: srtfileeng = os.path.splitext(filename)[0] + u".en.srt" else: srtfileeng = os.path.splitext(filename)[0] + u"." + autosub.SUBENG + u".srt" # Check which languages we want to download based on user settings and check the skipstring if autosub.DOWNLOADDUTCH: Skipped = False for SkipItem in SkipListNL: if re.search(SkipItem.lower(), filename.lower()): Skipped = True log.debug("scanDir: %s found in %s so skipped for Dutch subs" % (SkipItem, filename)) break if not Skipped and os.path.exists(os.path.join(dirname, srtfilenl)): Skipped = True log.debug("scanDir: %s skipped because the subtitle already exists" % filename) # If the Dutch subtitle doesn't exist en has no skip item, then add it to the wanted list. if not Skipped: lang.append(autosub.DUTCH) if autosub.DOWNLOADENG: Skipped = False for SkipItem in SkipListEN: if re.search(SkipItem.lower(), filename.lower()): Skipped = True log.debug("scanDir: %s found in %s so skipped for English subs" % (SkipItem, filename)) break if not Skipped and os.path.exists(os.path.join(dirname, srtfileeng)): Skipped = True log.debug("scanDir: %s skipped because the subtitle already exists" % filename) # If the English subtitle doesn't exist en has no skip item, then add it to the wanted list. if not Skipped: lang.append(autosub.ENGLISH) if (autosub.FALLBACKTOENG and autosub.DOWNLOADDUTCH) and not autosub.DOWNLOADENG and not Skipped: # If the Dutch and English subtitles do not exist and not skipped, then add English to the wanted list. if not os.path.exists(os.path.join(dirname, srtfilenl)) and not os.path.exists(os.path.join(dirname, srtfileeng)): lang.append(autosub.ENGLISH) if not lang: # autosub.WANTEDQUEUE empty continue log.debug("scanDir: File %s is missing subtitle(s): %s" % (filename, ', '.join(map(str,lang)))) filenameResults = ProcessFilename(os.path.splitext(filename)[0], os.path.splitext(filename)[1]) if 'title' in filenameResults.keys(): if 'season' in filenameResults.keys(): if 'episode' in filenameResults.keys(): title = filenameResults['title'] season = filenameResults['season'] episode = filenameResults['episode'] if not filenameResults['releasegrp'] and not filenameResults['source'] and not filenameResults['quality'] and not filenameResults['source']: continue if autosub.Helpers.SkipShow(title, season, episode) == True: log.debug("scanDir: SkipShow returned True") log.info("scanDir: Skipping %s - Season %s Episode %s" % (title, season, episode)) continue if len(lang) == 1: log.info("scanDir: %s subtitle wanted for %s and added to wantedQueue" % (lang[0], filename)) else: log.info("scanDir: %s subtitles wanted for %s and added to wantedQueue" % (' and '.join(map(str,lang)), filename)) filenameResults['originalFileLocationOnDisk'] = os.path.join(dirname, filename) filenameResults['timestamp'] = unicode(time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(os.path.getctime(filenameResults['originalFileLocationOnDisk'])))) filenameResults['lang'] = lang autosub.WANTEDQUEUE.append(filenameResults) else: log.error("scanDir: Could not process the filename properly filename: %s" % filename) continue else: log.error("scanDir: Could not process the filename properly filename: %s" % filename) continue else: log.error("scanDir: Could not process the filename properly filename: %s" % filename) continue
def walkDir(path): SkipListFR = autosub.SKIPSTRINGFR.split(",") if len(autosub.SKIPSTRINGFR) > 0 else [] SkipListEN = autosub.SKIPSTRINGEN.split(",") if len(autosub.SKIPSTRINGEN) > 0 else [] # Check for dutch folder skip if len(autosub.SKIPFOLDERSFR) == 0: SkipFoldersFR = [] else: SkipFoldersFR = autosub.SKIPFOLDERSFR.split(",") if len(autosub.SKIPFOLDERSFR) > 0 else [] for idx,folder in enumerate(SkipFoldersFR): SkipFoldersFR[idx] = os.path.normpath(os.path.join(path,folder.strip(" \/"))) # Check for english folder skip if len(autosub.SKIPFOLDERSEN) == 0: SkipFoldersEN = [] else: SkipFoldersEN = autosub.SKIPFOLDERSEN.split(",") if len(autosub.SKIPFOLDERSEN) > 0 else [] for idx,folder in enumerate(SkipFoldersEN): SkipFoldersEN[idx] = os.path.normpath(os.path.join(path,folder.strip(" \/"))) for dirname, dirnames, filenames in os.walk(path, True, WalkError): #filenames = [decodeName(f) for f in filenames] SkipThisFolderFR = False for skip in SkipFoldersFR: if dirname.startswith(skip): SkipThisFolderFR = True break SkipThisFolderEN = False for skip in SkipFoldersEN: if dirname.startswith(skip): SkipThisFolderEN = True break log.debug("scanDisk: directory name: %s" %dirname) if re.search('_unpack_', dirname, re.IGNORECASE): log.debug("scanDisk: found a unpack directory, skipping.") continue if autosub.SKIPHIDDENDIRS and os.path.split(dirname)[1].startswith(u'.'): continue if re.search('_failed_', dirname, re.IGNORECASE): log.debug("scanDisk: found a failed directory, skipping.") continue if re.search('@eaDir', dirname, re.IGNORECASE): log.debug("scanDisk: found a Synology indexing directory, skipping.") tmpdirs = dirnames[:] for dir in tmpdirs: dirnames.remove(dir) continue if re.search("@.*thumb", dirname, re.IGNORECASE): log.debug("scanDisk: found a Qnap multimedia thumbnail folder, skipping.") continue langs = [] FileDict = {} for filename in filenames: if autosub.SEARCHSTOP: log.info('scanDisk: Forced Stop by user') return try: root,ext = os.path.splitext(filename) if ext[1:] in ('avi', 'mkv', 'wmv', 'ts', 'mp4'): if re.search('sample', filename): continue if not platform.system() == 'Windows': # Get best ascii compatible character for special characters try: if not isinstance(filename, unicode): coding = detect(filename)['encoding'] filename = unicode(filename.decode(coding),errors='replace') correctedFilename = ''.join((c for c in unicodedata.normalize('NFD', filename) if unicodedata.category(c) != 'Mn')) if filename != correctedFilename: os.rename(os.path.join(dirname, filename), os.path.join(dirname, correctedFilename)) log.info("scanDir: Renamed file %s" % correctedFilename) filename = correctedFilename except: log.error("scanDir: Skipping directory, file %s, %s" % (dirname,filename)) continue # What subtitle files should we expect? langs = [] FRext = u'.' + autosub.SUBFR + u'.srt' if autosub.SUBFR else u'.srt' ENext = u'.' + autosub.SUBENG + u'.srt' if autosub.SUBENG else u'.srt' ENext = u'.en.srt'if FRext == ENext and autosub.DOWNLOADFRENCH else ENext if not os.access(dirname, os.W_OK): log.error('scandisk: No write access to folder: %s' % dirname) continue # Check which languages we want to download based on user settings. log.debug('scanDir: Processing file: %s' % filename) if autosub.DOWNLOADFRENCH and not SkipThisFolderFR: Skipped = False for SkipItem in SkipListFR: if not SkipItem: break if re.search(SkipItem.lower(), filename.lower()): Skipped = True break if Skipped: log.info("scanDir: %s found in %s so skipped for French subs" % (SkipItem, filename)) elif os.path.exists(os.path.join(dirname, root + FRext)): Skipped = True log.debug("scanDir: %s skipped because the French subtitle already exists" % filename) else: # If the French subtitle not skipped and doesn't exist, then add it to the wanted list langs.append(autosub.FRENCH) if (autosub.DOWNLOADENG or (autosub.FALLBACKTOENG and autosub.DOWNLOADFRENCH and not Skipped)) and not SkipThisFolderEN: Skipped = False for SkipItem in SkipListEN: if not SkipItem: break if re.search(SkipItem.lower(), filename.lower()): Skipped = True break if Skipped: log.info("scanDir: %s found in %s so skipped for English subs" % (SkipItem, filename)) elif os.path.exists(os.path.join(dirname, root + ENext)): log.debug("scanDir: %s skipped because the English subtitle already exists" % filename) else: # If the English subtitle not skipped and doesn't exist, then add it to the wanted list if not os.path.exists(os.path.join(dirname, root + ENext)): langs.append(autosub.ENGLISH) if not langs: # nothing to do for this file continue FileDict = ProcessFilename(os.path.splitext(filename)[0].strip(), ext) if not FileDict: log.debug('scanDisk: not enough info in the filename: %s' % filename) continue Skip = False if autosub.MINMATCHSCORE & 8 and not FileDict['source'] : Skip = True elif autosub.MINMATCHSCORE & 4 and not FileDict['quality'] : Skip = True elif autosub.MINMATCHSCORE & 2 and not FileDict['codec'] : Skip = True elif autosub.MINMATCHSCORE & 1 and not FileDict['releasegrp']: Skip = True if Skip: log.debug('scanDisk: Filespec does not meet minmatchscore so skipping this one') FileDict['timestamp'] = unicode(time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(os.path.getctime(os.path.join(dirname, filename))))) FileDict['langs'] = langs FileDict['FRext'] = FRext FileDict['ENext'] = ENext FileDict['file'] = root FileDict['container'] = ext FileDict['folder'] = dirname FileDict['ImdbId'],FileDict['A7Id'], FileDict['TvdbId'], FileDict['title'] = Helpers.getShowid(FileDict['title']) if autosub.Helpers.SkipShow(FileDict['ImdbId'],FileDict['title'], FileDict['season'], FileDict['episode']): log.debug("scanDir: SKIPPED %s by Skipshow rules." % FileDict['file']) continue log.info("scanDir: %s WANTED FOR: %s" % (langs, filename)) autosub.WANTEDQUEUE.append(FileDict) time.sleep(0) except Exception as error: log.error('scanDir: Problem scanning file %s. Error is: %s' %(filename, error)) return
def walkDir(path): for dirname, dirnames, filenames in os.walk(path): log.debug("scanDisk: directory name: %s" % dirname) if re.search("_unpack_", dirname, re.IGNORECASE): log.debug("scanDisk: found a unpack directory, skipping") continue if autosub.SKIPHIDDENDIRS and os.path.split(dirname)[1].startswith(u"."): continue if re.search("_failed_", dirname, re.IGNORECASE): log.debug("scanDisk: found a failed directory, skipping") continue if re.search("@eaDir", dirname, re.IGNORECASE): log.debug("scanDisk: found a Synology indexing directory, skipping this folder and all subfolders.") tmpdirs = dirnames[:] for dir in tmpdirs: dirnames.remove(dir) continue if re.search("@.*thumb", dirname, re.IGNORECASE): # Ingnore QNAP multimedia thumbnail folders continue for filename in filenames: splitname = filename.split(".") ext = splitname[len(splitname) - 1] if ext in ("avi", "mkv", "wmv", "ts", "mp4"): if re.search("sample", filename): continue if autosub.WEBDL == "None": if re.search("web-dl", filename.lower()): log.debug("scanDisk: WEB-DL is set to 'None', skipping %s" % filename) continue if not platform.system() == "Windows": # Get best ascii compatible character for special characters try: if not isinstance(filename, unicode): coding = detect(filename) filename = unicode(filename.decode(coding)) correctedFilename = "".join( (c for c in unicodedata.normalize("NFD", filename) if unicodedata.category(c) != "Mn") ) if filename != correctedFilename: os.rename(os.path.join(dirname, filename), os.path.join(dirname, correctedFilename)) log.info("scanDir: Renamed file %s" % correctedFilename) filename = correctedFilename except: log.error("scanDir: Skipping directory %s" % dirname) log.error("scanDir: Skipping file %s" % filename) continue # What subtitle files should we expect? lang = [] # Check what the Dutch subtitle would be. if autosub.SUBNL != "": srtfilenl = os.path.splitext(filename)[0] + u"." + autosub.SUBNL + u".srt" else: srtfilenl = os.path.splitext(filename)[0] + u".srt" # Check what the English subtitle would be. if autosub.SUBENG == "": # Check for overlapping names if autosub.SUBNL != "" or not autosub.DOWNLOADDUTCH: srtfileeng = os.path.splitext(filename)[0] + u".srt" # Hardcoded fallback else: srtfileeng = os.path.splitext(filename)[0] + u".en.srt" else: srtfileeng = os.path.splitext(filename)[0] + u"." + autosub.SUBENG + u".srt" # Check which languages we want to download based on user settings. if autosub.DOWNLOADDUTCH: # If the Dutch subtitle doesn't exist, then add it to the wanted list. if not os.path.exists(os.path.join(dirname, srtfilenl)): lang.append(autosub.DUTCH) if autosub.DOWNLOADENG: # If the English subtitle doesn't exist, then add it to the wanted list. if not os.path.exists(os.path.join(dirname, srtfileeng)): if autosub.WEBDL == "DutchOnly" and re.search("web-dl", filename.lower()): log.debug( "scanDisk: WEB-DL is set to 'Dutch Only', not adding English as wanted for %s" % filename ) else: lang.append(autosub.ENGLISH) if (autosub.FALLBACKTOENG and autosub.DOWNLOADDUTCH) and not autosub.DOWNLOADENG: # If the Dutch and English subtitles do not exist, then add English to the wanted list. if not os.path.exists(os.path.join(dirname, srtfilenl)) and not os.path.exists( os.path.join(dirname, srtfileeng) ): if autosub.WEBDL == "DutchOnly" and re.search("web-dl", filename.lower()): log.debug( "scanDisk: WEB-DL is set to 'Dutch Only', not adding English as wanted for %s" % filename ) else: lang.append(autosub.ENGLISH) if not lang: # autosub.WANTEDQUEUE empty continue log.debug("scanDir: File %s is missing subtitle(s): %s" % (filename, ", ".join(map(str, lang)))) filenameResults = ProcessFilename(os.path.splitext(filename)[0], os.path.splitext(filename)[1]) if "title" in filenameResults.keys(): if "season" in filenameResults.keys(): if "episode" in filenameResults.keys(): title = filenameResults["title"] season = filenameResults["season"] episode = filenameResults["episode"] if ( not filenameResults["releasegrp"] and not filenameResults["source"] and not filenameResults["quality"] and not filenameResults["source"] ): continue if autosub.Helpers.SkipShow(title, season, episode) == True: log.debug("scanDir: SkipShow returned True") log.info("scanDir: Skipping %s - Season %s Episode %s" % (title, season, episode)) continue if len(lang) == 1: log.info( "scanDir: %s subtitle wanted for %s and added to wantedQueue" % (lang[0], filename) ) else: log.info( "scanDir: %s subtitles wanted for %s and added to wantedQueue" % (" and ".join(map(str, lang)), filename) ) filenameResults["originalFileLocationOnDisk"] = os.path.join(dirname, filename) filenameResults["timestamp"] = unicode( time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(os.path.getctime(filenameResults["originalFileLocationOnDisk"])), ) ) filenameResults["lang"] = lang autosub.WANTEDQUEUE.append(filenameResults) else: log.error("scanDir: Could not process the filename properly filename: %s" % filename) continue else: log.error("scanDir: Could not process the filename properly filename: %s" % filename) continue else: log.error("scanDir: Could not process the filename properly filename: %s" % filename) continue