def _OSdownload(SubId, SubCodec):
log.debug("Download subtitle: %s" % SubId)
time.sleep(1)
if not OS_NoOp():
return None
try:
Result = autosub.OPENSUBTITLESSERVER.DownloadSubtitles(
autosub.OPENSUBTITLESTOKEN, [SubId])
except:
autosub.OPENSUBTITLESTOKEN = None
log.error('Error from Opensubtitles download API. DownloadId is: %s' %
SubId)
return None
if Result['status'] == '200 OK':
try:
CompressedData = Result['data'][0]['data'].decode('base64')
except Exception as error:
log.error(
'Error decompressing sub from opensubtitles. Message is: %s' %
error)
return None
if not CompressedData:
log.debug(
'No data returned from DownloadSubtitles API call. Skipping this one.'
)
return None
SubDataBytes = gzip.GzipFile(fileobj=io.BytesIO(CompressedData)).read()
# Opensubtitles makes no difference in UTF-8 and UTF8-SIG so we check with chardet the correct encoding
# if Opensubtile does not know the encoding we assume windows-1252 is used.
if SubCodec:
if 'UTF' in SubCodec.upper() or SubCodec == 'Unknown':
SubCodec = chardet.detect(SubDataBytes)['encoding']
elif '1252' in SubCodec:
SubCodec = u'cp1252'
elif '850' in SubCodec:
SubCodec = u'cp850'
else:
SubCodec = chardet.detect(SubDataBytes)['encoding']
if not 'UTF' in SubCodec.upper():
SubCodec = u'cp1252'
try:
SubData = SubDataBytes.decode(SubCodec, errors='replace')
except Exception as error:
log.error('Error decoding sub from opensubtitles. Message is: %s' %
error)
return None
return (SubData)
else:
if Result['status'][:3] == '406':
autosub.OPENSUBTITLESTOKEN = None
log.error('Message : %s' % Result['status'])
return None
def unzip(Session, url):
# returns a file-like StringIO object
try:
Result = Session.get(url, verify=autosub.CERTIFICATEPATH)
except:
log.debug("unzip: Zip file at %s couldn't be retrieved" % url)
return None
try:
zip = zipfile.ZipFile(io.BytesIO(Result.content))
except Exception as error:
log.debug("unzip: Expected a zip file but got error for link %s" % url)
log.debug("unzip: %s is likely a dead link" % url)
return None
nameList = zip.namelist()
for name in nameList:
# sometimes .nfo files are in the zip container
if name.lower().endswith('srt'):
try:
Data = zip.read(name)
Codec = detect(Data)['encoding']
fpr = io.TextIOWrapper(zip.open(name),
errors='replace',
encoding=Codec,
newline='')
SubData = fpr.read()
fpr.close()
if SubData:
return SubData
except Exception as error:
pass
log.debug("unzip: No subtitle files was found in the zip archive for %s" %
url)
return None
def _getzip(Session, url):
# returns a file-like String object
try:
Result = Session.get(url, verify=autosub.CERT, timeout=15)
except:
log.debug("Zip file at %s couldn't be retrieved" % url)
return None
try:
zip = zipfile.ZipFile(io.BytesIO(Result.content))
except Exception as error:
log.debug("Expected a zip file but got error for link %s" % url)
log.debug("%s is likely a dead link" % url)
return None
nameList = zip.namelist()
for name in nameList:
# sometimes .nfo files are in the zip container
if name.lower().endswith('srt'):
try:
Data = zip.read(name)
if Data.startswith(codecs.BOM_UTF8):
SubData = unicode(Data[3:], 'UTF-8')
else:
Codec = chardet.detect(Data)['encoding']
SubData = unicode(Data, Codec)
if SubData:
return SubData
except Exception as error:
log.error(error.message)
log.debug("No subtitle files was found in the zip archive for %s" % url)
return None
def unzip(Session, url):
# returns a file-like StringIO object
try:
Result = Session.get(url,verify=autosub.CERTIFICATEPATH)
except:
log.debug("unzip: Zip file at %s couldn't be retrieved" % url)
return None
try:
zip = zipfile.ZipFile(io.BytesIO(Result.content))
except Exception as error:
log.debug("unzip: Expected a zip file but got error for link %s" % url)
log.debug("unzip: %s is likely a dead link" % url)
return None
nameList = zip.namelist()
for name in nameList:
# sometimes .nfo files are in the zip container
if name.lower().endswith('srt'):
try:
Data = zip.read(name)
Codec = detect(Data)['encoding']
fpr = io.TextIOWrapper(zip.open(name),errors='replace',encoding = Codec,newline='')
SubData = fpr.read()
fpr.close()
if SubData:
return SubData
except Exception as error:
pass
log.debug("unzip: No subtitle files was found in the zip archive for %s" % url)
return None
def openSubtitles(SubId, SubCodec):
log.debug("OpenSubtitles: Download subtitle: %s" % SubId)
Result = autosub.OPENSUBTITLESSERVER.DownloadSubtitles(autosub.OPENSUBTITLESTOKEN, [SubId])
if Result['status'] == '200 OK':
compressed_data = Result['data'][0]['data'].decode('base64')
SubDataBytes = gzip.GzipFile(fileobj=io.BytesIO(compressed_data)).read()
# Opensubtitles sees not difference in UTF-8 and UTF8-SIG so we check with chardet the correct encoding
if SubCodec == 'UTF-8':
SubCodec = chardet.detect(SubDataBytes)['encoding']
SubData = SubDataBytes.decode(SubCodec)
return(SubData)
else:
log.debug('OpenSubtitles: Download subtitle with ID: %s failed' %SubId)
return None
def openSubtitles(SubId, SubCodec):
log.debug("OpenSubtitles: Download subtitle: %s" % SubId)
time.sleep(6)
if not OpenSubtitlesNoOp():
return None
try:
Result = autosub.OPENSUBTITLESSERVER.DownloadSubtitles(autosub.OPENSUBTITLESTOKEN, [SubId])
except:
autosub.OPENSUBTITLESTOKEN = None
log.error('Opensubtitles: Error from Opensubtitles download API. DownloadId is: %s' %SubId)
return None
if Result['status'] == '200 OK':
try:
CompressedData = Result['data'][0]['data'].decode('base64')
except Exception as error:
log.error('downloadSubs: Error decompressing sub from opensubtitles. Message is: %s' % error)
return None
if not CompressedData:
log.debug('DownloadSub: No data returned from DownloadSubtitles API call. Skipping this one.')
return None
SubDataBytes = gzip.GzipFile(fileobj=io.BytesIO(CompressedData)).read()
# Opensubtitles makes no difference in UTF-8 and UTF8-SIG so we check with chardet the correct encoding
# if Opensubtile does not know the encoding we assume windows-1252 is used.
if SubCodec:
if 'UTF' in SubCodec.upper() or SubCodec == 'Unknown':
SubCodec = detect(SubDataBytes)['encoding']
if not SubCodec:
SubCodec = u'windows-1252'
else:
SubCodec = u'windows-1252'
try:
SubData = SubDataBytes.decode(SubCodec,errors='replace')
except Exception as error:
log.error('downloadSubs: Error decoding sub from opensubtitles. Message is: %s' % error)
return None
return(SubData)
else:
if Result['status'][:3] == '407':
autosub.OPENSUBTITLESTOKEN = 'limit'
log.error('Opensubtitles: Error from Opensubtitles downloadsubs API. Message : %s' % Result['status'])
return None
def walkDir(path):
SkipListNL = autosub.SKIPSTRINGNL.split(",") if len(autosub.SKIPSTRINGNL) > 0 else []
SkipListEN = autosub.SKIPSTRINGEN.split(",") if len(autosub.SKIPSTRINGEN) > 0 else []
if len(autosub.SKIPFOLDERSNL) == 0:
SkipFoldersNL = []
else:
SkipFoldersNL = autosub.SKIPFOLDERSNL.split(",") if len(autosub.SKIPFOLDERSNL) > 0 else []
for idx,folder in enumerate(SkipFoldersNL):
SkipFoldersNL[idx] = os.path.normpath(os.path.join(path,folder.rstrip("\/").lstrip("\/")))
if len(autosub.SKIPFOLDERSNL) == 0:
SkipFoldersNL = []
else:
SkipFoldersNL = autosub.SKIPFOLDERSNL.split(",") if len(autosub.SKIPFOLDERSNL) > 0 else []
for idx,folder in enumerate(SkipFoldersNL):
SkipFoldersNL[idx] = os.path.normpath(path + folder)
if len(autosub.SKIPFOLDERSEN) == 0:
SkipFoldersEN = []
else:
SkipFoldersEN = autosub.SKIPFOLDERSEN.split(",") if len(autosub.SKIPFOLDERSEN) > 0 else []
for idx,folder in enumerate(SkipFoldersEN):
SkipFoldersEN[idx] = os.path.normpath(path + folder)
for dirname, dirnames, filenames in os.walk(path, True, WalkError):
SkipThisFolderNL = False
for skip in SkipFoldersNL:
if dirname.startswith(skip):
SkipThisFolderNL = True
break
SkipThisFolderEN = False
for skip in SkipFoldersEN:
if dirname.startswith(skip):
SkipThisFolderEN = True
break
log.debug("scanDisk: directory name: %s" %dirname)
if re.search('_unpack_', dirname, re.IGNORECASE):
log.debug("scanDisk: found a unpack directory, skipping.")
continue
if autosub.SKIPHIDDENDIRS and os.path.split(dirname)[1].startswith(u'.'):
continue
if re.search('_failed_', dirname, re.IGNORECASE):
log.debug("scanDisk: found a failed directory, skipping.")
continue
if re.search('@eaDir', dirname, re.IGNORECASE):
log.debug("scanDisk: found a Synology indexing directory, skipping.")
tmpdirs = dirnames[:]
for dir in tmpdirs:
dirnames.remove(dir)
continue
if re.search("@.*thumb", dirname, re.IGNORECASE):
log.debug("scanDisk: found a Qnap multimedia thumbnail folder, skipping.")
continue
langs = []
FileDict = {}
for filename in filenames:
root,ext = os.path.splitext(filename)
if ext[1:] in ('avi', 'mkv', 'wmv', 'ts', 'mp4'):
if re.search('sample', filename):
continue
if not platform.system() == 'Windows':
# Get best ascii compatible character for special characters
try:
if not isinstance(filename, unicode):
coding = detect(filename)['encoding']
filename = unicode(filename.decode(coding),errors='replace')
correctedFilename = ''.join((c for c in unicodedata.normalize('NFD', filename) if unicodedata.category(c) != 'Mn'))
if filename != correctedFilename:
os.rename(os.path.join(dirname, filename), os.path.join(dirname, correctedFilename))
log.info("scanDir: Renamed file %s" % correctedFilename)
filename = correctedFilename
except:
log.error("scanDir: Skipping directory, file %s, %s" % (dirname,filename))
continue
# What subtitle files should we expect?
langs = []
NLext = u'.' + autosub.SUBNL + u'.srt' if autosub.SUBNL else u'.srt'
ENext = u'.' + autosub.SUBENG + u'.srt' if autosub.SUBENG else u'.srt'
ENext = u'.en.srt'if NLext == ENext and autosub.DOWNLOADDUTCH else ENext
if not os.access(dirname, os.W_OK):
log.error('scandisk: No write access to folder: %s' % dirname)
continue
# Check which languages we want to download based on user settings.
if autosub.DOWNLOADDUTCH and not SkipThisFolderNL:
Skipped = False
for SkipItem in SkipListNL:
if not SkipItem: break
if re.search(SkipItem.lower(), filename.lower()):
Skipped = True
break
if Skipped:
log.info("scanDir: %s found in %s so skipped for Dutch subs" % (SkipItem, filename))
elif os.path.exists(os.path.join(dirname, root + NLext)):
Skipped = True
log.debug("scanDir: %s skipped because the Dutch subtitle already exists" % filename)
else:
# If the Dutch subtitle not skipped and doesn't exist, then add it to the wanted list
langs.append(autosub.DUTCH)
if (autosub.DOWNLOADENG or (autosub.FALLBACKTOENG and autosub.DOWNLOADDUTCH and not Skipped)) and not SkipThisFolderEN:
Skipped = False
for SkipItem in SkipListEN:
if not SkipItem: break
if re.search(SkipItem.lower(), filename.lower()):
Skipped = True
break
if Skipped:
log.info("scanDir: %s found in %s so skipped for English subs" % (SkipItem, filename))
elif os.path.exists(os.path.join(dirname, root + ENext)):
log.debug("scanDir: %s skipped because the English subtitle already exists" % filename)
else:
# If the English subtitle not skipped and doesn't exist, then add it to the wanted list
if not os.path.exists(os.path.join(dirname, root + ENext)):
langs.append(autosub.ENGLISH)
if not langs:
# nothing to do for this file
continue
FileDict = ProcessFilename(os.path.splitext(filename)[0].strip(), ext)
time.sleep(0)
if not FileDict:
continue
if not 'title' in FileDict.keys() or not 'season' in FileDict.keys() or not 'episode' in FileDict.keys():
continue
if not FileDict['releasegrp'] and not FileDict['source'] and not FileDict['quality'] and not FileDict['source']:
log.error("scanDir: Not enough info in filename: %s" % filename)
continue
FileDict['timestamp'] = unicode(time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(os.path.getctime(os.path.join(dirname, filename)))))
FileDict['langs'] = langs
FileDict['NLext'] = NLext
FileDict['ENext'] = ENext
FileDict['file'] = root
FileDict['container'] = ext
FileDict['folder'] = dirname
FileDict['ImdbId'],FileDict['A7Id'], FileDict['TvdbId'], FileDict['title'] = Helpers.getShowid(FileDict['title'],autosub.ADDIC7EDLOGGED_IN)
if autosub.Helpers.SkipShow(FileDict['ImdbId'],FileDict['title'], FileDict['season'], FileDict['episode']):
continue
log.info("scanDir: %s WANTED FOR: %s" % (langs, filename))
autosub.WANTEDQUEUE.append(FileDict)
return
def walkDir(path):
SkipListNL = []
SkipListEN = []
SkipListNL = autosub.SKIPSTRINGNL.split(",")
SkipListEN = autosub.SKIPSTRINGEN.split(",")
# Here we use os.walk to find all files in the path.
for dirname, dirnames, filenames in os.walk(path):
log.debug("scanDisk: directory name: %s" % dirname)
if re.search('_unpack_', dirname, re.IGNORECASE):
log.debug("scanDisk: found a unpack directory, skipping")
continue
if autosub.SKIPHIDDENDIRS and os.path.split(dirname)[1].startswith(
u'.'):
continue
if re.search('_failed_', dirname, re.IGNORECASE):
log.debug("scanDisk: found a failed directory, skipping")
continue
if re.search('@eaDir', dirname, re.IGNORECASE):
log.debug(
"scanDisk: found a Synology indexing directory, skipping this folder and all subfolders."
)
tmpdirs = dirnames[:]
for dir in tmpdirs:
dirnames.remove(dir)
continue
if re.search("@.*thumb", dirname, re.IGNORECASE):
# Ingnore QNAP multimedia thumbnail folders
continue
for filename in filenames:
splitname = filename.split(".")
ext = splitname[len(splitname) - 1]
if ext in ('avi', 'mkv', 'wmv', 'ts', 'mp4'):
if re.search('sample', filename): continue
if not platform.system() == 'Windows':
# Get best ascii compatible character for special characters
try:
if not isinstance(filename, unicode):
coding = detect(filename)
filename = unicode(filename.decode(coding))
correctedFilename = ''.join(
(c for c in unicodedata.normalize('NFD', filename)
if unicodedata.category(c) != 'Mn'))
if filename != correctedFilename:
os.rename(os.path.join(dirname, filename),
os.path.join(dirname, correctedFilename))
log.info("scanDir: Renamed file %s" %
correctedFilename)
filename = correctedFilename
except:
log.error("scanDir: Skipping directory %s" % dirname)
log.error("scanDir: Skipping file %s" % filename)
continue
# What subtitle files should we expect?
lang = []
#Check what the Dutch subtitle would be.
if autosub.SUBNL != "":
srtfilenl = os.path.splitext(
filename)[0] + u"." + autosub.SUBNL + u".srt"
else:
srtfilenl = os.path.splitext(filename)[0] + u".srt"
#Check what the English subtitle would be.
if autosub.SUBENG == "":
# Check for overlapping names
if autosub.SUBNL != "" or not autosub.DOWNLOADDUTCH:
srtfileeng = os.path.splitext(filename)[0] + u".srt"
# Hardcoded fallback
else:
srtfileeng = os.path.splitext(filename)[0] + u".en.srt"
else:
srtfileeng = os.path.splitext(
filename)[0] + u"." + autosub.SUBENG + u".srt"
# Check which languages we want to download based on user settings and check the skipstring
if autosub.DOWNLOADDUTCH:
Skipped = False
for SkipItem in SkipListNL:
if re.search(SkipItem.lower(), filename.lower()):
Skipped = True
log.debug(
"scanDir: %s found in %s so skipped for Dutch subs"
% (SkipItem, filename))
break
if not Skipped and os.path.exists(
os.path.join(dirname, srtfilenl)):
Skipped = True
log.debug(
"scanDir: %s skipped because the subtitle already exists"
% filename)
# If the Dutch subtitle doesn't exist en has no skip item, then add it to the wanted list.
if not Skipped:
lang.append(autosub.DUTCH)
if autosub.DOWNLOADENG:
Skipped = False
for SkipItem in SkipListEN:
if re.search(SkipItem.lower(), filename.lower()):
Skipped = True
log.debug(
"scanDir: %s found in %s so skipped for English subs"
% (SkipItem, filename))
break
if not Skipped and os.path.exists(
os.path.join(dirname, srtfileeng)):
Skipped = True
log.debug(
"scanDir: %s skipped because the subtitle already exists"
% filename)
# If the English subtitle doesn't exist en has no skip item, then add it to the wanted list.
if not Skipped:
lang.append(autosub.ENGLISH)
if (autosub.FALLBACKTOENG and autosub.DOWNLOADDUTCH
) and not autosub.DOWNLOADENG and not Skipped:
# If the Dutch and English subtitles do not exist and not skipped, then add English to the wanted list.
if not os.path.exists(os.path.join(
dirname, srtfilenl)) and not os.path.exists(
os.path.join(dirname, srtfileeng)):
lang.append(autosub.ENGLISH)
if not lang:
# autosub.WANTEDQUEUE empty
continue
log.debug("scanDir: File %s is missing subtitle(s): %s" %
(filename, ', '.join(map(str, lang))))
filenameResults = ProcessFilename(
os.path.splitext(filename)[0],
os.path.splitext(filename)[1])
if 'title' in filenameResults.keys():
if 'season' in filenameResults.keys():
if 'episode' in filenameResults.keys():
title = filenameResults['title']
season = filenameResults['season']
episode = filenameResults['episode']
if not filenameResults[
'releasegrp'] and not filenameResults[
'source'] and not filenameResults[
'quality'] and not filenameResults[
'source']:
continue
if autosub.Helpers.SkipShow(
title, season, episode) == True:
log.debug("scanDir: SkipShow returned True")
log.info(
"scanDir: Skipping %s - Season %s Episode %s"
% (title, season, episode))
continue
if len(lang) == 1:
log.info(
"scanDir: %s subtitle wanted for %s and added to wantedQueue"
% (lang[0], filename))
else:
log.info(
"scanDir: %s subtitles wanted for %s and added to wantedQueue"
% (' and '.join(map(str, lang)), filename))
filenameResults[
'originalFileLocationOnDisk'] = os.path.join(
dirname, filename)
filenameResults['timestamp'] = unicode(
time.strftime(
'%Y-%m-%d %H:%M:%S',
time.localtime(
os.path.getctime(filenameResults[
'originalFileLocationOnDisk']))))
filenameResults['lang'] = lang
autosub.WANTEDQUEUE.append(filenameResults)
else:
log.error(
"scanDir: Could not process the filename properly filename: %s"
% filename)
continue
else:
log.error(
"scanDir: Could not process the filename properly filename: %s"
% filename)
continue
else:
log.error(
"scanDir: Could not process the filename properly filename: %s"
% filename)
continue
def walkDir(path):
SkipListNL = autosub.SKIPSTRINGNL.split(",") if len(
autosub.SKIPSTRINGNL) > 0 else []
SkipListEN = autosub.SKIPSTRINGEN.split(",") if len(
autosub.SKIPSTRINGEN) > 0 else []
# Check for dutch folder skip
if len(autosub.SKIPFOLDERSNL) == 0:
SkipFoldersNL = []
else:
SkipFoldersNL = autosub.SKIPFOLDERSNL.split(",") if len(
autosub.SKIPFOLDERSNL) > 0 else []
for idx, folder in enumerate(SkipFoldersNL):
SkipFoldersNL[idx] = os.path.normpath(
os.path.join(path, folder.strip(" \/")))
# Check for english folder skip
if len(autosub.SKIPFOLDERSEN) == 0:
SkipFoldersEN = []
else:
SkipFoldersEN = autosub.SKIPFOLDERSEN.split(",") if len(
autosub.SKIPFOLDERSEN) > 0 else []
for idx, folder in enumerate(SkipFoldersEN):
SkipFoldersEN[idx] = os.path.normpath(
os.path.join(path, folder.strip(" \/")))
for dirname, dirnames, filenames in os.walk(path, True, WalkError):
SkipThisFolderNL = False
for skip in SkipFoldersNL:
if dirname.startswith(skip):
SkipThisFolderNL = True
break
SkipThisFolderEN = False
for skip in SkipFoldersEN:
if dirname.startswith(skip):
SkipThisFolderEN = True
break
log.debug("scanDisk: directory name: %s" % dirname)
if re.search('_unpack_', dirname, re.IGNORECASE):
log.debug("scanDisk: found a unpack directory, skipping.")
continue
if autosub.SKIPHIDDENDIRS and os.path.split(dirname)[1].startswith(
u'.'):
continue
if re.search('_failed_', dirname, re.IGNORECASE):
log.debug("scanDisk: found a failed directory, skipping.")
continue
if re.search('@eaDir', dirname, re.IGNORECASE):
log.debug(
"scanDisk: found a Synology indexing directory, skipping.")
tmpdirs = dirnames[:]
for dir in tmpdirs:
dirnames.remove(dir)
continue
if re.search("@.*thumb", dirname, re.IGNORECASE):
log.debug(
"scanDisk: found a Qnap multimedia thumbnail folder, skipping."
)
continue
langs = []
FileDict = {}
for filename in filenames:
if autosub.SEARCHSTOP:
log.info('scanDisk: Forced Stop by user')
return
try:
root, ext = os.path.splitext(filename)
if ext[1:] in ('avi', 'mkv', 'wmv', 'ts', 'mp4'):
if re.search('sample', filename):
continue
if not platform.system() == 'Windows':
# Get best ascii compatible character for special characters
try:
if not isinstance(filename, unicode):
coding = detect(filename)['encoding']
filename = unicode(filename.decode(coding),
errors='replace')
correctedFilename = ''.join(
(c for c in unicodedata.normalize(
'NFD', filename)
if unicodedata.category(c) != 'Mn'))
if filename != correctedFilename:
os.rename(
os.path.join(dirname, filename),
os.path.join(dirname, correctedFilename))
log.info("scanDir: Renamed file %s" %
correctedFilename)
filename = correctedFilename
except:
log.error(
"scanDir: Skipping directory, file %s, %s" %
(dirname, filename))
continue
# What subtitle files should we expect?
langs = []
NLext = u'.' + autosub.SUBNL + u'.srt' if autosub.SUBNL else u'.srt'
ENext = u'.' + autosub.SUBENG + u'.srt' if autosub.SUBENG else u'.srt'
ENext = u'.en.srt' if NLext == ENext and autosub.DOWNLOADDUTCH else ENext
if not os.access(dirname, os.W_OK):
log.error('scandisk: No write access to folder: %s' %
dirname)
continue
# Check which languages we want to download based on user settings.
log.debug('scanDir: Processing file: %s' % filename)
if autosub.DOWNLOADDUTCH and not SkipThisFolderNL:
Skipped = False
for SkipItem in SkipListNL:
if not SkipItem: break
if re.search(SkipItem.lower(), filename.lower()):
Skipped = True
break
if Skipped:
log.info(
"scanDir: %s found in %s so skipped for Dutch subs"
% (SkipItem, filename))
elif os.path.exists(os.path.join(
dirname, root + NLext)):
Skipped = True
log.debug(
"scanDir: %s skipped because the Dutch subtitle already exists"
% filename)
else:
# If the Dutch subtitle not skipped and doesn't exist, then add it to the wanted list
langs.append(autosub.DUTCH)
if (autosub.DOWNLOADENG or
(autosub.FALLBACKTOENG and autosub.DOWNLOADDUTCH
and not Skipped)) and not SkipThisFolderEN:
Skipped = False
for SkipItem in SkipListEN:
if not SkipItem: break
if re.search(SkipItem.lower(), filename.lower()):
Skipped = True
break
if Skipped:
log.info(
"scanDir: %s found in %s so skipped for English subs"
% (SkipItem, filename))
elif os.path.exists(os.path.join(
dirname, root + ENext)):
log.debug(
"scanDir: %s skipped because the English subtitle already exists"
% filename)
else:
# If the English subtitle not skipped and doesn't exist, then add it to the wanted list
if not os.path.exists(
os.path.join(dirname, root + ENext)):
langs.append(autosub.ENGLISH)
if not langs:
# nothing to do for this file
continue
FileDict = ProcessFilename(
os.path.splitext(filename)[0].strip(), ext)
if not FileDict:
log.debug(
'scanDisk: not enough info in the filename: %s' %
filename)
continue
Skip = False
if autosub.MINMATCHSCORE & 8 and not FileDict['source']:
Skip = True
elif autosub.MINMATCHSCORE & 4 and not FileDict['quality']:
Skip = True
elif autosub.MINMATCHSCORE & 2 and not FileDict['codec']:
Skip = True
elif autosub.MINMATCHSCORE & 1 and not FileDict[
'releasegrp']:
Skip = True
if Skip:
log.debug(
'scanDisk: Filespec does not meet minmatchscore so skipping this one'
)
continue
FileDict['timestamp'] = unicode(
time.strftime(
'%Y-%m-%d %H:%M:%S',
time.localtime(
os.path.getmtime(
os.path.join(dirname, filename)))))
FileDict['langs'] = langs
FileDict['NLext'] = NLext
FileDict['ENext'] = ENext
FileDict['file'] = root
FileDict['container'] = ext
FileDict['folder'] = dirname
FileDict['ImdbId'], FileDict['A7Id'], FileDict[
'TvdbId'], FileDict['title'] = Helpers.getShowid(
FileDict['title'])
if autosub.Helpers.SkipShow(FileDict['ImdbId'],
FileDict['title'],
FileDict['season'],
FileDict['episode']):
log.debug("scanDir: SKIPPED %s by Skipshow rules." %
FileDict['file'])
continue
log.info("scanDir: %s WANTED FOR: %s" % (langs, filename))
autosub.WANTEDQUEUE.append(FileDict)
time.sleep(0)
except Exception as error:
log.error('scanDir: Problem scanning file %s. Error is: %s' %
(filename, error))
return
def walkDir(path):
SkipListNL = []
SkipListEN = []
SkipListNL = autosub.SKIPSTRINGNL.split(",")
SkipListEN = autosub.SKIPSTRINGEN.split(",")
# Here we use os.walk to find all files in the path.
for dirname, dirnames, filenames in os.walk(path):
log.debug("scanDisk: directory name: %s" %dirname)
if re.search('_unpack_', dirname, re.IGNORECASE):
log.debug("scanDisk: found a unpack directory, skipping")
continue
if autosub.SKIPHIDDENDIRS and os.path.split(dirname)[1].startswith(u'.'):
continue
if re.search('_failed_', dirname, re.IGNORECASE):
log.debug("scanDisk: found a failed directory, skipping")
continue
if re.search('@eaDir', dirname, re.IGNORECASE):
log.debug("scanDisk: found a Synology indexing directory, skipping this folder and all subfolders.")
tmpdirs = dirnames[:]
for dir in tmpdirs:
dirnames.remove(dir)
continue
if re.search("@.*thumb", dirname, re.IGNORECASE):
# Ingnore QNAP multimedia thumbnail folders
continue
for filename in filenames:
splitname = filename.split(".")
ext = splitname[len(splitname) - 1]
if ext in ('avi', 'mkv', 'wmv', 'ts', 'mp4'):
if re.search('sample', filename): continue
if not platform.system() == 'Windows':
# Get best ascii compatible character for special characters
try:
if not isinstance(filename, unicode):
coding = detect(filename)
filename = unicode(filename.decode(coding))
correctedFilename = ''.join((c for c in unicodedata.normalize('NFD', filename) if unicodedata.category(c) != 'Mn'))
if filename != correctedFilename:
os.rename(os.path.join(dirname, filename), os.path.join(dirname, correctedFilename))
log.info("scanDir: Renamed file %s" % correctedFilename)
filename = correctedFilename
except:
log.error("scanDir: Skipping directory %s" % dirname)
log.error("scanDir: Skipping file %s" % filename)
continue
# What subtitle files should we expect?
lang=[]
#Check what the Dutch subtitle would be.
if autosub.SUBNL != "":
srtfilenl = os.path.splitext(filename)[0] + u"." + autosub.SUBNL + u".srt"
else:
srtfilenl = os.path.splitext(filename)[0] + u".srt"
#Check what the English subtitle would be.
if autosub.SUBENG == "":
# Check for overlapping names
if autosub.SUBNL != "" or not autosub.DOWNLOADDUTCH:
srtfileeng = os.path.splitext(filename)[0] + u".srt"
# Hardcoded fallback
else:
srtfileeng = os.path.splitext(filename)[0] + u".en.srt"
else:
srtfileeng = os.path.splitext(filename)[0] + u"." + autosub.SUBENG + u".srt"
# Check which languages we want to download based on user settings and check the skipstring
if autosub.DOWNLOADDUTCH:
Skipped = False
for SkipItem in SkipListNL:
if re.search(SkipItem.lower(), filename.lower()):
Skipped = True
log.debug("scanDir: %s found in %s so skipped for Dutch subs" % (SkipItem, filename))
break
if not Skipped and os.path.exists(os.path.join(dirname, srtfilenl)):
Skipped = True
log.debug("scanDir: %s skipped because the subtitle already exists" % filename)
# If the Dutch subtitle doesn't exist en has no skip item, then add it to the wanted list.
if not Skipped:
lang.append(autosub.DUTCH)
if autosub.DOWNLOADENG:
Skipped = False
for SkipItem in SkipListEN:
if re.search(SkipItem.lower(), filename.lower()):
Skipped = True
log.debug("scanDir: %s found in %s so skipped for English subs" % (SkipItem, filename))
break
if not Skipped and os.path.exists(os.path.join(dirname, srtfileeng)):
Skipped = True
log.debug("scanDir: %s skipped because the subtitle already exists" % filename)
# If the English subtitle doesn't exist en has no skip item, then add it to the wanted list.
if not Skipped:
lang.append(autosub.ENGLISH)
if (autosub.FALLBACKTOENG and autosub.DOWNLOADDUTCH) and not autosub.DOWNLOADENG and not Skipped:
# If the Dutch and English subtitles do not exist and not skipped, then add English to the wanted list.
if not os.path.exists(os.path.join(dirname, srtfilenl)) and not os.path.exists(os.path.join(dirname, srtfileeng)):
lang.append(autosub.ENGLISH)
if not lang:
# autosub.WANTEDQUEUE empty
continue
log.debug("scanDir: File %s is missing subtitle(s): %s" % (filename, ', '.join(map(str,lang))))
filenameResults = ProcessFilename(os.path.splitext(filename)[0], os.path.splitext(filename)[1])
if 'title' in filenameResults.keys():
if 'season' in filenameResults.keys():
if 'episode' in filenameResults.keys():
title = filenameResults['title']
season = filenameResults['season']
episode = filenameResults['episode']
if not filenameResults['releasegrp'] and not filenameResults['source'] and not filenameResults['quality'] and not filenameResults['source']:
continue
if autosub.Helpers.SkipShow(title, season, episode) == True:
log.debug("scanDir: SkipShow returned True")
log.info("scanDir: Skipping %s - Season %s Episode %s" % (title, season, episode))
continue
if len(lang) == 1:
log.info("scanDir: %s subtitle wanted for %s and added to wantedQueue" % (lang[0], filename))
else:
log.info("scanDir: %s subtitles wanted for %s and added to wantedQueue" % (' and '.join(map(str,lang)), filename))
filenameResults['originalFileLocationOnDisk'] = os.path.join(dirname, filename)
filenameResults['timestamp'] = unicode(time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(os.path.getctime(filenameResults['originalFileLocationOnDisk']))))
filenameResults['lang'] = lang
autosub.WANTEDQUEUE.append(filenameResults)
else:
log.error("scanDir: Could not process the filename properly filename: %s" % filename)
continue
else:
log.error("scanDir: Could not process the filename properly filename: %s" % filename)
continue
else:
log.error("scanDir: Could not process the filename properly filename: %s" % filename)
continue
def walkDir(path):
SkipListFR = autosub.SKIPSTRINGFR.split(",") if len(autosub.SKIPSTRINGFR) > 0 else []
SkipListEN = autosub.SKIPSTRINGEN.split(",") if len(autosub.SKIPSTRINGEN) > 0 else []
# Check for dutch folder skip
if len(autosub.SKIPFOLDERSFR) == 0:
SkipFoldersFR = []
else:
SkipFoldersFR = autosub.SKIPFOLDERSFR.split(",") if len(autosub.SKIPFOLDERSFR) > 0 else []
for idx,folder in enumerate(SkipFoldersFR):
SkipFoldersFR[idx] = os.path.normpath(os.path.join(path,folder.strip(" \/")))
# Check for english folder skip
if len(autosub.SKIPFOLDERSEN) == 0:
SkipFoldersEN = []
else:
SkipFoldersEN = autosub.SKIPFOLDERSEN.split(",") if len(autosub.SKIPFOLDERSEN) > 0 else []
for idx,folder in enumerate(SkipFoldersEN):
SkipFoldersEN[idx] = os.path.normpath(os.path.join(path,folder.strip(" \/")))
for dirname, dirnames, filenames in os.walk(path, True, WalkError):
#filenames = [decodeName(f) for f in filenames]
SkipThisFolderFR = False
for skip in SkipFoldersFR:
if dirname.startswith(skip):
SkipThisFolderFR = True
break
SkipThisFolderEN = False
for skip in SkipFoldersEN:
if dirname.startswith(skip):
SkipThisFolderEN = True
break
log.debug("scanDisk: directory name: %s" %dirname)
if re.search('_unpack_', dirname, re.IGNORECASE):
log.debug("scanDisk: found a unpack directory, skipping.")
continue
if autosub.SKIPHIDDENDIRS and os.path.split(dirname)[1].startswith(u'.'):
continue
if re.search('_failed_', dirname, re.IGNORECASE):
log.debug("scanDisk: found a failed directory, skipping.")
continue
if re.search('@eaDir', dirname, re.IGNORECASE):
log.debug("scanDisk: found a Synology indexing directory, skipping.")
tmpdirs = dirnames[:]
for dir in tmpdirs:
dirnames.remove(dir)
continue
if re.search("@.*thumb", dirname, re.IGNORECASE):
log.debug("scanDisk: found a Qnap multimedia thumbnail folder, skipping.")
continue
langs = []
FileDict = {}
for filename in filenames:
if autosub.SEARCHSTOP:
log.info('scanDisk: Forced Stop by user')
return
try:
root,ext = os.path.splitext(filename)
if ext[1:] in ('avi', 'mkv', 'wmv', 'ts', 'mp4'):
if re.search('sample', filename):
continue
if not platform.system() == 'Windows':
# Get best ascii compatible character for special characters
try:
if not isinstance(filename, unicode):
coding = detect(filename)['encoding']
filename = unicode(filename.decode(coding),errors='replace')
correctedFilename = ''.join((c for c in unicodedata.normalize('NFD', filename) if unicodedata.category(c) != 'Mn'))
if filename != correctedFilename:
os.rename(os.path.join(dirname, filename), os.path.join(dirname, correctedFilename))
log.info("scanDir: Renamed file %s" % correctedFilename)
filename = correctedFilename
except:
log.error("scanDir: Skipping directory, file %s, %s" % (dirname,filename))
continue
# What subtitle files should we expect?
langs = []
FRext = u'.' + autosub.SUBFR + u'.srt' if autosub.SUBFR else u'.srt'
ENext = u'.' + autosub.SUBENG + u'.srt' if autosub.SUBENG else u'.srt'
ENext = u'.en.srt'if FRext == ENext and autosub.DOWNLOADFRENCH else ENext
if not os.access(dirname, os.W_OK):
log.error('scandisk: No write access to folder: %s' % dirname)
continue
# Check which languages we want to download based on user settings.
log.debug('scanDir: Processing file: %s' % filename)
if autosub.DOWNLOADFRENCH and not SkipThisFolderFR:
Skipped = False
for SkipItem in SkipListFR:
if not SkipItem: break
if re.search(SkipItem.lower(), filename.lower()):
Skipped = True
break
if Skipped:
log.info("scanDir: %s found in %s so skipped for French subs" % (SkipItem, filename))
elif os.path.exists(os.path.join(dirname, root + FRext)):
Skipped = True
log.debug("scanDir: %s skipped because the French subtitle already exists" % filename)
else:
# If the French subtitle not skipped and doesn't exist, then add it to the wanted list
langs.append(autosub.FRENCH)
if (autosub.DOWNLOADENG or (autosub.FALLBACKTOENG and autosub.DOWNLOADFRENCH and not Skipped)) and not SkipThisFolderEN:
Skipped = False
for SkipItem in SkipListEN:
if not SkipItem: break
if re.search(SkipItem.lower(), filename.lower()):
Skipped = True
break
if Skipped:
log.info("scanDir: %s found in %s so skipped for English subs" % (SkipItem, filename))
elif os.path.exists(os.path.join(dirname, root + ENext)):
log.debug("scanDir: %s skipped because the English subtitle already exists" % filename)
else:
# If the English subtitle not skipped and doesn't exist, then add it to the wanted list
if not os.path.exists(os.path.join(dirname, root + ENext)):
langs.append(autosub.ENGLISH)
if not langs:
# nothing to do for this file
continue
FileDict = ProcessFilename(os.path.splitext(filename)[0].strip(), ext)
if not FileDict:
log.debug('scanDisk: not enough info in the filename: %s' % filename)
continue
Skip = False
if autosub.MINMATCHSCORE & 8 and not FileDict['source'] : Skip = True
elif autosub.MINMATCHSCORE & 4 and not FileDict['quality'] : Skip = True
elif autosub.MINMATCHSCORE & 2 and not FileDict['codec'] : Skip = True
elif autosub.MINMATCHSCORE & 1 and not FileDict['releasegrp']: Skip = True
if Skip:
log.debug('scanDisk: Filespec does not meet minmatchscore so skipping this one')
FileDict['timestamp'] = unicode(time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(os.path.getctime(os.path.join(dirname, filename)))))
FileDict['langs'] = langs
FileDict['FRext'] = FRext
FileDict['ENext'] = ENext
FileDict['file'] = root
FileDict['container'] = ext
FileDict['folder'] = dirname
FileDict['ImdbId'],FileDict['A7Id'], FileDict['TvdbId'], FileDict['title'] = Helpers.getShowid(FileDict['title'])
if autosub.Helpers.SkipShow(FileDict['ImdbId'],FileDict['title'], FileDict['season'], FileDict['episode']):
log.debug("scanDir: SKIPPED %s by Skipshow rules." % FileDict['file'])
continue
log.info("scanDir: %s WANTED FOR: %s" % (langs, filename))
autosub.WANTEDQUEUE.append(FileDict)
time.sleep(0)
except Exception as error:
log.error('scanDir: Problem scanning file %s. Error is: %s' %(filename, error))
return
def walkDir(path):
for dirname, dirnames, filenames in os.walk(path):
log.debug("scanDisk: directory name: %s" % dirname)
if re.search("_unpack_", dirname, re.IGNORECASE):
log.debug("scanDisk: found a unpack directory, skipping")
continue
if autosub.SKIPHIDDENDIRS and os.path.split(dirname)[1].startswith(u"."):
continue
if re.search("_failed_", dirname, re.IGNORECASE):
log.debug("scanDisk: found a failed directory, skipping")
continue
if re.search("@eaDir", dirname, re.IGNORECASE):
log.debug("scanDisk: found a Synology indexing directory, skipping this folder and all subfolders.")
tmpdirs = dirnames[:]
for dir in tmpdirs:
dirnames.remove(dir)
continue
if re.search("@.*thumb", dirname, re.IGNORECASE):
# Ingnore QNAP multimedia thumbnail folders
continue
for filename in filenames:
splitname = filename.split(".")
ext = splitname[len(splitname) - 1]
if ext in ("avi", "mkv", "wmv", "ts", "mp4"):
if re.search("sample", filename):
continue
if autosub.WEBDL == "None":
if re.search("web-dl", filename.lower()):
log.debug("scanDisk: WEB-DL is set to 'None', skipping %s" % filename)
continue
if not platform.system() == "Windows":
# Get best ascii compatible character for special characters
try:
if not isinstance(filename, unicode):
coding = detect(filename)
filename = unicode(filename.decode(coding))
correctedFilename = "".join(
(c for c in unicodedata.normalize("NFD", filename) if unicodedata.category(c) != "Mn")
)
if filename != correctedFilename:
os.rename(os.path.join(dirname, filename), os.path.join(dirname, correctedFilename))
log.info("scanDir: Renamed file %s" % correctedFilename)
filename = correctedFilename
except:
log.error("scanDir: Skipping directory %s" % dirname)
log.error("scanDir: Skipping file %s" % filename)
continue
# What subtitle files should we expect?
lang = []
# Check what the Dutch subtitle would be.
if autosub.SUBNL != "":
srtfilenl = os.path.splitext(filename)[0] + u"." + autosub.SUBNL + u".srt"
else:
srtfilenl = os.path.splitext(filename)[0] + u".srt"
# Check what the English subtitle would be.
if autosub.SUBENG == "":
# Check for overlapping names
if autosub.SUBNL != "" or not autosub.DOWNLOADDUTCH:
srtfileeng = os.path.splitext(filename)[0] + u".srt"
# Hardcoded fallback
else:
srtfileeng = os.path.splitext(filename)[0] + u".en.srt"
else:
srtfileeng = os.path.splitext(filename)[0] + u"." + autosub.SUBENG + u".srt"
# Check which languages we want to download based on user settings.
if autosub.DOWNLOADDUTCH:
# If the Dutch subtitle doesn't exist, then add it to the wanted list.
if not os.path.exists(os.path.join(dirname, srtfilenl)):
lang.append(autosub.DUTCH)
if autosub.DOWNLOADENG:
# If the English subtitle doesn't exist, then add it to the wanted list.
if not os.path.exists(os.path.join(dirname, srtfileeng)):
if autosub.WEBDL == "DutchOnly" and re.search("web-dl", filename.lower()):
log.debug(
"scanDisk: WEB-DL is set to 'Dutch Only', not adding English as wanted for %s"
% filename
)
else:
lang.append(autosub.ENGLISH)
if (autosub.FALLBACKTOENG and autosub.DOWNLOADDUTCH) and not autosub.DOWNLOADENG:
# If the Dutch and English subtitles do not exist, then add English to the wanted list.
if not os.path.exists(os.path.join(dirname, srtfilenl)) and not os.path.exists(
os.path.join(dirname, srtfileeng)
):
if autosub.WEBDL == "DutchOnly" and re.search("web-dl", filename.lower()):
log.debug(
"scanDisk: WEB-DL is set to 'Dutch Only', not adding English as wanted for %s"
% filename
)
else:
lang.append(autosub.ENGLISH)
if not lang:
# autosub.WANTEDQUEUE empty
continue
log.debug("scanDir: File %s is missing subtitle(s): %s" % (filename, ", ".join(map(str, lang))))
filenameResults = ProcessFilename(os.path.splitext(filename)[0], os.path.splitext(filename)[1])
if "title" in filenameResults.keys():
if "season" in filenameResults.keys():
if "episode" in filenameResults.keys():
title = filenameResults["title"]
season = filenameResults["season"]
episode = filenameResults["episode"]
if (
not filenameResults["releasegrp"]
and not filenameResults["source"]
and not filenameResults["quality"]
and not filenameResults["source"]
):
continue
if autosub.Helpers.SkipShow(title, season, episode) == True:
log.debug("scanDir: SkipShow returned True")
log.info("scanDir: Skipping %s - Season %s Episode %s" % (title, season, episode))
continue
if len(lang) == 1:
log.info(
"scanDir: %s subtitle wanted for %s and added to wantedQueue" % (lang[0], filename)
)
else:
log.info(
"scanDir: %s subtitles wanted for %s and added to wantedQueue"
% (" and ".join(map(str, lang)), filename)
)
filenameResults["originalFileLocationOnDisk"] = os.path.join(dirname, filename)
filenameResults["timestamp"] = unicode(
time.strftime(
"%Y-%m-%d %H:%M:%S",
time.localtime(os.path.getctime(filenameResults["originalFileLocationOnDisk"])),
)
)
filenameResults["lang"] = lang
autosub.WANTEDQUEUE.append(filenameResults)
else:
log.error("scanDir: Could not process the filename properly filename: %s" % filename)
continue
else:
log.error("scanDir: Could not process the filename properly filename: %s" % filename)
continue
else:
log.error("scanDir: Could not process the filename properly filename: %s" % filename)
continue
