def SubtitleSeeker(Wanted, sourceWebsites): # Get the scored list for all SubtitleSeeker hits ScoreListNL, ScoreListEN = [],[] log.debug('getSubLinks: SubtitlesSeeker search started for %s on sites: %s ' % (Wanted['ImdbId'],sourceWebsites)) # Compose the search URL for the subtitle and language we want. langs = Wanted['langs'][0] + ',' + Wanted['langs'][1] if len(Wanted['langs']) == 2 else Wanted['langs'][0] SearchUrl = "%s&imdb=%s&season=%s&episode=%s&language=%s&return_type=json" % (autosub.API, Wanted['ImdbId'], Wanted['season'], Wanted['episode'], langs) # Let Subtitle seeker do a search voor the wanted sub if autosub.Helpers.checkAPICallsSubSeeker(use=True): try: SubseekerSession = requests.session() Result = SubseekerSession.get(SearchUrl).json() SubseekerSession.close() except Exception as error: log.error("getSubLink: The SubtitleSeeker server returned an error. Message is %s" % error) return ScoreListNL,ScoreListEN else: log.error("getSubLink: out of api calls for SubtitleSeeker.com") return ScoreListNL,ScoreListEN # Check to see whether we have results or not try: if not 'total_matches' in Result['results'].keys(): return ScoreListNL,ScoreListEN except Exception as error: log.info('getSublink: No subtitle found on Subtitleseeker for this video : %s' % Wanted['file']) return ScoreListNL,ScoreListEN if int(Result['results']['total_matches']) == 0: return ScoreListNL,ScoreListEN # Split the result in the two languages(if needed) and score the subs NameDict = {} for Item in Result['results']['items']: if Item['site'].lower() in sourceWebsites: Item['release'] = Item['release'][:-4] if Item['release'].lower().endswith(".srt") else Item['release'] NameDict.clear() NameDict = ProcessFilename(Item['release'],Wanted['container']) if not NameDict: continue score = autosub.Helpers.scoreMatch(NameDict,Wanted) if score == 0: continue log.debug('SubtitleSeeker: Score = %s of %s for %s sub of %s on %s.' % (score, autosub.MINMATCHSCORE, Item['language'], Item['release'], Item['site'])) if score >= autosub.MINMATCHSCORE: if Item['language'] == autosub.ENGLISH: ScoreListEN.append({'score':score, 'url':Item['url'] , 'website':Item['site'].lower(),'Lang':Item['language'], 'releaseName':Item['release'],'SubCodec':u''}) if Item['language'] == autosub.DUTCH: ScoreListNL.append({'score':score, 'url':Item['url'] , 'website':Item['site'].lower(),'Lang':Item['language'], 'releaseName':Item['release'],'SubCodec':u''}) return ScoreListNL,ScoreListEN
def Opensubtitles(Wanted): ScoreListNL, ScoreListEN = [],[] # Format a dict for the opensubtitles API call Data = {} Data['sublanguageid'] = Wanted['langs'][0][:3] if len(Wanted['langs']) == 1 else Wanted['langs'][0][:3] + ',' + Wanted['langs'][1][:3] Data['imdbid' ] = Wanted['ImdbId'] Data['season'] = Wanted['season'] Data['episode'] = Wanted['episode'] log.debug('getSubLinks: Opensubtitles search started for %s.' % Wanted['ImdbId']) time.sleep(3) if not OpenSubtitlesNoOp(): return ScoreListNL,ScoreListEN try: Subs = autosub.OPENSUBTITLESSERVER.SearchSubtitles(autosub.OPENSUBTITLESTOKEN, [Data]) except: autosub.OPENSUBTITLESTOKEN = None log.error('Opensubtitles: Error from Opensubtitles search API') return ScoreListNL,ScoreListEN if Subs['status'] != '200 OK': log.debug('Opensubtitles: No subs found for %s on Opensubtitles.' % Wanted['releaseName']) return ScoreListNL,ScoreListEN NameDict = {} for Sub in Subs['data']: try: if int(Sub['SubBad']) > 0 or not Sub['MovieReleaseName'] or not Sub['IDSubtitleFile'] or (Sub['SubHearingImpaired'] != '0' and not autosub.HI): continue except: continue NameDict.clear() NameDict = ProcessFilename(Sub['MovieReleaseName'],Wanted['container']) if not NameDict: continue # here we score the subtitle and if it's high enough we add it to the list score = autosub.Helpers.scoreMatch(NameDict,Wanted) if score == 0: continue log.debug('Opensubtitles: Score = %s of %s for %s sub of %s.' % (score, autosub.MINMATCHSCORE, Sub['LanguageName'], Sub['MovieReleaseName'])) if score >= autosub.MINMATCHSCORE: if Sub['LanguageName'] == autosub.DUTCH: ScoreListNL.append({'score':score, 'url':Sub['IDSubtitleFile'] , 'website':'opensubtitles.org','releaseName':Sub['MovieReleaseName'], 'SubCodec':Sub['SubEncoding'],'Lang':Sub['LanguageName']}) if Sub['LanguageName'] == autosub.ENGLISH: ScoreListEN.append({'score':score, 'url':Sub['IDSubtitleFile'] , 'website':'opensubtitles.org','releaseName':Sub['MovieReleaseName'], 'SubCodec':Sub['SubEncoding'],'Lang':Sub['LanguageName']}) return ScoreListNL,ScoreListEN
def walkDir(path): SkipListNL = autosub.SKIPSTRINGNL.split(",") if len(autosub.SKIPSTRINGNL) > 0 else [] SkipListEN = autosub.SKIPSTRINGEN.split(",") if len(autosub.SKIPSTRINGEN) > 0 else [] if len(autosub.SKIPFOLDERSNL) == 0: SkipFoldersNL = [] else: SkipFoldersNL = autosub.SKIPFOLDERSNL.split(",") if len(autosub.SKIPFOLDERSNL) > 0 else [] for idx,folder in enumerate(SkipFoldersNL): SkipFoldersNL[idx] = os.path.normpath(os.path.join(path,folder.rstrip("\/").lstrip("\/"))) if len(autosub.SKIPFOLDERSNL) == 0: SkipFoldersNL = [] else: SkipFoldersNL = autosub.SKIPFOLDERSNL.split(",") if len(autosub.SKIPFOLDERSNL) > 0 else [] for idx,folder in enumerate(SkipFoldersNL): SkipFoldersNL[idx] = os.path.normpath(path + folder) if len(autosub.SKIPFOLDERSEN) == 0: SkipFoldersEN = [] else: SkipFoldersEN = autosub.SKIPFOLDERSEN.split(",") if len(autosub.SKIPFOLDERSEN) > 0 else [] for idx,folder in enumerate(SkipFoldersEN): SkipFoldersEN[idx] = os.path.normpath(path + folder) for dirname, dirnames, filenames in os.walk(path, True, WalkError): SkipThisFolderNL = False for skip in SkipFoldersNL: if dirname.startswith(skip): SkipThisFolderNL = True break SkipThisFolderEN = False for skip in SkipFoldersEN: if dirname.startswith(skip): SkipThisFolderEN = True break log.debug("scanDisk: directory name: %s" %dirname) if re.search('_unpack_', dirname, re.IGNORECASE): log.debug("scanDisk: found a unpack directory, skipping.") continue if autosub.SKIPHIDDENDIRS and os.path.split(dirname)[1].startswith(u'.'): continue if re.search('_failed_', dirname, re.IGNORECASE): log.debug("scanDisk: found a failed directory, skipping.") continue if re.search('@eaDir', dirname, re.IGNORECASE): log.debug("scanDisk: found a Synology indexing directory, skipping.") tmpdirs = dirnames[:] for dir in tmpdirs: dirnames.remove(dir) continue if re.search("@.*thumb", dirname, re.IGNORECASE): log.debug("scanDisk: found a Qnap multimedia thumbnail folder, skipping.") continue langs = [] FileDict = {} for filename in filenames: root,ext = os.path.splitext(filename) if ext[1:] in ('avi', 'mkv', 'wmv', 'ts', 'mp4'): if re.search('sample', filename): continue if not platform.system() == 'Windows': # Get best ascii compatible character for special characters try: if not isinstance(filename, unicode): coding = detect(filename)['encoding'] filename = unicode(filename.decode(coding),errors='replace') correctedFilename = ''.join((c for c in unicodedata.normalize('NFD', filename) if unicodedata.category(c) != 'Mn')) if filename != correctedFilename: os.rename(os.path.join(dirname, filename), os.path.join(dirname, correctedFilename)) log.info("scanDir: Renamed file %s" % correctedFilename) filename = correctedFilename except: log.error("scanDir: Skipping directory, file %s, %s" % (dirname,filename)) continue # What subtitle files should we expect? langs = [] NLext = u'.' + autosub.SUBNL + u'.srt' if autosub.SUBNL else u'.srt' ENext = u'.' + autosub.SUBENG + u'.srt' if autosub.SUBENG else u'.srt' ENext = u'.en.srt'if NLext == ENext and autosub.DOWNLOADDUTCH else ENext if not os.access(dirname, os.W_OK): log.error('scandisk: No write access to folder: %s' % dirname) continue # Check which languages we want to download based on user settings. if autosub.DOWNLOADDUTCH and not SkipThisFolderNL: Skipped = False for SkipItem in SkipListNL: if not SkipItem: break if re.search(SkipItem.lower(), filename.lower()): Skipped = True break if Skipped: log.info("scanDir: %s found in %s so skipped for Dutch subs" % (SkipItem, filename)) elif os.path.exists(os.path.join(dirname, root + NLext)): Skipped = True log.debug("scanDir: %s skipped because the Dutch subtitle already exists" % filename) else: # If the Dutch subtitle not skipped and doesn't exist, then add it to the wanted list langs.append(autosub.DUTCH) if (autosub.DOWNLOADENG or (autosub.FALLBACKTOENG and autosub.DOWNLOADDUTCH and not Skipped)) and not SkipThisFolderEN: Skipped = False for SkipItem in SkipListEN: if not SkipItem: break if re.search(SkipItem.lower(), filename.lower()): Skipped = True break if Skipped: log.info("scanDir: %s found in %s so skipped for English subs" % (SkipItem, filename)) elif os.path.exists(os.path.join(dirname, root + ENext)): log.debug("scanDir: %s skipped because the English subtitle already exists" % filename) else: # If the English subtitle not skipped and doesn't exist, then add it to the wanted list if not os.path.exists(os.path.join(dirname, root + ENext)): langs.append(autosub.ENGLISH) if not langs: # nothing to do for this file continue FileDict = ProcessFilename(os.path.splitext(filename)[0].strip(), ext) time.sleep(0) if not FileDict: continue if not 'title' in FileDict.keys() or not 'season' in FileDict.keys() or not 'episode' in FileDict.keys(): continue if not FileDict['releasegrp'] and not FileDict['source'] and not FileDict['quality'] and not FileDict['source']: log.error("scanDir: Not enough info in filename: %s" % filename) continue FileDict['timestamp'] = unicode(time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(os.path.getctime(os.path.join(dirname, filename))))) FileDict['langs'] = langs FileDict['NLext'] = NLext FileDict['ENext'] = ENext FileDict['file'] = root FileDict['container'] = ext FileDict['folder'] = dirname FileDict['ImdbId'],FileDict['A7Id'], FileDict['TvdbId'], FileDict['title'] = Helpers.getShowid(FileDict['title'],autosub.ADDIC7EDLOGGED_IN) if autosub.Helpers.SkipShow(FileDict['ImdbId'],FileDict['title'], FileDict['season'], FileDict['episode']): continue log.info("scanDir: %s WANTED FOR: %s" % (langs, filename)) autosub.WANTEDQUEUE.append(FileDict) return
def run(self): log.debug("checkRSS: Starting round of RSS checking") if autosub.WANTEDQUEUELOCK: log.debug("checkRSS: Exiting, another threat is using the queues") return False else: autosub.WANTEDQUEUELOCK = True toDelete_wantedQueue = [] langs = ["nl"] # default is to only check for Dutch subs # but if English should be downloaden, check them too # It is very important that the dutch language is run first! if autosub.FALLBACKTOENG or autosub.DOWNLOADENG: langs.append("en") log.debug("checkRSS: We also want to check the English RSS feed") for lang in langs: if lang == "en": RSSURL = autosub.ENRSSURL log.debug("checkRSS: Now using the English RSS feed") else: RSSURL = autosub.NLRSSURL log.debug("checkRSS: Now using the Dutch RSS feed") try: bierdopjeapi = API(RSSURL,RSS=True) soup = BeautifulStoneSoup(bierdopjeapi.resp.read()) log.debug("checkRss: Succussfully connected to %s" %RSSURL) bierdopjeapi.close() except: log.error("checkRss: The server returned an error for request %s" % RSSURL) autosub.WANTEDQUEUELOCK = False continue # Parse all the item-tags from the RSSFeed # The information that is parsed is: title, link and show_id # The show_id is later used to match with the wanted items # The title is used the determine the quality / source / releasegrp rssItemList = [] items = soup.findAll('item') if not len(items) > 0: log.error("checkRss: invalid RssFeed") log.debug("checkRss: dumping Rssfeed %s" %(soup.prettify())) else: log.debug("checkRss: Valid RssFeed") for x in items: soupx = BeautifulStoneSoup(str(x)) title = soupx.find('title').string show_id = soupx.find('show_id').string link = soupx.find('enclosure')['url'] item = {} item['title'] = title item['link'] = link item['show_id'] = show_id log.debug("checkRSS: Parsed from RSSFEED: %s %s %s" %(title,link,show_id)) rssItemList.append(item) normalizedRssTitleList = [] # Now we create a new rsslist, containing information like: episode, season, etc for item in rssItemList: title = item['title'] link = item['link'] show_id = item['show_id'] log.debug("checkRSS: Normalizing the following entry in the RSS results: %s" % title) normalizedRssTitle = ProcessFilename(title, '') normalizedRssTitle['rssfile'] = title normalizedRssTitle['link'] = link normalizedRssTitle['show_id'] = str(show_id) if 'title' in normalizedRssTitle.keys(): if 'season' in normalizedRssTitle.keys(): if 'episode' in normalizedRssTitle.keys(): normalizedRssTitleList.append(normalizedRssTitle) #check versus wantedItem list for index, wantedItem in enumerate(autosub.WANTEDQUEUE): wantedItemquality = None wantedItemreleasegrp = None wantedItemsource = None wantedItemcodec = None wantedItemtitle = wantedItem['title'] wantedItemseason = wantedItem['season'] wantedItemepisode = wantedItem['episode'] originalfile = wantedItem['originalFileLocationOnDisk'] if lang not in wantedItem['lang']: continue if 'quality' in wantedItem.keys(): wantedItemquality = wantedItem['quality'] if 'releasegrp' in wantedItem.keys(): wantedItemreleasegrp = wantedItem['releasegrp'] if 'source' in wantedItem.keys(): wantedItemsource = wantedItem['source'] if 'codec' in wantedItem.keys(): wantedItemcodec = wantedItem['codec'] #lets try to find a showid showid = autosub.Helpers.getShowid(wantedItemtitle) #no showid? skip this item if not showid: continue for normalizedRssTitle in normalizedRssTitleList: downloadLink = None normalizedRssTitleseason = normalizedRssTitle['season'] normalizedRssTitleepisode = normalizedRssTitle['episode'] normalizedRssTitlerssfile = normalizedRssTitle['rssfile'] normalizedRssTitleshowid = int(normalizedRssTitle['show_id']) normalizedRssTitlelink = normalizedRssTitle['link'] log.debug("checkRSS: Trying to match ID:%r S:%r E:%r (wantedlist) with ID:%r S:%r E:%r (rss)" %(showid, wantedItemseason, wantedItemepisode, normalizedRssTitleshowid, normalizedRssTitleseason, normalizedRssTitleepisode)) if showid == normalizedRssTitleshowid and wantedItemseason == normalizedRssTitleseason and wantedItemepisode == normalizedRssTitleepisode: log.debug("checkRSS: The episode %s - Season %s Episode %s was found in the RSS list, attempting to match a proper match" % (wantedItemtitle, wantedItemseason, wantedItemepisode)) score = autosub.Helpers.scoreMatch(normalizedRssTitle, normalizedRssTitlerssfile, wantedItemquality, wantedItemreleasegrp, wantedItemsource, wantedItemcodec) if score >= autosub.MINMATCHSCORERSS: log.debug ("checkRss: A match got a high enough score. MinMatchscore is %s " % autosub.MINMATCHSCORERSS) downloadLink = normalizedRssTitlelink + autosub.APIRSS log.info ("checkRss: Got a match, matching file is: %s" %normalizedRssTitlerssfile) log.debug("checkRss: Dumping downloadlink for debug purpose: %s" %downloadLink) if downloadLink: originalfile = wantedItem['originalFileLocationOnDisk'] # Dutch subs if autosub.SUBNL != "" and lang == "nl": srtfile = os.path.splitext(originalfile)[0] + u"." + autosub.SUBNL + u".srt" elif lang == "nl": srtfile = os.path.splitext(originalfile)[0] + u".srt" # English subs if autosub.SUBENG != "" and lang == "en": srtfile = os.path.splitext(originalfile)[0] + u"." + autosub.SUBENG + u".srt" elif lang == "en": srtfile = os.path.splitext(originalfile)[0] + u".srt" wantedItem['downloadLink'] = downloadLink wantedItem['destinationFileLocationOnDisk'] = srtfile log.info("checkRSS: The episode %s - Season %s Episode %s has a matching subtitle on the RSSFeed, downloading it!" % (wantedItemtitle, wantedItemseason, wantedItemepisode)) downloadItem = wantedItem.copy() downloadItem['downlang'] = lang downloadItem['subtitle'] = normalizedRssTitlerssfile if not DownloadSub(downloadItem): break if lang == 'nl' and (autosub.FALLBACKTOENG and not autosub.DOWNLOADENG) and 'en' in wantedItem['lang']: log.debug('checkRss: We found a dutch subtitle and fallback is true. Removing the english subtitle from the wantedlist.') wantedItem['lang'].remove('en') wantedItem['lang'].remove(lang) if len(wantedItem['lang']) == 0: toDelete_wantedQueue.append(index) break else: log.debug("checkRss: Matching score is not high enough. Score is %s should be %s" %(str(score),autosub.MINMATCHSCORERSS)) i = len(toDelete_wantedQueue)-1 while i >= 0: log.debug("checkRSS: Removed item from the wantedQueue at index %s" % toDelete_wantedQueue[i]) autosub.WANTEDQUEUE.pop(toDelete_wantedQueue[i]) i = i-1 # Resetting the toDelete queue for the next run (if need) toDelete_wantedQueue =[] log.debug("checkRSS: Finished round of RSS checking") autosub.WANTEDQUEUELOCK = False return True
def getSubLink(showid, lang, releaseDetails): """ Return the link to download the best matching subtitle. Matching is based on the provided release details. Keyword arguments: showid -- The id of the show on Bierdopje lang -- Language of the wanted subtitle, Dutch or English releaseDetails -- Dict containing the quality, releasegrp, source season and episode. """ api = autosub.API if showid == -1: return (None, None) quality = None releasegrp = None source = None season = releaseDetails['season'] episode = releaseDetails['episode'] getSubLinkUrl = "%sGetAllSubsFor/%s/%s/%s/%s" % (api, showid, season, episode, lang) try: bierdopjeapi = API(getSubLinkUrl) dom = minidom.parse(bierdopjeapi.resp) bierdopjeapi.resp.close() except: log.error("getSubLink: The server returned an error for request %s" % getSubLinkUrl) return (None, None) if 'quality' in releaseDetails.keys(): quality = releaseDetails['quality'] if 'releasegrp' in releaseDetails.keys(): releasegrp = releaseDetails['releasegrp'] if 'source' in releaseDetails.keys(): source = releaseDetails['source'] if 'codec' in releaseDetails.keys(): codec = releaseDetails['codec'] if not dom or len(dom.getElementsByTagName('result')) == 0: return (None, None) scoredict = {} releasedict = {} for sub in dom.getElementsByTagName('result'): release = sub.getElementsByTagName('filename')[0].firstChild.data release = release.lower() # Remove the .srt extension some of the uploaders leave on the file if release.endswith(".srt"): release = release[:-4] # Scoredict is a dictionary with a download link and its match score. This will be used to determine the best match (the highest matchscore) scoredict[sub.getElementsByTagName( 'downloadlink')[0].firstChild.data] = autosub.Helpers.scoreMatch( ProcessFilename(release, ''), release, quality, releasegrp, source, codec) # Releasedict is a dictionary with the release name, used for the lastdownload database releasedict[sub.getElementsByTagName('downloadlink') [0].firstChild.data] = release if scoredict[sub.getElementsByTagName('downloadlink') [0].firstChild.data] == 15: # Sometimes you just find a perfect match, why should we continue to search if we got a perfect match? log.debug( 'getSubLink: A perfect match found, returning the download link' ) return ( sub.getElementsByTagName('downloadlink')[0].firstChild.data, release) # Done comparing all the results, lets sort them and return the highest result # If there are results with the same score, the download links which comes first (alphabetically) will be returned # Also check if the result match the minimal score sortedscoredict = sorted(scoredict.items(), key=itemgetter(1), reverse=True) toDelete = [] for index, item in enumerate(sortedscoredict): log.debug( 'getSubLink: checking minimal match score for %s. Minimal match score is: %s' % (item, autosub.MINMATCHSCORE)) score = item[1] link = item[0] if not score >= autosub.MINMATCHSCORE: log.debug('getSubLink: %s does not match the minimal match score' % link) toDelete.append(index) i = len(toDelete) - 1 while i >= 0: log.debug("getSubLink: Removed item from the ScoreDict at index %s" % toDelete[i]) sortedscoredict.pop(toDelete[i]) i = i - 1 if len(sortedscoredict) > 0: release = releasedict[sortedscoredict[0][0]] return (sortedscoredict[0][0], release) return (None, None)
def walkDir(path): SkipListNL = [] SkipListEN = [] SkipListNL = autosub.SKIPSTRINGNL.split(",") SkipListEN = autosub.SKIPSTRINGEN.split(",") # Here we use os.walk to find all files in the path. for dirname, dirnames, filenames in os.walk(path): log.debug("scanDisk: directory name: %s" % dirname) if re.search('_unpack_', dirname, re.IGNORECASE): log.debug("scanDisk: found a unpack directory, skipping") continue if autosub.SKIPHIDDENDIRS and os.path.split(dirname)[1].startswith( u'.'): continue if re.search('_failed_', dirname, re.IGNORECASE): log.debug("scanDisk: found a failed directory, skipping") continue if re.search('@eaDir', dirname, re.IGNORECASE): log.debug( "scanDisk: found a Synology indexing directory, skipping this folder and all subfolders." ) tmpdirs = dirnames[:] for dir in tmpdirs: dirnames.remove(dir) continue if re.search("@.*thumb", dirname, re.IGNORECASE): # Ingnore QNAP multimedia thumbnail folders continue for filename in filenames: splitname = filename.split(".") ext = splitname[len(splitname) - 1] if ext in ('avi', 'mkv', 'wmv', 'ts', 'mp4'): if re.search('sample', filename): continue if not platform.system() == 'Windows': # Get best ascii compatible character for special characters try: if not isinstance(filename, unicode): coding = detect(filename) filename = unicode(filename.decode(coding)) correctedFilename = ''.join( (c for c in unicodedata.normalize('NFD', filename) if unicodedata.category(c) != 'Mn')) if filename != correctedFilename: os.rename(os.path.join(dirname, filename), os.path.join(dirname, correctedFilename)) log.info("scanDir: Renamed file %s" % correctedFilename) filename = correctedFilename except: log.error("scanDir: Skipping directory %s" % dirname) log.error("scanDir: Skipping file %s" % filename) continue # What subtitle files should we expect? lang = [] #Check what the Dutch subtitle would be. if autosub.SUBNL != "": srtfilenl = os.path.splitext( filename)[0] + u"." + autosub.SUBNL + u".srt" else: srtfilenl = os.path.splitext(filename)[0] + u".srt" #Check what the English subtitle would be. if autosub.SUBENG == "": # Check for overlapping names if autosub.SUBNL != "" or not autosub.DOWNLOADDUTCH: srtfileeng = os.path.splitext(filename)[0] + u".srt" # Hardcoded fallback else: srtfileeng = os.path.splitext(filename)[0] + u".en.srt" else: srtfileeng = os.path.splitext( filename)[0] + u"." + autosub.SUBENG + u".srt" # Check which languages we want to download based on user settings and check the skipstring if autosub.DOWNLOADDUTCH: Skipped = False for SkipItem in SkipListNL: if re.search(SkipItem.lower(), filename.lower()): Skipped = True log.debug( "scanDir: %s found in %s so skipped for Dutch subs" % (SkipItem, filename)) break if not Skipped and os.path.exists( os.path.join(dirname, srtfilenl)): Skipped = True log.debug( "scanDir: %s skipped because the subtitle already exists" % filename) # If the Dutch subtitle doesn't exist en has no skip item, then add it to the wanted list. if not Skipped: lang.append(autosub.DUTCH) if autosub.DOWNLOADENG: Skipped = False for SkipItem in SkipListEN: if re.search(SkipItem.lower(), filename.lower()): Skipped = True log.debug( "scanDir: %s found in %s so skipped for English subs" % (SkipItem, filename)) break if not Skipped and os.path.exists( os.path.join(dirname, srtfileeng)): Skipped = True log.debug( "scanDir: %s skipped because the subtitle already exists" % filename) # If the English subtitle doesn't exist en has no skip item, then add it to the wanted list. if not Skipped: lang.append(autosub.ENGLISH) if (autosub.FALLBACKTOENG and autosub.DOWNLOADDUTCH ) and not autosub.DOWNLOADENG and not Skipped: # If the Dutch and English subtitles do not exist and not skipped, then add English to the wanted list. if not os.path.exists(os.path.join( dirname, srtfilenl)) and not os.path.exists( os.path.join(dirname, srtfileeng)): lang.append(autosub.ENGLISH) if not lang: # autosub.WANTEDQUEUE empty continue log.debug("scanDir: File %s is missing subtitle(s): %s" % (filename, ', '.join(map(str, lang)))) filenameResults = ProcessFilename( os.path.splitext(filename)[0], os.path.splitext(filename)[1]) if 'title' in filenameResults.keys(): if 'season' in filenameResults.keys(): if 'episode' in filenameResults.keys(): title = filenameResults['title'] season = filenameResults['season'] episode = filenameResults['episode'] if not filenameResults[ 'releasegrp'] and not filenameResults[ 'source'] and not filenameResults[ 'quality'] and not filenameResults[ 'source']: continue if autosub.Helpers.SkipShow( title, season, episode) == True: log.debug("scanDir: SkipShow returned True") log.info( "scanDir: Skipping %s - Season %s Episode %s" % (title, season, episode)) continue if len(lang) == 1: log.info( "scanDir: %s subtitle wanted for %s and added to wantedQueue" % (lang[0], filename)) else: log.info( "scanDir: %s subtitles wanted for %s and added to wantedQueue" % (' and '.join(map(str, lang)), filename)) filenameResults[ 'originalFileLocationOnDisk'] = os.path.join( dirname, filename) filenameResults['timestamp'] = unicode( time.strftime( '%Y-%m-%d %H:%M:%S', time.localtime( os.path.getctime(filenameResults[ 'originalFileLocationOnDisk'])))) filenameResults['lang'] = lang autosub.WANTEDQUEUE.append(filenameResults) else: log.error( "scanDir: Could not process the filename properly filename: %s" % filename) continue else: log.error( "scanDir: Could not process the filename properly filename: %s" % filename) continue else: log.error( "scanDir: Could not process the filename properly filename: %s" % filename) continue
def walkDir(path): SkipListNL = autosub.SKIPSTRINGNL.split(",") if len( autosub.SKIPSTRINGNL) > 0 else [] SkipListEN = autosub.SKIPSTRINGEN.split(",") if len( autosub.SKIPSTRINGEN) > 0 else [] # Check for dutch folder skip if len(autosub.SKIPFOLDERSNL) == 0: SkipFoldersNL = [] else: SkipFoldersNL = autosub.SKIPFOLDERSNL.split(",") if len( autosub.SKIPFOLDERSNL) > 0 else [] for idx, folder in enumerate(SkipFoldersNL): SkipFoldersNL[idx] = os.path.normpath( os.path.join(path, folder.strip(" \/"))) # Check for english folder skip if len(autosub.SKIPFOLDERSEN) == 0: SkipFoldersEN = [] else: SkipFoldersEN = autosub.SKIPFOLDERSEN.split(",") if len( autosub.SKIPFOLDERSEN) > 0 else [] for idx, folder in enumerate(SkipFoldersEN): SkipFoldersEN[idx] = os.path.normpath( os.path.join(path, folder.strip(" \/"))) for dirname, dirnames, filenames in os.walk(path, True, WalkError): SkipThisFolderNL = False for skip in SkipFoldersNL: if dirname.startswith(skip): SkipThisFolderNL = True break SkipThisFolderEN = False for skip in SkipFoldersEN: if dirname.startswith(skip): SkipThisFolderEN = True break log.debug("scanDisk: directory name: %s" % dirname) if re.search('_unpack_', dirname, re.IGNORECASE): log.debug("scanDisk: found a unpack directory, skipping.") continue if autosub.SKIPHIDDENDIRS and os.path.split(dirname)[1].startswith( u'.'): continue if re.search('_failed_', dirname, re.IGNORECASE): log.debug("scanDisk: found a failed directory, skipping.") continue if re.search('@eaDir', dirname, re.IGNORECASE): log.debug( "scanDisk: found a Synology indexing directory, skipping.") tmpdirs = dirnames[:] for dir in tmpdirs: dirnames.remove(dir) continue if re.search("@.*thumb", dirname, re.IGNORECASE): log.debug( "scanDisk: found a Qnap multimedia thumbnail folder, skipping." ) continue langs = [] FileDict = {} for filename in filenames: if autosub.SEARCHSTOP: log.info('scanDisk: Forced Stop by user') return try: root, ext = os.path.splitext(filename) if ext[1:] in ('avi', 'mkv', 'wmv', 'ts', 'mp4'): if re.search('sample', filename): continue if not platform.system() == 'Windows': # Get best ascii compatible character for special characters try: if not isinstance(filename, unicode): coding = detect(filename)['encoding'] filename = unicode(filename.decode(coding), errors='replace') correctedFilename = ''.join( (c for c in unicodedata.normalize( 'NFD', filename) if unicodedata.category(c) != 'Mn')) if filename != correctedFilename: os.rename( os.path.join(dirname, filename), os.path.join(dirname, correctedFilename)) log.info("scanDir: Renamed file %s" % correctedFilename) filename = correctedFilename except: log.error( "scanDir: Skipping directory, file %s, %s" % (dirname, filename)) continue # What subtitle files should we expect? langs = [] NLext = u'.' + autosub.SUBNL + u'.srt' if autosub.SUBNL else u'.srt' ENext = u'.' + autosub.SUBENG + u'.srt' if autosub.SUBENG else u'.srt' ENext = u'.en.srt' if NLext == ENext and autosub.DOWNLOADDUTCH else ENext if not os.access(dirname, os.W_OK): log.error('scandisk: No write access to folder: %s' % dirname) continue # Check which languages we want to download based on user settings. log.debug('scanDir: Processing file: %s' % filename) if autosub.DOWNLOADDUTCH and not SkipThisFolderNL: Skipped = False for SkipItem in SkipListNL: if not SkipItem: break if re.search(SkipItem.lower(), filename.lower()): Skipped = True break if Skipped: log.info( "scanDir: %s found in %s so skipped for Dutch subs" % (SkipItem, filename)) elif os.path.exists(os.path.join( dirname, root + NLext)): Skipped = True log.debug( "scanDir: %s skipped because the Dutch subtitle already exists" % filename) else: # If the Dutch subtitle not skipped and doesn't exist, then add it to the wanted list langs.append(autosub.DUTCH) if (autosub.DOWNLOADENG or (autosub.FALLBACKTOENG and autosub.DOWNLOADDUTCH and not Skipped)) and not SkipThisFolderEN: Skipped = False for SkipItem in SkipListEN: if not SkipItem: break if re.search(SkipItem.lower(), filename.lower()): Skipped = True break if Skipped: log.info( "scanDir: %s found in %s so skipped for English subs" % (SkipItem, filename)) elif os.path.exists(os.path.join( dirname, root + ENext)): log.debug( "scanDir: %s skipped because the English subtitle already exists" % filename) else: # If the English subtitle not skipped and doesn't exist, then add it to the wanted list if not os.path.exists( os.path.join(dirname, root + ENext)): langs.append(autosub.ENGLISH) if not langs: # nothing to do for this file continue FileDict = ProcessFilename( os.path.splitext(filename)[0].strip(), ext) if not FileDict: log.debug( 'scanDisk: not enough info in the filename: %s' % filename) continue Skip = False if autosub.MINMATCHSCORE & 8 and not FileDict['source']: Skip = True elif autosub.MINMATCHSCORE & 4 and not FileDict['quality']: Skip = True elif autosub.MINMATCHSCORE & 2 and not FileDict['codec']: Skip = True elif autosub.MINMATCHSCORE & 1 and not FileDict[ 'releasegrp']: Skip = True if Skip: log.debug( 'scanDisk: Filespec does not meet minmatchscore so skipping this one' ) continue FileDict['timestamp'] = unicode( time.strftime( '%Y-%m-%d %H:%M:%S', time.localtime( os.path.getmtime( os.path.join(dirname, filename))))) FileDict['langs'] = langs FileDict['NLext'] = NLext FileDict['ENext'] = ENext FileDict['file'] = root FileDict['container'] = ext FileDict['folder'] = dirname FileDict['ImdbId'], FileDict['A7Id'], FileDict[ 'TvdbId'], FileDict['title'] = Helpers.getShowid( FileDict['title']) if autosub.Helpers.SkipShow(FileDict['ImdbId'], FileDict['title'], FileDict['season'], FileDict['episode']): log.debug("scanDir: SKIPPED %s by Skipshow rules." % FileDict['file']) continue log.info("scanDir: %s WANTED FOR: %s" % (langs, filename)) autosub.WANTEDQUEUE.append(FileDict) time.sleep(0) except Exception as error: log.error('scanDir: Problem scanning file %s. Error is: %s' % (filename, error)) return
def walkDir(path): SkipListNL = [] SkipListEN = [] SkipListNL = autosub.SKIPSTRINGNL.split(",") SkipListEN = autosub.SKIPSTRINGEN.split(",") # Here we use os.walk to find all files in the path. for dirname, dirnames, filenames in os.walk(path): log.debug("scanDisk: directory name: %s" %dirname) if re.search('_unpack_', dirname, re.IGNORECASE): log.debug("scanDisk: found a unpack directory, skipping") continue if autosub.SKIPHIDDENDIRS and os.path.split(dirname)[1].startswith(u'.'): continue if re.search('_failed_', dirname, re.IGNORECASE): log.debug("scanDisk: found a failed directory, skipping") continue if re.search('@eaDir', dirname, re.IGNORECASE): log.debug("scanDisk: found a Synology indexing directory, skipping this folder and all subfolders.") tmpdirs = dirnames[:] for dir in tmpdirs: dirnames.remove(dir) continue if re.search("@.*thumb", dirname, re.IGNORECASE): # Ingnore QNAP multimedia thumbnail folders continue for filename in filenames: splitname = filename.split(".") ext = splitname[len(splitname) - 1] if ext in ('avi', 'mkv', 'wmv', 'ts', 'mp4'): if re.search('sample', filename): continue if not platform.system() == 'Windows': # Get best ascii compatible character for special characters try: if not isinstance(filename, unicode): coding = detect(filename) filename = unicode(filename.decode(coding)) correctedFilename = ''.join((c for c in unicodedata.normalize('NFD', filename) if unicodedata.category(c) != 'Mn')) if filename != correctedFilename: os.rename(os.path.join(dirname, filename), os.path.join(dirname, correctedFilename)) log.info("scanDir: Renamed file %s" % correctedFilename) filename = correctedFilename except: log.error("scanDir: Skipping directory %s" % dirname) log.error("scanDir: Skipping file %s" % filename) continue # What subtitle files should we expect? lang=[] #Check what the Dutch subtitle would be. if autosub.SUBNL != "": srtfilenl = os.path.splitext(filename)[0] + u"." + autosub.SUBNL + u".srt" else: srtfilenl = os.path.splitext(filename)[0] + u".srt" #Check what the English subtitle would be. if autosub.SUBENG == "": # Check for overlapping names if autosub.SUBNL != "" or not autosub.DOWNLOADDUTCH: srtfileeng = os.path.splitext(filename)[0] + u".srt" # Hardcoded fallback else: srtfileeng = os.path.splitext(filename)[0] + u".en.srt" else: srtfileeng = os.path.splitext(filename)[0] + u"." + autosub.SUBENG + u".srt" # Check which languages we want to download based on user settings and check the skipstring if autosub.DOWNLOADDUTCH: Skipped = False for SkipItem in SkipListNL: if re.search(SkipItem.lower(), filename.lower()): Skipped = True log.debug("scanDir: %s found in %s so skipped for Dutch subs" % (SkipItem, filename)) break if not Skipped and os.path.exists(os.path.join(dirname, srtfilenl)): Skipped = True log.debug("scanDir: %s skipped because the subtitle already exists" % filename) # If the Dutch subtitle doesn't exist en has no skip item, then add it to the wanted list. if not Skipped: lang.append(autosub.DUTCH) if autosub.DOWNLOADENG: Skipped = False for SkipItem in SkipListEN: if re.search(SkipItem.lower(), filename.lower()): Skipped = True log.debug("scanDir: %s found in %s so skipped for English subs" % (SkipItem, filename)) break if not Skipped and os.path.exists(os.path.join(dirname, srtfileeng)): Skipped = True log.debug("scanDir: %s skipped because the subtitle already exists" % filename) # If the English subtitle doesn't exist en has no skip item, then add it to the wanted list. if not Skipped: lang.append(autosub.ENGLISH) if (autosub.FALLBACKTOENG and autosub.DOWNLOADDUTCH) and not autosub.DOWNLOADENG and not Skipped: # If the Dutch and English subtitles do not exist and not skipped, then add English to the wanted list. if not os.path.exists(os.path.join(dirname, srtfilenl)) and not os.path.exists(os.path.join(dirname, srtfileeng)): lang.append(autosub.ENGLISH) if not lang: # autosub.WANTEDQUEUE empty continue log.debug("scanDir: File %s is missing subtitle(s): %s" % (filename, ', '.join(map(str,lang)))) filenameResults = ProcessFilename(os.path.splitext(filename)[0], os.path.splitext(filename)[1]) if 'title' in filenameResults.keys(): if 'season' in filenameResults.keys(): if 'episode' in filenameResults.keys(): title = filenameResults['title'] season = filenameResults['season'] episode = filenameResults['episode'] if not filenameResults['releasegrp'] and not filenameResults['source'] and not filenameResults['quality'] and not filenameResults['source']: continue if autosub.Helpers.SkipShow(title, season, episode) == True: log.debug("scanDir: SkipShow returned True") log.info("scanDir: Skipping %s - Season %s Episode %s" % (title, season, episode)) continue if len(lang) == 1: log.info("scanDir: %s subtitle wanted for %s and added to wantedQueue" % (lang[0], filename)) else: log.info("scanDir: %s subtitles wanted for %s and added to wantedQueue" % (' and '.join(map(str,lang)), filename)) filenameResults['originalFileLocationOnDisk'] = os.path.join(dirname, filename) filenameResults['timestamp'] = unicode(time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(os.path.getctime(filenameResults['originalFileLocationOnDisk'])))) filenameResults['lang'] = lang autosub.WANTEDQUEUE.append(filenameResults) else: log.error("scanDir: Could not process the filename properly filename: %s" % filename) continue else: log.error("scanDir: Could not process the filename properly filename: %s" % filename) continue else: log.error("scanDir: Could not process the filename properly filename: %s" % filename) continue
def walkDir(path): for dirname, dirnames, filenames in os.walk(os.path.join(path)): log.debug("scanDisk: directory name: %s" %dirname) if re.search('_unpack_', dirname, re.IGNORECASE): log.debug("scanDisk: found a unpack directory, skipping") continue if autosub.SKIPHIDDENDIRS and os.path.split(dirname)[1].startswith(u'.'): continue if re.search('_failed_', dirname, re.IGNORECASE): log.debug("scanDisk: found a failed directory, skipping") continue for filename in filenames: splitname = filename.split(".") ext = splitname[len(splitname) - 1] if ext in ('avi', 'mkv', 'wmv', 'ts', 'mp4'): if re.search('sample', filename): continue # What subtitle files should we expect? if autosub.SUBNL != "": srtfile = os.path.splitext(filename)[0] + u"." + autosub.SUBNL + u".srt" else: srtfile = os.path.splitext(filename)[0] + u".srt" srtfileeng = os.path.splitext(filename)[0] + u"." + autosub.SUBENG + u".srt" if not os.path.exists(os.path.join(dirname, srtfile)) or (not os.path.exists(os.path.join(dirname, srtfileeng)) and autosub.DOWNLOADENG): log.debug("scanDir: File %s is missing a subtitle" % filename) lang = [] filenameResults = ProcessFilename(os.path.splitext(filename)[0], os.path.splitext(filename)[1]) if 'title' in filenameResults.keys(): if 'season' in filenameResults.keys(): if 'episode' in filenameResults.keys(): title = filenameResults['title'] season = filenameResults['season'] episode = filenameResults['episode'] if autosub.Helpers.SkipShow(title, season, episode) == True: log.debug("scanDir: SkipShow returned True") log.info("scanDir: Skipping %s - Season %s Episode %s" % (title, season, episode)) continue log.info("scanDir: Dutch subtitle wanted for %s and added to wantedQueue" % filename) filenameResults['originalFileLocationOnDisk'] = os.path.join(dirname, filename) filenameResults['timestamp'] = unicode(time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(os.path.getctime(filenameResults['originalFileLocationOnDisk'])))) if not os.path.exists(os.path.join(dirname, srtfile)): lang.append('nl') if not os.path.exists(os.path.join(dirname, srtfileeng)) and (autosub.FALLBACKTOENG or autosub.DOWNLOADENG): lang.append('en') filenameResults['lang'] = lang autosub.WANTEDQUEUE.append(filenameResults) else: log.error("scanDir: Could not process the filename properly filename: %s" % filename) continue else: log.error("scanDir: Could not process the filename properly filename: %s" % filename) continue else: log.error("scanDir: Could not process the filename properly filename: %s" % filename) continue
def run(self): log.debug("checkRSS: Starting round of RSS checking") if autosub.WANTEDQUEUELOCK: log.debug("checkRSS: Exiting, another threat is using the queues") return False else: autosub.WANTEDQUEUELOCK = True toDelete_wantedQueue = [] langs = ["nl"] # default is to only check for Dutch subs # but if English should be downloaden, check them too # It is very important that the dutch language is run first! if autosub.FALLBACKTOENG or autosub.DOWNLOADENG: langs.append("en") log.debug("checkRSS: We also want to check the English RSS feed") for lang in langs: if lang == "en": RSSURL = autosub.ENRSSURL log.debug("checkRSS: Now using the English RSS feed") else: RSSURL = autosub.NLRSSURL log.debug("checkRSS: Now using the Dutch RSS feed") try: bierdopjeapi = API(RSSURL, RSS=True) soup = BeautifulStoneSoup(bierdopjeapi.resp.read()) log.debug("checkRss: Succussfully connected to %s" % RSSURL) bierdopjeapi.close() except: log.error( "checkRss: The server returned an error for request %s" % RSSURL) autosub.WANTEDQUEUELOCK = False continue # Parse all the item-tags from the RSSFeed # The information that is parsed is: title, link and show_id # The show_id is later used to match with the wanted items # The title is used the determine the quality / source / releasegrp rssItemList = [] items = soup.findAll('item') if not len(items) > 0: log.error("checkRss: invalid RssFeed") log.debug("checkRss: dumping Rssfeed %s" % (soup.prettify())) else: log.debug("checkRss: Valid RssFeed") for x in items: soupx = BeautifulStoneSoup(str(x)) title = soupx.find('title').string show_id = soupx.find('show_id').string link = soupx.find('enclosure')['url'] item = {} item['title'] = title item['link'] = link item['show_id'] = show_id log.debug("checkRSS: Parsed from RSSFEED: %s %s %s" % (title, link, show_id)) rssItemList.append(item) normalizedRssTitleList = [] # Now we create a new rsslist, containing information like: episode, season, etc for item in rssItemList: title = item['title'] link = item['link'] show_id = item['show_id'] log.debug( "checkRSS: Normalizing the following entry in the RSS results: %s" % title) normalizedRssTitle = ProcessFilename(title, '') normalizedRssTitle['rssfile'] = title normalizedRssTitle['link'] = link normalizedRssTitle['show_id'] = str(show_id) if 'title' in normalizedRssTitle.keys(): if 'season' in normalizedRssTitle.keys(): if 'episode' in normalizedRssTitle.keys(): normalizedRssTitleList.append(normalizedRssTitle) #check versus wantedItem list for index, wantedItem in enumerate(autosub.WANTEDQUEUE): wantedItemquality = None wantedItemreleasegrp = None wantedItemsource = None wantedItemcodec = None wantedItemtitle = wantedItem['title'] wantedItemseason = wantedItem['season'] wantedItemepisode = wantedItem['episode'] originalfile = wantedItem['originalFileLocationOnDisk'] if lang not in wantedItem['lang']: continue if 'quality' in wantedItem.keys(): wantedItemquality = wantedItem['quality'] if 'releasegrp' in wantedItem.keys(): wantedItemreleasegrp = wantedItem['releasegrp'] if 'source' in wantedItem.keys(): wantedItemsource = wantedItem['source'] if 'codec' in wantedItem.keys(): wantedItemcodec = wantedItem['codec'] #lets try to find a showid showid = autosub.Helpers.getShowid(wantedItemtitle) #no showid? skip this item if not showid: continue for normalizedRssTitle in normalizedRssTitleList: downloadLink = None normalizedRssTitleseason = normalizedRssTitle['season'] normalizedRssTitleepisode = normalizedRssTitle['episode'] normalizedRssTitlerssfile = normalizedRssTitle['rssfile'] normalizedRssTitleshowid = int( normalizedRssTitle['show_id']) normalizedRssTitlelink = normalizedRssTitle['link'] log.debug( "checkRSS: Trying to match ID:%r S:%r E:%r (wantedlist) with ID:%r S:%r E:%r (rss)" % (showid, wantedItemseason, wantedItemepisode, normalizedRssTitleshowid, normalizedRssTitleseason, normalizedRssTitleepisode)) if showid == normalizedRssTitleshowid and wantedItemseason == normalizedRssTitleseason and wantedItemepisode == normalizedRssTitleepisode: log.debug( "checkRSS: The episode %s - Season %s Episode %s was found in the RSS list, attempting to match a proper match" % (wantedItemtitle, wantedItemseason, wantedItemepisode)) score = autosub.Helpers.scoreMatch( normalizedRssTitle, normalizedRssTitlerssfile, wantedItemquality, wantedItemreleasegrp, wantedItemsource, wantedItemcodec) if score >= autosub.MINMATCHSCORERSS: log.debug( "checkRss: A match got a high enough score. MinMatchscore is %s " % autosub.MINMATCHSCORERSS) downloadLink = normalizedRssTitlelink + autosub.APIRSS log.info( "checkRss: Got a match, matching file is: %s" % normalizedRssTitlerssfile) log.debug( "checkRss: Dumping downloadlink for debug purpose: %s" % downloadLink) if downloadLink: originalfile = wantedItem[ 'originalFileLocationOnDisk'] # Dutch subs if autosub.SUBNL != "" and lang == "nl": srtfile = os.path.splitext(originalfile)[ 0] + u"." + autosub.SUBNL + u".srt" elif lang == "nl": srtfile = os.path.splitext( originalfile)[0] + u".srt" # English subs if autosub.SUBENG != "" and lang == "en": srtfile = os.path.splitext(originalfile)[ 0] + u"." + autosub.SUBENG + u".srt" elif lang == "en": srtfile = os.path.splitext( originalfile)[0] + u".srt" wantedItem['downloadLink'] = downloadLink wantedItem[ 'destinationFileLocationOnDisk'] = srtfile log.info( "checkRSS: The episode %s - Season %s Episode %s has a matching subtitle on the RSSFeed, downloading it!" % (wantedItemtitle, wantedItemseason, wantedItemepisode)) downloadItem = wantedItem.copy() downloadItem['downlang'] = lang downloadItem[ 'subtitle'] = normalizedRssTitlerssfile if not DownloadSub(downloadItem): break if lang == 'nl' and ( autosub.FALLBACKTOENG and not autosub.DOWNLOADENG ) and 'en' in wantedItem['lang']: log.debug( 'checkRss: We found a dutch subtitle and fallback is true. Removing the english subtitle from the wantedlist.' ) wantedItem['lang'].remove('en') wantedItem['lang'].remove(lang) if len(wantedItem['lang']) == 0: toDelete_wantedQueue.append(index) break else: log.debug( "checkRss: Matching score is not high enough. Score is %s should be %s" % (str(score), autosub.MINMATCHSCORERSS)) i = len(toDelete_wantedQueue) - 1 while i >= 0: log.debug( "checkRSS: Removed item from the wantedQueue at index %s" % toDelete_wantedQueue[i]) autosub.WANTEDQUEUE.pop(toDelete_wantedQueue[i]) i = i - 1 # Resetting the toDelete queue for the next run (if need) toDelete_wantedQueue = [] log.debug("checkRSS: Finished round of RSS checking") autosub.WANTEDQUEUELOCK = False return True
def SubtitleSeeker(showid, lang, releaseDetails, sourceWebsites): # Get the scored list for all SubtitleSeeker hits api = autosub.API season = releaseDetails['season'] if 'season' in releaseDetails.keys( ) else None episode = releaseDetails['episode'] if 'episode' in releaseDetails.keys( ) else None quality = releaseDetails['quality'] if 'quality' in releaseDetails.keys( ) else None releasegrp = releaseDetails[ 'releasegrp'] if 'releasegrp' in releaseDetails.keys() else None source = releaseDetails['source'] if 'source' in releaseDetails.keys( ) else None codec = releaseDetails['codec'] if 'codec' in releaseDetails.keys( ) else None # this is the API search getSubLinkUrl = "%s&imdb=%s&season=%s&episode=%s&language=%s" % ( api, showid, season, episode, lang) log.info('getSubLinks: SubtitleSeeker request URL: %s' % getSubLinkUrl) if autosub.Helpers.checkAPICallsSubSeeker(use=True): try: subseekerapi = autosub.Helpers.API(getSubLinkUrl) dom = minidom.parse(subseekerapi.resp) subseekerapi.resp.close() except: log.error( "getSubLink: The server returned an error for request %s" % getSubLinkUrl) return None else: log.error("API: out of api calls for SubtitleSeeker.com") return None if not len(dom.getElementsByTagName('error')) == 0: for error in dom.getElementsByTagName('error'): try: errormsg = error.getElementsByTagName('msg')[0].firstChild.data log.error("getSubLink: Error found in API response: %s" % errormsg) except AttributeError: log.debug( "getSubLink: Invalid msg tag in API response, unable to read error message." ) return None if not dom or len(dom.getElementsByTagName('item')) == 0: return None scoreList = [] for sub in dom.getElementsByTagName('item'): try: release = sub.getElementsByTagName('release')[0].firstChild.data release = release.lower() # Remove the .srt extension some of the uploaders leave on the file if release.endswith(".srt"): release = release[:-4] except AttributeError: log.debug( "getSubLink: Invalid release tag in API response, skipping this item." ) continue try: website = sub.getElementsByTagName('site')[0].firstChild.data website = website.lower() except AttributeError: log.debug( "getSubLink: Invalid website tag in API response, skipping this item." ) continue try: url = sub.getElementsByTagName('url')[0].firstChild.data except AttributeError: log.debug( "getSubLink: Invalid URL tag in API response, skipping this item." ) continue tmpDict = ProcessFilename(release, '') if not website in sourceWebsites or not tmpDict: continue # ReleaseDict is a dictionary with the score, releasename and source website for the subtitle release releaseDict = { 'score': None, 'releasename': release, 'url': url, 'website': website } releaseDict['score'] = autosub.Helpers.scoreMatch( tmpDict, release, quality, releasegrp, source, codec) scoreList.append(releaseDict)
def Opensubtitles(EpisodeId, language, releaseDetails): title = releaseDetails['title'] if 'title' in releaseDetails.keys( ) else None season = releaseDetails['season'] if 'season' in releaseDetails.keys( ) else None episode = releaseDetails['episode'] if 'episode' in releaseDetails.keys( ) else None quality = releaseDetails['quality'] if 'quality' in releaseDetails.keys( ) else None releasegrp = releaseDetails[ 'releasegrp'] if 'releasegrp' in releaseDetails.keys() else None source = releaseDetails['source'] if 'source' in releaseDetails.keys( ) else None codec = releaseDetails['codec'] if 'codec' in releaseDetails.keys( ) else None LangId = 'dut' if language == 'Dutch' else 'eng' SearchUrl = '/xml/search/sublanguageid-' + str(LangId) + '/imdbid-' + str( EpisodeId) try: TimeOut() RequestResult = autosub.OPENSUBTTITLESSESSION.get( autosub.OPENSUBTITLESURL + SearchUrl, timeout=10) Referer = SearchUrl.replace('/xml', '') autosub.OPENSUBTTITLESSESSION.headers.update({'referer': Referer}) except: log.debug('getSubLinks: Could not connect to OpenSubtitles.') return None if 'text/xml' not in RequestResult.headers['Content-Type']: log.error('getSubLinks: OpenSubtitles responded with an error') return None try: root = ET.fromstring(RequestResult.content) except: log.debug( 'getSubLinks: Serie with IMDB ID %s could not be found on OpenSubtitles.' % SerieImdb) return None try: SubTitles = root.find('.//search/results') except: log.debug( 'getSubLinks: Serie with IMDB ID %s could not be found on OpenSubtitles.' % SerieImdb) return None # We fetch the show overview page and search voor the Id's of the epiode we want # Because as we have this whole page, we put the other Episode Id's in the cache scoreList = [] for Sub in SubTitles: try: if Sub.tag != 'subtitle': continue try: SubBad = int(Sub.find('SubBad').text) if SubBad > 0: log.debug( "getSubLinks: OpenSubtitles has %d bad reports for this subtitle, skipping." % SubBad) continue else: log.debug( "getSubLinks: OpenSubtitles has %d bad reports for this subtitle." % SubBad) except: pass Link = Sub.find('IDSubtitle').attrib['Link'] release = Sub.find('MovieReleaseName').text.split('[]')[0].lower() except: continue url = autosub.OPENSUBTITLESURL[:-3] + Link + '/xml' tmpDict = ProcessFilename(release, '') if not tmpDict: continue # ReleaseDict is a dictionary with the score, releasename and source website for the subtitle release releaseDict = { 'score': None, 'releasename': release, 'url': url, 'website': 'opensubtitles.org' } releaseDict['score'] = autosub.Helpers.scoreMatch( tmpDict, release, quality, releasegrp, source, codec) scoreList.append(releaseDict) return scoreList
def walkDir(path): for dirname, dirnames, filenames in os.walk(path): log.debug("scanDisk: directory name: %s" % dirname) if re.search("_unpack_", dirname, re.IGNORECASE): log.debug("scanDisk: found a unpack directory, skipping") continue if autosub.SKIPHIDDENDIRS and os.path.split(dirname)[1].startswith(u"."): continue if re.search("_failed_", dirname, re.IGNORECASE): log.debug("scanDisk: found a failed directory, skipping") continue if re.search("@eaDir", dirname, re.IGNORECASE): log.debug("scanDisk: found a Synology indexing directory, skipping this folder and all subfolders.") tmpdirs = dirnames[:] for dir in tmpdirs: dirnames.remove(dir) continue if re.search("@.*thumb", dirname, re.IGNORECASE): # Ingnore QNAP multimedia thumbnail folders continue for filename in filenames: splitname = filename.split(".") ext = splitname[len(splitname) - 1] if ext in ("avi", "mkv", "wmv", "ts", "mp4"): if re.search("sample", filename): continue if autosub.WEBDL == "None": if re.search("web-dl", filename.lower()): log.debug("scanDisk: WEB-DL is set to 'None', skipping %s" % filename) continue if not platform.system() == "Windows": # Get best ascii compatible character for special characters try: if not isinstance(filename, unicode): coding = detect(filename) filename = unicode(filename.decode(coding)) correctedFilename = "".join( (c for c in unicodedata.normalize("NFD", filename) if unicodedata.category(c) != "Mn") ) if filename != correctedFilename: os.rename(os.path.join(dirname, filename), os.path.join(dirname, correctedFilename)) log.info("scanDir: Renamed file %s" % correctedFilename) filename = correctedFilename except: log.error("scanDir: Skipping directory %s" % dirname) log.error("scanDir: Skipping file %s" % filename) continue # What subtitle files should we expect? lang = [] # Check what the Dutch subtitle would be. if autosub.SUBNL != "": srtfilenl = os.path.splitext(filename)[0] + u"." + autosub.SUBNL + u".srt" else: srtfilenl = os.path.splitext(filename)[0] + u".srt" # Check what the English subtitle would be. if autosub.SUBENG == "": # Check for overlapping names if autosub.SUBNL != "" or not autosub.DOWNLOADDUTCH: srtfileeng = os.path.splitext(filename)[0] + u".srt" # Hardcoded fallback else: srtfileeng = os.path.splitext(filename)[0] + u".en.srt" else: srtfileeng = os.path.splitext(filename)[0] + u"." + autosub.SUBENG + u".srt" # Check which languages we want to download based on user settings. if autosub.DOWNLOADDUTCH: # If the Dutch subtitle doesn't exist, then add it to the wanted list. if not os.path.exists(os.path.join(dirname, srtfilenl)): lang.append(autosub.DUTCH) if autosub.DOWNLOADENG: # If the English subtitle doesn't exist, then add it to the wanted list. if not os.path.exists(os.path.join(dirname, srtfileeng)): if autosub.WEBDL == "DutchOnly" and re.search("web-dl", filename.lower()): log.debug( "scanDisk: WEB-DL is set to 'Dutch Only', not adding English as wanted for %s" % filename ) else: lang.append(autosub.ENGLISH) if (autosub.FALLBACKTOENG and autosub.DOWNLOADDUTCH) and not autosub.DOWNLOADENG: # If the Dutch and English subtitles do not exist, then add English to the wanted list. if not os.path.exists(os.path.join(dirname, srtfilenl)) and not os.path.exists( os.path.join(dirname, srtfileeng) ): if autosub.WEBDL == "DutchOnly" and re.search("web-dl", filename.lower()): log.debug( "scanDisk: WEB-DL is set to 'Dutch Only', not adding English as wanted for %s" % filename ) else: lang.append(autosub.ENGLISH) if not lang: # autosub.WANTEDQUEUE empty continue log.debug("scanDir: File %s is missing subtitle(s): %s" % (filename, ", ".join(map(str, lang)))) filenameResults = ProcessFilename(os.path.splitext(filename)[0], os.path.splitext(filename)[1]) if "title" in filenameResults.keys(): if "season" in filenameResults.keys(): if "episode" in filenameResults.keys(): title = filenameResults["title"] season = filenameResults["season"] episode = filenameResults["episode"] if ( not filenameResults["releasegrp"] and not filenameResults["source"] and not filenameResults["quality"] and not filenameResults["source"] ): continue if autosub.Helpers.SkipShow(title, season, episode) == True: log.debug("scanDir: SkipShow returned True") log.info("scanDir: Skipping %s - Season %s Episode %s" % (title, season, episode)) continue if len(lang) == 1: log.info( "scanDir: %s subtitle wanted for %s and added to wantedQueue" % (lang[0], filename) ) else: log.info( "scanDir: %s subtitles wanted for %s and added to wantedQueue" % (" and ".join(map(str, lang)), filename) ) filenameResults["originalFileLocationOnDisk"] = os.path.join(dirname, filename) filenameResults["timestamp"] = unicode( time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(os.path.getctime(filenameResults["originalFileLocationOnDisk"])), ) ) filenameResults["lang"] = lang autosub.WANTEDQUEUE.append(filenameResults) else: log.error("scanDir: Could not process the filename properly filename: %s" % filename) continue else: log.error("scanDir: Could not process the filename properly filename: %s" % filename) continue else: log.error("scanDir: Could not process the filename properly filename: %s" % filename) continue