def walkDir(path): SkipListNL = autosub.SKIPSTRINGNL.split(",") if len(autosub.SKIPSTRINGNL) > 0 else [] SkipListEN = autosub.SKIPSTRINGEN.split(",") if len(autosub.SKIPSTRINGEN) > 0 else [] if len(autosub.SKIPFOLDERSNL) == 0: SkipFoldersNL = [] else: SkipFoldersNL = autosub.SKIPFOLDERSNL.split(",") if len(autosub.SKIPFOLDERSNL) > 0 else [] for idx,folder in enumerate(SkipFoldersNL): SkipFoldersNL[idx] = os.path.normpath(os.path.join(path,folder.rstrip("\/").lstrip("\/"))) if len(autosub.SKIPFOLDERSNL) == 0: SkipFoldersNL = [] else: SkipFoldersNL = autosub.SKIPFOLDERSNL.split(",") if len(autosub.SKIPFOLDERSNL) > 0 else [] for idx,folder in enumerate(SkipFoldersNL): SkipFoldersNL[idx] = os.path.normpath(path + folder) if len(autosub.SKIPFOLDERSEN) == 0: SkipFoldersEN = [] else: SkipFoldersEN = autosub.SKIPFOLDERSEN.split(",") if len(autosub.SKIPFOLDERSEN) > 0 else [] for idx,folder in enumerate(SkipFoldersEN): SkipFoldersEN[idx] = os.path.normpath(path + folder) for dirname, dirnames, filenames in os.walk(path, True, WalkError): SkipThisFolderNL = False for skip in SkipFoldersNL: if dirname.startswith(skip): SkipThisFolderNL = True break SkipThisFolderEN = False for skip in SkipFoldersEN: if dirname.startswith(skip): SkipThisFolderEN = True break log.debug("scanDisk: directory name: %s" %dirname) if re.search('_unpack_', dirname, re.IGNORECASE): log.debug("scanDisk: found a unpack directory, skipping.") continue if autosub.SKIPHIDDENDIRS and os.path.split(dirname)[1].startswith(u'.'): continue if re.search('_failed_', dirname, re.IGNORECASE): log.debug("scanDisk: found a failed directory, skipping.") continue if re.search('@eaDir', dirname, re.IGNORECASE): log.debug("scanDisk: found a Synology indexing directory, skipping.") tmpdirs = dirnames[:] for dir in tmpdirs: dirnames.remove(dir) continue if re.search("@.*thumb", dirname, re.IGNORECASE): log.debug("scanDisk: found a Qnap multimedia thumbnail folder, skipping.") continue langs = [] FileDict = {} for filename in filenames: root,ext = os.path.splitext(filename) if ext[1:] in ('avi', 'mkv', 'wmv', 'ts', 'mp4'): if re.search('sample', filename): continue if not platform.system() == 'Windows': # Get best ascii compatible character for special characters try: if not isinstance(filename, unicode): coding = detect(filename)['encoding'] filename = unicode(filename.decode(coding),errors='replace') correctedFilename = ''.join((c for c in unicodedata.normalize('NFD', filename) if unicodedata.category(c) != 'Mn')) if filename != correctedFilename: os.rename(os.path.join(dirname, filename), os.path.join(dirname, correctedFilename)) log.info("scanDir: Renamed file %s" % correctedFilename) filename = correctedFilename except: log.error("scanDir: Skipping directory, file %s, %s" % (dirname,filename)) continue # What subtitle files should we expect? langs = [] NLext = u'.' + autosub.SUBNL + u'.srt' if autosub.SUBNL else u'.srt' ENext = u'.' + autosub.SUBENG + u'.srt' if autosub.SUBENG else u'.srt' ENext = u'.en.srt'if NLext == ENext and autosub.DOWNLOADDUTCH else ENext if not os.access(dirname, os.W_OK): log.error('scandisk: No write access to folder: %s' % dirname) continue # Check which languages we want to download based on user settings. if autosub.DOWNLOADDUTCH and not SkipThisFolderNL: Skipped = False for SkipItem in SkipListNL: if not SkipItem: break if re.search(SkipItem.lower(), filename.lower()): Skipped = True break if Skipped: log.info("scanDir: %s found in %s so skipped for Dutch subs" % (SkipItem, filename)) elif os.path.exists(os.path.join(dirname, root + NLext)): Skipped = True log.debug("scanDir: %s skipped because the Dutch subtitle already exists" % filename) else: # If the Dutch subtitle not skipped and doesn't exist, then add it to the wanted list langs.append(autosub.DUTCH) if (autosub.DOWNLOADENG or (autosub.FALLBACKTOENG and autosub.DOWNLOADDUTCH and not Skipped)) and not SkipThisFolderEN: Skipped = False for SkipItem in SkipListEN: if not SkipItem: break if re.search(SkipItem.lower(), filename.lower()): Skipped = True break if Skipped: log.info("scanDir: %s found in %s so skipped for English subs" % (SkipItem, filename)) elif os.path.exists(os.path.join(dirname, root + ENext)): log.debug("scanDir: %s skipped because the English subtitle already exists" % filename) else: # If the English subtitle not skipped and doesn't exist, then add it to the wanted list if not os.path.exists(os.path.join(dirname, root + ENext)): langs.append(autosub.ENGLISH) if not langs: # nothing to do for this file continue FileDict = ProcessFilename(os.path.splitext(filename)[0].strip(), ext) time.sleep(0) if not FileDict: continue if not 'title' in FileDict.keys() or not 'season' in FileDict.keys() or not 'episode' in FileDict.keys(): continue if not FileDict['releasegrp'] and not FileDict['source'] and not FileDict['quality'] and not FileDict['source']: log.error("scanDir: Not enough info in filename: %s" % filename) continue FileDict['timestamp'] = unicode(time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(os.path.getctime(os.path.join(dirname, filename))))) FileDict['langs'] = langs FileDict['NLext'] = NLext FileDict['ENext'] = ENext FileDict['file'] = root FileDict['container'] = ext FileDict['folder'] = dirname FileDict['ImdbId'],FileDict['A7Id'], FileDict['TvdbId'], FileDict['title'] = Helpers.getShowid(FileDict['title'],autosub.ADDIC7EDLOGGED_IN) if autosub.Helpers.SkipShow(FileDict['ImdbId'],FileDict['title'], FileDict['season'], FileDict['episode']): continue log.info("scanDir: %s WANTED FOR: %s" % (langs, filename)) autosub.WANTEDQUEUE.append(FileDict) return
def walkDir(path): SkipListNL = [] SkipListEN = [] SkipListNL = autosub.SKIPSTRINGNL.split(",") SkipListEN = autosub.SKIPSTRINGEN.split(",") # Here we use os.walk to find all files in the path. for dirname, dirnames, filenames in os.walk(path): log.debug("scanDisk: directory name: %s" % dirname) if re.search('_unpack_', dirname, re.IGNORECASE): log.debug("scanDisk: found a unpack directory, skipping") continue if autosub.SKIPHIDDENDIRS and os.path.split(dirname)[1].startswith( u'.'): continue if re.search('_failed_', dirname, re.IGNORECASE): log.debug("scanDisk: found a failed directory, skipping") continue if re.search('@eaDir', dirname, re.IGNORECASE): log.debug( "scanDisk: found a Synology indexing directory, skipping this folder and all subfolders." ) tmpdirs = dirnames[:] for dir in tmpdirs: dirnames.remove(dir) continue if re.search("@.*thumb", dirname, re.IGNORECASE): # Ingnore QNAP multimedia thumbnail folders continue for filename in filenames: splitname = filename.split(".") ext = splitname[len(splitname) - 1] if ext in ('avi', 'mkv', 'wmv', 'ts', 'mp4'): if re.search('sample', filename): continue if not platform.system() == 'Windows': # Get best ascii compatible character for special characters try: if not isinstance(filename, unicode): coding = detect(filename) filename = unicode(filename.decode(coding)) correctedFilename = ''.join( (c for c in unicodedata.normalize('NFD', filename) if unicodedata.category(c) != 'Mn')) if filename != correctedFilename: os.rename(os.path.join(dirname, filename), os.path.join(dirname, correctedFilename)) log.info("scanDir: Renamed file %s" % correctedFilename) filename = correctedFilename except: log.error("scanDir: Skipping directory %s" % dirname) log.error("scanDir: Skipping file %s" % filename) continue # What subtitle files should we expect? lang = [] #Check what the Dutch subtitle would be. if autosub.SUBNL != "": srtfilenl = os.path.splitext( filename)[0] + u"." + autosub.SUBNL + u".srt" else: srtfilenl = os.path.splitext(filename)[0] + u".srt" #Check what the English subtitle would be. if autosub.SUBENG == "": # Check for overlapping names if autosub.SUBNL != "" or not autosub.DOWNLOADDUTCH: srtfileeng = os.path.splitext(filename)[0] + u".srt" # Hardcoded fallback else: srtfileeng = os.path.splitext(filename)[0] + u".en.srt" else: srtfileeng = os.path.splitext( filename)[0] + u"." + autosub.SUBENG + u".srt" # Check which languages we want to download based on user settings and check the skipstring if autosub.DOWNLOADDUTCH: Skipped = False for SkipItem in SkipListNL: if re.search(SkipItem.lower(), filename.lower()): Skipped = True log.debug( "scanDir: %s found in %s so skipped for Dutch subs" % (SkipItem, filename)) break if not Skipped and os.path.exists( os.path.join(dirname, srtfilenl)): Skipped = True log.debug( "scanDir: %s skipped because the subtitle already exists" % filename) # If the Dutch subtitle doesn't exist en has no skip item, then add it to the wanted list. if not Skipped: lang.append(autosub.DUTCH) if autosub.DOWNLOADENG: Skipped = False for SkipItem in SkipListEN: if re.search(SkipItem.lower(), filename.lower()): Skipped = True log.debug( "scanDir: %s found in %s so skipped for English subs" % (SkipItem, filename)) break if not Skipped and os.path.exists( os.path.join(dirname, srtfileeng)): Skipped = True log.debug( "scanDir: %s skipped because the subtitle already exists" % filename) # If the English subtitle doesn't exist en has no skip item, then add it to the wanted list. if not Skipped: lang.append(autosub.ENGLISH) if (autosub.FALLBACKTOENG and autosub.DOWNLOADDUTCH ) and not autosub.DOWNLOADENG and not Skipped: # If the Dutch and English subtitles do not exist and not skipped, then add English to the wanted list. if not os.path.exists(os.path.join( dirname, srtfilenl)) and not os.path.exists( os.path.join(dirname, srtfileeng)): lang.append(autosub.ENGLISH) if not lang: # autosub.WANTEDQUEUE empty continue log.debug("scanDir: File %s is missing subtitle(s): %s" % (filename, ', '.join(map(str, lang)))) filenameResults = ProcessFilename( os.path.splitext(filename)[0], os.path.splitext(filename)[1]) if 'title' in filenameResults.keys(): if 'season' in filenameResults.keys(): if 'episode' in filenameResults.keys(): title = filenameResults['title'] season = filenameResults['season'] episode = filenameResults['episode'] if not filenameResults[ 'releasegrp'] and not filenameResults[ 'source'] and not filenameResults[ 'quality'] and not filenameResults[ 'source']: continue if autosub.Helpers.SkipShow( title, season, episode) == True: log.debug("scanDir: SkipShow returned True") log.info( "scanDir: Skipping %s - Season %s Episode %s" % (title, season, episode)) continue if len(lang) == 1: log.info( "scanDir: %s subtitle wanted for %s and added to wantedQueue" % (lang[0], filename)) else: log.info( "scanDir: %s subtitles wanted for %s and added to wantedQueue" % (' and '.join(map(str, lang)), filename)) filenameResults[ 'originalFileLocationOnDisk'] = os.path.join( dirname, filename) filenameResults['timestamp'] = unicode( time.strftime( '%Y-%m-%d %H:%M:%S', time.localtime( os.path.getctime(filenameResults[ 'originalFileLocationOnDisk'])))) filenameResults['lang'] = lang autosub.WANTEDQUEUE.append(filenameResults) else: log.error( "scanDir: Could not process the filename properly filename: %s" % filename) continue else: log.error( "scanDir: Could not process the filename properly filename: %s" % filename) continue else: log.error( "scanDir: Could not process the filename properly filename: %s" % filename) continue
def run(self): log.debug("checkRSS: Starting round of RSS checking") if autosub.WANTEDQUEUELOCK: log.debug("checkRSS: Exiting, another threat is using the queues") return False else: autosub.WANTEDQUEUELOCK = True toDelete_wantedQueue = [] langs = ["nl"] # default is to only check for Dutch subs # but if English should be downloaden, check them too # It is very important that the dutch language is run first! if autosub.FALLBACKTOENG or autosub.DOWNLOADENG: langs.append("en") log.debug("checkRSS: We also want to check the English RSS feed") for lang in langs: if lang == "en": RSSURL = autosub.ENRSSURL log.debug("checkRSS: Now using the English RSS feed") else: RSSURL = autosub.NLRSSURL log.debug("checkRSS: Now using the Dutch RSS feed") try: bierdopjeapi = API(RSSURL,RSS=True) soup = BeautifulStoneSoup(bierdopjeapi.resp.read()) log.debug("checkRss: Succussfully connected to %s" %RSSURL) bierdopjeapi.close() except: log.error("checkRss: The server returned an error for request %s" % RSSURL) autosub.WANTEDQUEUELOCK = False continue # Parse all the item-tags from the RSSFeed # The information that is parsed is: title, link and show_id # The show_id is later used to match with the wanted items # The title is used the determine the quality / source / releasegrp rssItemList = [] items = soup.findAll('item') if not len(items) > 0: log.error("checkRss: invalid RssFeed") log.debug("checkRss: dumping Rssfeed %s" %(soup.prettify())) else: log.debug("checkRss: Valid RssFeed") for x in items: soupx = BeautifulStoneSoup(str(x)) title = soupx.find('title').string show_id = soupx.find('show_id').string link = soupx.find('enclosure')['url'] item = {} item['title'] = title item['link'] = link item['show_id'] = show_id log.debug("checkRSS: Parsed from RSSFEED: %s %s %s" %(title,link,show_id)) rssItemList.append(item) normalizedRssTitleList = [] # Now we create a new rsslist, containing information like: episode, season, etc for item in rssItemList: title = item['title'] link = item['link'] show_id = item['show_id'] log.debug("checkRSS: Normalizing the following entry in the RSS results: %s" % title) normalizedRssTitle = ProcessFilename(title, '') normalizedRssTitle['rssfile'] = title normalizedRssTitle['link'] = link normalizedRssTitle['show_id'] = str(show_id) if 'title' in normalizedRssTitle.keys(): if 'season' in normalizedRssTitle.keys(): if 'episode' in normalizedRssTitle.keys(): normalizedRssTitleList.append(normalizedRssTitle) #check versus wantedItem list for index, wantedItem in enumerate(autosub.WANTEDQUEUE): wantedItemquality = None wantedItemreleasegrp = None wantedItemsource = None wantedItemcodec = None wantedItemtitle = wantedItem['title'] wantedItemseason = wantedItem['season'] wantedItemepisode = wantedItem['episode'] originalfile = wantedItem['originalFileLocationOnDisk'] if lang not in wantedItem['lang']: continue if 'quality' in wantedItem.keys(): wantedItemquality = wantedItem['quality'] if 'releasegrp' in wantedItem.keys(): wantedItemreleasegrp = wantedItem['releasegrp'] if 'source' in wantedItem.keys(): wantedItemsource = wantedItem['source'] if 'codec' in wantedItem.keys(): wantedItemcodec = wantedItem['codec'] #lets try to find a showid showid = autosub.Helpers.getShowid(wantedItemtitle) #no showid? skip this item if not showid: continue for normalizedRssTitle in normalizedRssTitleList: downloadLink = None normalizedRssTitleseason = normalizedRssTitle['season'] normalizedRssTitleepisode = normalizedRssTitle['episode'] normalizedRssTitlerssfile = normalizedRssTitle['rssfile'] normalizedRssTitleshowid = int(normalizedRssTitle['show_id']) normalizedRssTitlelink = normalizedRssTitle['link'] log.debug("checkRSS: Trying to match ID:%r S:%r E:%r (wantedlist) with ID:%r S:%r E:%r (rss)" %(showid, wantedItemseason, wantedItemepisode, normalizedRssTitleshowid, normalizedRssTitleseason, normalizedRssTitleepisode)) if showid == normalizedRssTitleshowid and wantedItemseason == normalizedRssTitleseason and wantedItemepisode == normalizedRssTitleepisode: log.debug("checkRSS: The episode %s - Season %s Episode %s was found in the RSS list, attempting to match a proper match" % (wantedItemtitle, wantedItemseason, wantedItemepisode)) score = autosub.Helpers.scoreMatch(normalizedRssTitle, normalizedRssTitlerssfile, wantedItemquality, wantedItemreleasegrp, wantedItemsource, wantedItemcodec) if score >= autosub.MINMATCHSCORERSS: log.debug ("checkRss: A match got a high enough score. MinMatchscore is %s " % autosub.MINMATCHSCORERSS) downloadLink = normalizedRssTitlelink + autosub.APIRSS log.info ("checkRss: Got a match, matching file is: %s" %normalizedRssTitlerssfile) log.debug("checkRss: Dumping downloadlink for debug purpose: %s" %downloadLink) if downloadLink: originalfile = wantedItem['originalFileLocationOnDisk'] # Dutch subs if autosub.SUBNL != "" and lang == "nl": srtfile = os.path.splitext(originalfile)[0] + u"." + autosub.SUBNL + u".srt" elif lang == "nl": srtfile = os.path.splitext(originalfile)[0] + u".srt" # English subs if autosub.SUBENG != "" and lang == "en": srtfile = os.path.splitext(originalfile)[0] + u"." + autosub.SUBENG + u".srt" elif lang == "en": srtfile = os.path.splitext(originalfile)[0] + u".srt" wantedItem['downloadLink'] = downloadLink wantedItem['destinationFileLocationOnDisk'] = srtfile log.info("checkRSS: The episode %s - Season %s Episode %s has a matching subtitle on the RSSFeed, downloading it!" % (wantedItemtitle, wantedItemseason, wantedItemepisode)) downloadItem = wantedItem.copy() downloadItem['downlang'] = lang downloadItem['subtitle'] = normalizedRssTitlerssfile if not DownloadSub(downloadItem): break if lang == 'nl' and (autosub.FALLBACKTOENG and not autosub.DOWNLOADENG) and 'en' in wantedItem['lang']: log.debug('checkRss: We found a dutch subtitle and fallback is true. Removing the english subtitle from the wantedlist.') wantedItem['lang'].remove('en') wantedItem['lang'].remove(lang) if len(wantedItem['lang']) == 0: toDelete_wantedQueue.append(index) break else: log.debug("checkRss: Matching score is not high enough. Score is %s should be %s" %(str(score),autosub.MINMATCHSCORERSS)) i = len(toDelete_wantedQueue)-1 while i >= 0: log.debug("checkRSS: Removed item from the wantedQueue at index %s" % toDelete_wantedQueue[i]) autosub.WANTEDQUEUE.pop(toDelete_wantedQueue[i]) i = i-1 # Resetting the toDelete queue for the next run (if need) toDelete_wantedQueue =[] log.debug("checkRSS: Finished round of RSS checking") autosub.WANTEDQUEUELOCK = False return True
def walkDir(path): SkipListNL = [] SkipListEN = [] SkipListNL = autosub.SKIPSTRINGNL.split(",") SkipListEN = autosub.SKIPSTRINGEN.split(",") # Here we use os.walk to find all files in the path. for dirname, dirnames, filenames in os.walk(path): log.debug("scanDisk: directory name: %s" %dirname) if re.search('_unpack_', dirname, re.IGNORECASE): log.debug("scanDisk: found a unpack directory, skipping") continue if autosub.SKIPHIDDENDIRS and os.path.split(dirname)[1].startswith(u'.'): continue if re.search('_failed_', dirname, re.IGNORECASE): log.debug("scanDisk: found a failed directory, skipping") continue if re.search('@eaDir', dirname, re.IGNORECASE): log.debug("scanDisk: found a Synology indexing directory, skipping this folder and all subfolders.") tmpdirs = dirnames[:] for dir in tmpdirs: dirnames.remove(dir) continue if re.search("@.*thumb", dirname, re.IGNORECASE): # Ingnore QNAP multimedia thumbnail folders continue for filename in filenames: splitname = filename.split(".") ext = splitname[len(splitname) - 1] if ext in ('avi', 'mkv', 'wmv', 'ts', 'mp4'): if re.search('sample', filename): continue if not platform.system() == 'Windows': # Get best ascii compatible character for special characters try: if not isinstance(filename, unicode): coding = detect(filename) filename = unicode(filename.decode(coding)) correctedFilename = ''.join((c for c in unicodedata.normalize('NFD', filename) if unicodedata.category(c) != 'Mn')) if filename != correctedFilename: os.rename(os.path.join(dirname, filename), os.path.join(dirname, correctedFilename)) log.info("scanDir: Renamed file %s" % correctedFilename) filename = correctedFilename except: log.error("scanDir: Skipping directory %s" % dirname) log.error("scanDir: Skipping file %s" % filename) continue # What subtitle files should we expect? lang=[] #Check what the Dutch subtitle would be. if autosub.SUBNL != "": srtfilenl = os.path.splitext(filename)[0] + u"." + autosub.SUBNL + u".srt" else: srtfilenl = os.path.splitext(filename)[0] + u".srt" #Check what the English subtitle would be. if autosub.SUBENG == "": # Check for overlapping names if autosub.SUBNL != "" or not autosub.DOWNLOADDUTCH: srtfileeng = os.path.splitext(filename)[0] + u".srt" # Hardcoded fallback else: srtfileeng = os.path.splitext(filename)[0] + u".en.srt" else: srtfileeng = os.path.splitext(filename)[0] + u"." + autosub.SUBENG + u".srt" # Check which languages we want to download based on user settings and check the skipstring if autosub.DOWNLOADDUTCH: Skipped = False for SkipItem in SkipListNL: if re.search(SkipItem.lower(), filename.lower()): Skipped = True log.debug("scanDir: %s found in %s so skipped for Dutch subs" % (SkipItem, filename)) break if not Skipped and os.path.exists(os.path.join(dirname, srtfilenl)): Skipped = True log.debug("scanDir: %s skipped because the subtitle already exists" % filename) # If the Dutch subtitle doesn't exist en has no skip item, then add it to the wanted list. if not Skipped: lang.append(autosub.DUTCH) if autosub.DOWNLOADENG: Skipped = False for SkipItem in SkipListEN: if re.search(SkipItem.lower(), filename.lower()): Skipped = True log.debug("scanDir: %s found in %s so skipped for English subs" % (SkipItem, filename)) break if not Skipped and os.path.exists(os.path.join(dirname, srtfileeng)): Skipped = True log.debug("scanDir: %s skipped because the subtitle already exists" % filename) # If the English subtitle doesn't exist en has no skip item, then add it to the wanted list. if not Skipped: lang.append(autosub.ENGLISH) if (autosub.FALLBACKTOENG and autosub.DOWNLOADDUTCH) and not autosub.DOWNLOADENG and not Skipped: # If the Dutch and English subtitles do not exist and not skipped, then add English to the wanted list. if not os.path.exists(os.path.join(dirname, srtfilenl)) and not os.path.exists(os.path.join(dirname, srtfileeng)): lang.append(autosub.ENGLISH) if not lang: # autosub.WANTEDQUEUE empty continue log.debug("scanDir: File %s is missing subtitle(s): %s" % (filename, ', '.join(map(str,lang)))) filenameResults = ProcessFilename(os.path.splitext(filename)[0], os.path.splitext(filename)[1]) if 'title' in filenameResults.keys(): if 'season' in filenameResults.keys(): if 'episode' in filenameResults.keys(): title = filenameResults['title'] season = filenameResults['season'] episode = filenameResults['episode'] if not filenameResults['releasegrp'] and not filenameResults['source'] and not filenameResults['quality'] and not filenameResults['source']: continue if autosub.Helpers.SkipShow(title, season, episode) == True: log.debug("scanDir: SkipShow returned True") log.info("scanDir: Skipping %s - Season %s Episode %s" % (title, season, episode)) continue if len(lang) == 1: log.info("scanDir: %s subtitle wanted for %s and added to wantedQueue" % (lang[0], filename)) else: log.info("scanDir: %s subtitles wanted for %s and added to wantedQueue" % (' and '.join(map(str,lang)), filename)) filenameResults['originalFileLocationOnDisk'] = os.path.join(dirname, filename) filenameResults['timestamp'] = unicode(time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(os.path.getctime(filenameResults['originalFileLocationOnDisk'])))) filenameResults['lang'] = lang autosub.WANTEDQUEUE.append(filenameResults) else: log.error("scanDir: Could not process the filename properly filename: %s" % filename) continue else: log.error("scanDir: Could not process the filename properly filename: %s" % filename) continue else: log.error("scanDir: Could not process the filename properly filename: %s" % filename) continue
def walkDir(path): for dirname, dirnames, filenames in os.walk(os.path.join(path)): log.debug("scanDisk: directory name: %s" %dirname) if re.search('_unpack_', dirname, re.IGNORECASE): log.debug("scanDisk: found a unpack directory, skipping") continue if autosub.SKIPHIDDENDIRS and os.path.split(dirname)[1].startswith(u'.'): continue if re.search('_failed_', dirname, re.IGNORECASE): log.debug("scanDisk: found a failed directory, skipping") continue for filename in filenames: splitname = filename.split(".") ext = splitname[len(splitname) - 1] if ext in ('avi', 'mkv', 'wmv', 'ts', 'mp4'): if re.search('sample', filename): continue # What subtitle files should we expect? if autosub.SUBNL != "": srtfile = os.path.splitext(filename)[0] + u"." + autosub.SUBNL + u".srt" else: srtfile = os.path.splitext(filename)[0] + u".srt" srtfileeng = os.path.splitext(filename)[0] + u"." + autosub.SUBENG + u".srt" if not os.path.exists(os.path.join(dirname, srtfile)) or (not os.path.exists(os.path.join(dirname, srtfileeng)) and autosub.DOWNLOADENG): log.debug("scanDir: File %s is missing a subtitle" % filename) lang = [] filenameResults = ProcessFilename(os.path.splitext(filename)[0], os.path.splitext(filename)[1]) if 'title' in filenameResults.keys(): if 'season' in filenameResults.keys(): if 'episode' in filenameResults.keys(): title = filenameResults['title'] season = filenameResults['season'] episode = filenameResults['episode'] if autosub.Helpers.SkipShow(title, season, episode) == True: log.debug("scanDir: SkipShow returned True") log.info("scanDir: Skipping %s - Season %s Episode %s" % (title, season, episode)) continue log.info("scanDir: Dutch subtitle wanted for %s and added to wantedQueue" % filename) filenameResults['originalFileLocationOnDisk'] = os.path.join(dirname, filename) filenameResults['timestamp'] = unicode(time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(os.path.getctime(filenameResults['originalFileLocationOnDisk'])))) if not os.path.exists(os.path.join(dirname, srtfile)): lang.append('nl') if not os.path.exists(os.path.join(dirname, srtfileeng)) and (autosub.FALLBACKTOENG or autosub.DOWNLOADENG): lang.append('en') filenameResults['lang'] = lang autosub.WANTEDQUEUE.append(filenameResults) else: log.error("scanDir: Could not process the filename properly filename: %s" % filename) continue else: log.error("scanDir: Could not process the filename properly filename: %s" % filename) continue else: log.error("scanDir: Could not process the filename properly filename: %s" % filename) continue
def run(self): log.debug("checkRSS: Starting round of RSS checking") if autosub.WANTEDQUEUELOCK: log.debug("checkRSS: Exiting, another threat is using the queues") return False else: autosub.WANTEDQUEUELOCK = True toDelete_wantedQueue = [] langs = ["nl"] # default is to only check for Dutch subs # but if English should be downloaden, check them too # It is very important that the dutch language is run first! if autosub.FALLBACKTOENG or autosub.DOWNLOADENG: langs.append("en") log.debug("checkRSS: We also want to check the English RSS feed") for lang in langs: if lang == "en": RSSURL = autosub.ENRSSURL log.debug("checkRSS: Now using the English RSS feed") else: RSSURL = autosub.NLRSSURL log.debug("checkRSS: Now using the Dutch RSS feed") try: bierdopjeapi = API(RSSURL, RSS=True) soup = BeautifulStoneSoup(bierdopjeapi.resp.read()) log.debug("checkRss: Succussfully connected to %s" % RSSURL) bierdopjeapi.close() except: log.error( "checkRss: The server returned an error for request %s" % RSSURL) autosub.WANTEDQUEUELOCK = False continue # Parse all the item-tags from the RSSFeed # The information that is parsed is: title, link and show_id # The show_id is later used to match with the wanted items # The title is used the determine the quality / source / releasegrp rssItemList = [] items = soup.findAll('item') if not len(items) > 0: log.error("checkRss: invalid RssFeed") log.debug("checkRss: dumping Rssfeed %s" % (soup.prettify())) else: log.debug("checkRss: Valid RssFeed") for x in items: soupx = BeautifulStoneSoup(str(x)) title = soupx.find('title').string show_id = soupx.find('show_id').string link = soupx.find('enclosure')['url'] item = {} item['title'] = title item['link'] = link item['show_id'] = show_id log.debug("checkRSS: Parsed from RSSFEED: %s %s %s" % (title, link, show_id)) rssItemList.append(item) normalizedRssTitleList = [] # Now we create a new rsslist, containing information like: episode, season, etc for item in rssItemList: title = item['title'] link = item['link'] show_id = item['show_id'] log.debug( "checkRSS: Normalizing the following entry in the RSS results: %s" % title) normalizedRssTitle = ProcessFilename(title, '') normalizedRssTitle['rssfile'] = title normalizedRssTitle['link'] = link normalizedRssTitle['show_id'] = str(show_id) if 'title' in normalizedRssTitle.keys(): if 'season' in normalizedRssTitle.keys(): if 'episode' in normalizedRssTitle.keys(): normalizedRssTitleList.append(normalizedRssTitle) #check versus wantedItem list for index, wantedItem in enumerate(autosub.WANTEDQUEUE): wantedItemquality = None wantedItemreleasegrp = None wantedItemsource = None wantedItemcodec = None wantedItemtitle = wantedItem['title'] wantedItemseason = wantedItem['season'] wantedItemepisode = wantedItem['episode'] originalfile = wantedItem['originalFileLocationOnDisk'] if lang not in wantedItem['lang']: continue if 'quality' in wantedItem.keys(): wantedItemquality = wantedItem['quality'] if 'releasegrp' in wantedItem.keys(): wantedItemreleasegrp = wantedItem['releasegrp'] if 'source' in wantedItem.keys(): wantedItemsource = wantedItem['source'] if 'codec' in wantedItem.keys(): wantedItemcodec = wantedItem['codec'] #lets try to find a showid showid = autosub.Helpers.getShowid(wantedItemtitle) #no showid? skip this item if not showid: continue for normalizedRssTitle in normalizedRssTitleList: downloadLink = None normalizedRssTitleseason = normalizedRssTitle['season'] normalizedRssTitleepisode = normalizedRssTitle['episode'] normalizedRssTitlerssfile = normalizedRssTitle['rssfile'] normalizedRssTitleshowid = int( normalizedRssTitle['show_id']) normalizedRssTitlelink = normalizedRssTitle['link'] log.debug( "checkRSS: Trying to match ID:%r S:%r E:%r (wantedlist) with ID:%r S:%r E:%r (rss)" % (showid, wantedItemseason, wantedItemepisode, normalizedRssTitleshowid, normalizedRssTitleseason, normalizedRssTitleepisode)) if showid == normalizedRssTitleshowid and wantedItemseason == normalizedRssTitleseason and wantedItemepisode == normalizedRssTitleepisode: log.debug( "checkRSS: The episode %s - Season %s Episode %s was found in the RSS list, attempting to match a proper match" % (wantedItemtitle, wantedItemseason, wantedItemepisode)) score = autosub.Helpers.scoreMatch( normalizedRssTitle, normalizedRssTitlerssfile, wantedItemquality, wantedItemreleasegrp, wantedItemsource, wantedItemcodec) if score >= autosub.MINMATCHSCORERSS: log.debug( "checkRss: A match got a high enough score. MinMatchscore is %s " % autosub.MINMATCHSCORERSS) downloadLink = normalizedRssTitlelink + autosub.APIRSS log.info( "checkRss: Got a match, matching file is: %s" % normalizedRssTitlerssfile) log.debug( "checkRss: Dumping downloadlink for debug purpose: %s" % downloadLink) if downloadLink: originalfile = wantedItem[ 'originalFileLocationOnDisk'] # Dutch subs if autosub.SUBNL != "" and lang == "nl": srtfile = os.path.splitext(originalfile)[ 0] + u"." + autosub.SUBNL + u".srt" elif lang == "nl": srtfile = os.path.splitext( originalfile)[0] + u".srt" # English subs if autosub.SUBENG != "" and lang == "en": srtfile = os.path.splitext(originalfile)[ 0] + u"." + autosub.SUBENG + u".srt" elif lang == "en": srtfile = os.path.splitext( originalfile)[0] + u".srt" wantedItem['downloadLink'] = downloadLink wantedItem[ 'destinationFileLocationOnDisk'] = srtfile log.info( "checkRSS: The episode %s - Season %s Episode %s has a matching subtitle on the RSSFeed, downloading it!" % (wantedItemtitle, wantedItemseason, wantedItemepisode)) downloadItem = wantedItem.copy() downloadItem['downlang'] = lang downloadItem[ 'subtitle'] = normalizedRssTitlerssfile if not DownloadSub(downloadItem): break if lang == 'nl' and ( autosub.FALLBACKTOENG and not autosub.DOWNLOADENG ) and 'en' in wantedItem['lang']: log.debug( 'checkRss: We found a dutch subtitle and fallback is true. Removing the english subtitle from the wantedlist.' ) wantedItem['lang'].remove('en') wantedItem['lang'].remove(lang) if len(wantedItem['lang']) == 0: toDelete_wantedQueue.append(index) break else: log.debug( "checkRss: Matching score is not high enough. Score is %s should be %s" % (str(score), autosub.MINMATCHSCORERSS)) i = len(toDelete_wantedQueue) - 1 while i >= 0: log.debug( "checkRSS: Removed item from the wantedQueue at index %s" % toDelete_wantedQueue[i]) autosub.WANTEDQUEUE.pop(toDelete_wantedQueue[i]) i = i - 1 # Resetting the toDelete queue for the next run (if need) toDelete_wantedQueue = [] log.debug("checkRSS: Finished round of RSS checking") autosub.WANTEDQUEUELOCK = False return True
def walkDir(path): SkipListNL = autosub.SKIPSTRINGNL.split(",") if len( autosub.SKIPSTRINGNL) > 0 else [] SkipListEN = autosub.SKIPSTRINGEN.split(",") if len( autosub.SKIPSTRINGEN) > 0 else [] if len(autosub.SKIPFOLDERSNL) == 0: SkipFoldersNL = [] else: SkipFoldersNL = autosub.SKIPFOLDERSNL.split(",") if len( autosub.SKIPFOLDERSNL) > 0 else [] for idx, folder in enumerate(SkipFoldersNL): SkipFoldersNL[idx] = os.path.normpath( os.path.join(path, folder.rstrip("\/").lstrip("\/"))) if len(autosub.SKIPFOLDERSNL) == 0: SkipFoldersNL = [] else: SkipFoldersNL = autosub.SKIPFOLDERSNL.split(",") if len( autosub.SKIPFOLDERSNL) > 0 else [] for idx, folder in enumerate(SkipFoldersNL): SkipFoldersNL[idx] = os.path.normpath(path + folder) if len(autosub.SKIPFOLDERSEN) == 0: SkipFoldersEN = [] else: SkipFoldersEN = autosub.SKIPFOLDERSEN.split(",") if len( autosub.SKIPFOLDERSEN) > 0 else [] for idx, folder in enumerate(SkipFoldersEN): SkipFoldersEN[idx] = os.path.normpath(path + folder) for dirname, dirnames, filenames in os.walk(path, True, WalkError): SkipThisFolderNL = False for skip in SkipFoldersNL: if dirname.startswith(skip): SkipThisFolderNL = True break SkipThisFolderEN = False for skip in SkipFoldersEN: if dirname.startswith(skip): SkipThisFolderEN = True break log.debug("scanDisk: directory name: %s" % dirname) if re.search('_unpack_', dirname, re.IGNORECASE): log.debug("scanDisk: found a unpack directory, skipping.") continue if autosub.SKIPHIDDENDIRS and os.path.split(dirname)[1].startswith( u'.'): continue if re.search('_failed_', dirname, re.IGNORECASE): log.debug("scanDisk: found a failed directory, skipping.") continue if re.search('@eaDir', dirname, re.IGNORECASE): log.debug( "scanDisk: found a Synology indexing directory, skipping.") tmpdirs = dirnames[:] for dir in tmpdirs: dirnames.remove(dir) continue if re.search("@.*thumb", dirname, re.IGNORECASE): log.debug( "scanDisk: found a Qnap multimedia thumbnail folder, skipping." ) continue langs = [] FileDict = {} for filename in filenames: root, ext = os.path.splitext(filename) if ext[1:] in ('avi', 'mkv', 'wmv', 'ts', 'mp4'): if re.search('sample', filename): continue if not platform.system() == 'Windows': # Get best ascii compatible character for special characters try: if not isinstance(filename, unicode): coding = detect(filename)['encoding'] filename = unicode(filename.decode(coding), errors='replace') correctedFilename = ''.join( (c for c in unicodedata.normalize('NFD', filename) if unicodedata.category(c) != 'Mn')) if filename != correctedFilename: os.rename(os.path.join(dirname, filename), os.path.join(dirname, correctedFilename)) log.info("scanDir: Renamed file %s" % correctedFilename) filename = correctedFilename except: log.error("scanDir: Skipping directory, file %s, %s" % (dirname, filename)) continue # What subtitle files should we expect? langs = [] NLext = u'.' + autosub.SUBNL + u'.srt' if autosub.SUBNL else u'.srt' ENext = u'.' + autosub.SUBENG + u'.srt' if autosub.SUBENG else u'.srt' ENext = u'.en.srt' if NLext == ENext and autosub.DOWNLOADDUTCH else ENext if not os.access(dirname, os.W_OK): log.error('scandisk: No write access to folder: %s' % dirname) continue # Check which languages we want to download based on user settings. if autosub.DOWNLOADDUTCH and not SkipThisFolderNL: Skipped = False for SkipItem in SkipListNL: if not SkipItem: break if re.search(SkipItem.lower(), filename.lower()): Skipped = True break if Skipped: log.info( "scanDir: %s found in %s so skipped for Dutch subs" % (SkipItem, filename)) elif os.path.exists(os.path.join(dirname, root + NLext)): Skipped = True log.debug( "scanDir: %s skipped because the Dutch subtitle already exists" % filename) else: # If the Dutch subtitle not skipped and doesn't exist, then add it to the wanted list langs.append(autosub.DUTCH) if (autosub.DOWNLOADENG or (autosub.FALLBACKTOENG and autosub.DOWNLOADDUTCH and not Skipped)) and not SkipThisFolderEN: Skipped = False for SkipItem in SkipListEN: if not SkipItem: break if re.search(SkipItem.lower(), filename.lower()): Skipped = True break if Skipped: log.info( "scanDir: %s found in %s so skipped for English subs" % (SkipItem, filename)) elif os.path.exists(os.path.join(dirname, root + ENext)): log.debug( "scanDir: %s skipped because the English subtitle already exists" % filename) else: # If the English subtitle not skipped and doesn't exist, then add it to the wanted list if not os.path.exists( os.path.join(dirname, root + ENext)): langs.append(autosub.ENGLISH) if not langs: # nothing to do for this file continue FileDict = ProcessFilename( os.path.splitext(filename)[0].strip(), ext) time.sleep(0) if not FileDict: continue if not 'title' in FileDict.keys( ) or not 'season' in FileDict.keys( ) or not 'episode' in FileDict.keys(): continue if not FileDict['releasegrp'] and not FileDict[ 'source'] and not FileDict['quality'] and not FileDict[ 'source']: log.error("scanDir: Not enough info in filename: %s" % filename) continue FileDict['timestamp'] = unicode( time.strftime( '%Y-%m-%d %H:%M:%S', time.localtime( os.path.getctime(os.path.join(dirname, filename))))) FileDict['langs'] = langs FileDict['NLext'] = NLext FileDict['ENext'] = ENext FileDict['file'] = root FileDict['container'] = ext FileDict['folder'] = dirname FileDict['ImdbId'], FileDict['A7Id'], FileDict[ 'TvdbId'], FileDict['title'] = Helpers.getShowid( FileDict['title'], autosub.ADDIC7EDLOGGED_IN) if autosub.Helpers.SkipShow(FileDict['ImdbId'], FileDict['title'], FileDict['season'], FileDict['episode']): continue log.info("scanDir: %s WANTED FOR: %s" % (langs, filename)) autosub.WANTEDQUEUE.append(FileDict) return
def walkDir(path): for dirname, dirnames, filenames in os.walk(path): log.debug("scanDisk: directory name: %s" % dirname) if re.search("_unpack_", dirname, re.IGNORECASE): log.debug("scanDisk: found a unpack directory, skipping") continue if autosub.SKIPHIDDENDIRS and os.path.split(dirname)[1].startswith(u"."): continue if re.search("_failed_", dirname, re.IGNORECASE): log.debug("scanDisk: found a failed directory, skipping") continue if re.search("@eaDir", dirname, re.IGNORECASE): log.debug("scanDisk: found a Synology indexing directory, skipping this folder and all subfolders.") tmpdirs = dirnames[:] for dir in tmpdirs: dirnames.remove(dir) continue if re.search("@.*thumb", dirname, re.IGNORECASE): # Ingnore QNAP multimedia thumbnail folders continue for filename in filenames: splitname = filename.split(".") ext = splitname[len(splitname) - 1] if ext in ("avi", "mkv", "wmv", "ts", "mp4"): if re.search("sample", filename): continue if autosub.WEBDL == "None": if re.search("web-dl", filename.lower()): log.debug("scanDisk: WEB-DL is set to 'None', skipping %s" % filename) continue if not platform.system() == "Windows": # Get best ascii compatible character for special characters try: if not isinstance(filename, unicode): coding = detect(filename) filename = unicode(filename.decode(coding)) correctedFilename = "".join( (c for c in unicodedata.normalize("NFD", filename) if unicodedata.category(c) != "Mn") ) if filename != correctedFilename: os.rename(os.path.join(dirname, filename), os.path.join(dirname, correctedFilename)) log.info("scanDir: Renamed file %s" % correctedFilename) filename = correctedFilename except: log.error("scanDir: Skipping directory %s" % dirname) log.error("scanDir: Skipping file %s" % filename) continue # What subtitle files should we expect? lang = [] # Check what the Dutch subtitle would be. if autosub.SUBNL != "": srtfilenl = os.path.splitext(filename)[0] + u"." + autosub.SUBNL + u".srt" else: srtfilenl = os.path.splitext(filename)[0] + u".srt" # Check what the English subtitle would be. if autosub.SUBENG == "": # Check for overlapping names if autosub.SUBNL != "" or not autosub.DOWNLOADDUTCH: srtfileeng = os.path.splitext(filename)[0] + u".srt" # Hardcoded fallback else: srtfileeng = os.path.splitext(filename)[0] + u".en.srt" else: srtfileeng = os.path.splitext(filename)[0] + u"." + autosub.SUBENG + u".srt" # Check which languages we want to download based on user settings. if autosub.DOWNLOADDUTCH: # If the Dutch subtitle doesn't exist, then add it to the wanted list. if not os.path.exists(os.path.join(dirname, srtfilenl)): lang.append(autosub.DUTCH) if autosub.DOWNLOADENG: # If the English subtitle doesn't exist, then add it to the wanted list. if not os.path.exists(os.path.join(dirname, srtfileeng)): if autosub.WEBDL == "DutchOnly" and re.search("web-dl", filename.lower()): log.debug( "scanDisk: WEB-DL is set to 'Dutch Only', not adding English as wanted for %s" % filename ) else: lang.append(autosub.ENGLISH) if (autosub.FALLBACKTOENG and autosub.DOWNLOADDUTCH) and not autosub.DOWNLOADENG: # If the Dutch and English subtitles do not exist, then add English to the wanted list. if not os.path.exists(os.path.join(dirname, srtfilenl)) and not os.path.exists( os.path.join(dirname, srtfileeng) ): if autosub.WEBDL == "DutchOnly" and re.search("web-dl", filename.lower()): log.debug( "scanDisk: WEB-DL is set to 'Dutch Only', not adding English as wanted for %s" % filename ) else: lang.append(autosub.ENGLISH) if not lang: # autosub.WANTEDQUEUE empty continue log.debug("scanDir: File %s is missing subtitle(s): %s" % (filename, ", ".join(map(str, lang)))) filenameResults = ProcessFilename(os.path.splitext(filename)[0], os.path.splitext(filename)[1]) if "title" in filenameResults.keys(): if "season" in filenameResults.keys(): if "episode" in filenameResults.keys(): title = filenameResults["title"] season = filenameResults["season"] episode = filenameResults["episode"] if ( not filenameResults["releasegrp"] and not filenameResults["source"] and not filenameResults["quality"] and not filenameResults["source"] ): continue if autosub.Helpers.SkipShow(title, season, episode) == True: log.debug("scanDir: SkipShow returned True") log.info("scanDir: Skipping %s - Season %s Episode %s" % (title, season, episode)) continue if len(lang) == 1: log.info( "scanDir: %s subtitle wanted for %s and added to wantedQueue" % (lang[0], filename) ) else: log.info( "scanDir: %s subtitles wanted for %s and added to wantedQueue" % (" and ".join(map(str, lang)), filename) ) filenameResults["originalFileLocationOnDisk"] = os.path.join(dirname, filename) filenameResults["timestamp"] = unicode( time.strftime( "%Y-%m-%d %H:%M:%S", time.localtime(os.path.getctime(filenameResults["originalFileLocationOnDisk"])), ) ) filenameResults["lang"] = lang autosub.WANTEDQUEUE.append(filenameResults) else: log.error("scanDir: Could not process the filename properly filename: %s" % filename) continue else: log.error("scanDir: Could not process the filename properly filename: %s" % filename) continue else: log.error("scanDir: Could not process the filename properly filename: %s" % filename) continue