def update_network_dict(): """Update timezone information from SR repositories""" url = 'http://sickragetv.github.io/network_timezones/network_timezones.txt' url_data = getURL(url) if not url_data: sickrage.srLogger.warning( 'Updating network timezones failed, this can happen from time to time. URL: %s' % url) load_network_dict() return d = {} try: for line in url_data.splitlines(): (key, val) = line.strip().rsplit(':', 1) if key is None or val is None: continue d[key] = val except (IOError, OSError): pass network_timezones = dict( cache_db.CacheDB().select('SELECT * FROM network_timezones;')) queries = [] for network, timezone in d.iteritems(): existing = network in network_timezones if not existing: queries.append([ 'INSERT OR IGNORE INTO network_timezones VALUES (?,?);', [network, timezone] ]) elif network_timezones[network] is not timezone: queries.append([ 'UPDATE OR IGNORE network_timezones SET timezone = ? WHERE network_name = ?;', [timezone, network] ]) if existing: del network_timezones[network] if network_timezones: purged = [x for x in network_timezones] queries.append([ 'DELETE FROM network_timezones WHERE network_name IN (%s);' % ','.join(['?'] * len(purged)), purged ]) if len(queries) > 0: cache_db.CacheDB().mass_action(queries) del queries # cleanup
def saveNameCacheToDb(self): """Commit cache to database file""" for name, indexer_id in self.cache.items(): cache_db.CacheDB().action( "INSERT OR REPLACE INTO scene_names (indexer_id, name) VALUES (?, ?)", [indexer_id, name])
def setUp_test_db(force=False): """upgrades the db to the latest version """ global TESTDB_INITALIZED if not TESTDB_INITALIZED or force: # remove old db files tearDown_test_db() # upgrade main main_db.MainDB().InitialSchema().upgrade() # sanity check main main_db.MainDB().SanityCheck() # upgrade cache cache_db.CacheDB().InitialSchema().upgrade() # upgrade failed failed_db.FailedDB().InitialSchema().upgrade() # populate scene exceiptions table # retrieve_exceptions(False, False) TESTDB_INITALIZED = True
def load_network_dict(): """ Return network timezones from db """ try: cur_network_list = cache_db.CacheDB().select( 'SELECT * FROM network_timezones;') if not cur_network_list: update_network_dict() cur_network_list = cache_db.CacheDB().select( 'SELECT * FROM network_timezones;') d = dict(cur_network_list) except Exception: d = {} return d
def get_scene_exceptions(indexer_id, season=-1): """ Given a indexer_id, return a list of all the scene exceptions. """ exceptionsList = [] if indexer_id not in exceptionsCache or season not in exceptionsCache[ indexer_id]: exceptions = cache_db.CacheDB().select( "SELECT show_name FROM scene_exceptions WHERE indexer_id = ? AND season = ?", [indexer_id, season]) if exceptions: exceptionsList = list( set([ cur_exception[b"show_name"] for cur_exception in exceptions ])) if not indexer_id in exceptionsCache: exceptionsCache[indexer_id] = {} exceptionsCache[indexer_id][season] = exceptionsList else: exceptionsList = exceptionsCache[indexer_id][season] if season == 1: # if we where looking for season 1 we can add generic names exceptionsList += get_scene_exceptions(indexer_id, season=-1) return exceptionsList
def update_scene_exceptions(indexer_id, scene_exceptions, season=-1): """ Given a indexer_id, and a list of all show scene exceptions, update the db. """ cache_db.CacheDB().action( 'DELETE FROM scene_exceptions WHERE indexer_id=? AND season=?', [indexer_id, season]) sickrage.srLogger.info("Updating scene exceptions") # A change has been made to the scene exception list. Let's clear the cache, to make this visible if indexer_id in exceptionsCache: exceptionsCache[indexer_id] = {} exceptionsCache[indexer_id][season] = scene_exceptions for cur_exception in scene_exceptions: cache_db.CacheDB().action( "INSERT INTO scene_exceptions (indexer_id, show_name, season) VALUES (?,?,?)", [indexer_id, cur_exception, season])
def test_sceneExceptionsResetNameCache(self): # clear the exceptions cache_db.CacheDB().action("DELETE FROM scene_exceptions") # put something in the cache sickrage.srCore.NAMECACHE.addNameToCache('Cached Name', 0) # updating should not clear the cache this time since our exceptions didn't change self.assertEqual( sickrage.srCore.NAMECACHE.retrieveNameFromCache('Cached Name'), 0)
def setLastRefresh(exList): """ Update last cache update time for shows in list :param exList: exception list to set refresh time """ cache_db.CacheDB().upsert( "scene_exceptions_refresh", {'last_refreshed': int(time.mktime(datetime.today().timetuple()))}, {'list': exList})
def clearCache(self, indexerid=0): """ Deletes all "unknown" entries from the cache (names with indexer_id of 0). """ cache_db.CacheDB().action( "DELETE FROM scene_names WHERE indexer_id = ? OR indexer_id = ?", (indexerid, 0)) toRemove = [ key for key, value in self.cache.iteritems() if value == 0 or value == indexerid ] for key in toRemove: del self.cache[key]
def addNameToCache(self, name, indexer_id=0): """ Adds the show & tvdb id to the scene_names table in cache.db. :param name: The show name to cache :param indexer_id: the TVDB id that this show should be cached with (can be None/0 for unknown) """ # standardize the name we're using to account for small differences in providers name = full_sanitizeSceneName(name) if name not in self.cache: self.cache[name] = int(indexer_id) cache_db.CacheDB().action( "INSERT OR REPLACE INTO scene_names (indexer_id, name) VALUES (?, ?)", [indexer_id, name])
def get_scene_exception_by_name_multiple(show_name): """ Given a show name, return the indexerid of the exception, None if no exception is present. """ # try the obvious case first exception_result = cache_db.CacheDB().select( "SELECT indexer_id, season FROM scene_exceptions WHERE LOWER(show_name) = ? ORDER BY season ASC", [show_name.lower()]) if exception_result: return [(int(x[b"indexer_id"]), int(x[b"season"])) for x in exception_result] out = [] all_exception_results = cache_db.CacheDB().select( "SELECT show_name, indexer_id, season FROM scene_exceptions") for cur_exception in all_exception_results: cur_exception_name = cur_exception[b"show_name"] cur_indexer_id = int(cur_exception[b"indexer_id"]) cur_season = int(cur_exception[b"season"]) if show_name.lower() in ( cur_exception_name.lower(), sanitizeSceneName(cur_exception_name).lower().replace( '.', ' ')): sickrage.srLogger.debug("Scene exception lookup got indexer id " + str(cur_indexer_id) + ", using that") out.append((cur_indexer_id, cur_season)) if out: return out return [(None, None)]
def shouldRefresh(exList): """ Check if we should refresh cache for items in exList :param exList: exception list to check if needs a refresh :return: True if refresh is needed """ MAX_REFRESH_AGE_SECS = 86400 # 1 day rows = cache_db.CacheDB().select( "SELECT last_refreshed FROM scene_exceptions_refresh WHERE list = ?", [exList]) if rows: lastRefresh = int(rows[0][b'last_refreshed']) return int(time.mktime( datetime.today().timetuple())) > lastRefresh + MAX_REFRESH_AGE_SECS else: return True
def get_all_scene_exceptions(indexer_id): """ Get all scene exceptions for a show ID :param indexer_id: ID to check :return: dict of exceptions """ exceptionsDict = {} exceptions = cache_db.CacheDB().select( "SELECT show_name,season FROM scene_exceptions WHERE indexer_id = ?", [indexer_id]) if exceptions: for cur_exception in exceptions: if not cur_exception[b"season"] in exceptionsDict: exceptionsDict[cur_exception[b"season"]] = [] exceptionsDict[cur_exception[b"season"]].append( cur_exception[b"show_name"]) return exceptionsDict
def get_scene_seasons(indexer_id): """ return a list of season numbers that have scene exceptions """ exceptionsSeasonList = [] if indexer_id not in exceptionsSeasonCache: sqlResults = cache_db.CacheDB().select( "SELECT DISTINCT(season) AS season FROM scene_exceptions WHERE indexer_id = ?", [indexer_id]) if sqlResults: exceptionsSeasonList = list( set([int(x[b"season"]) for x in sqlResults])) if not indexer_id in exceptionsSeasonCache: exceptionsSeasonCache[indexer_id] = {} exceptionsSeasonCache[indexer_id] = exceptionsSeasonList else: exceptionsSeasonList = exceptionsSeasonCache[indexer_id] return exceptionsSeasonList
def test_allPossibleShowNames(self): exceptionsCache[-1] = ['Exception Test'] cache_db.CacheDB().action( "INSERT INTO scene_exceptions (indexer_id, show_name, season) VALUES (?,?,?)", [-1, 'Exception Test', -1]) countryList['Full Country Name'] = 'FCN' self._test_allPossibleShowNames('Show Name', expected=['Show Name']) self._test_allPossibleShowNames( 'Show Name', -1, expected=['Show Name', 'Exception Test']) self._test_allPossibleShowNames( 'Show Name FCN', expected=['Show Name FCN', 'Show Name (Full Country Name)']) self._test_allPossibleShowNames( 'Show Name (FCN)', expected=['Show Name (FCN)', 'Show Name (Full Country Name)']) self._test_allPossibleShowNames( 'Show Name Full Country Name', expected=['Show Name Full Country Name', 'Show Name (FCN)']) self._test_allPossibleShowNames( 'Show Name (Full Country Name)', expected=['Show Name (Full Country Name)', 'Show Name (FCN)'])
def start(self): self.PID = os.getpid() # set socket timeout socket.setdefaulttimeout(sickrage.srConfig.SOCKET_TIMEOUT) # init version updater self.VERSIONUPDATER = srVersionUpdater() # init updater and get current version self.VERSION = self.VERSIONUPDATER.updater.version # init services self.SCHEDULER = srScheduler() self.WEBSERVER = srWebServer() self.INDEXER_API = srIndexerApi # init caches self.NAMECACHE = srNameCache() # init queues self.SHOWUPDATER = srShowUpdater() self.SHOWQUEUE = srShowQueue() self.SEARCHQUEUE = srSearchQueue() # init searchers self.DAILYSEARCHER = srDailySearcher() self.BACKLOGSEARCHER = srBacklogSearcher() self.PROPERSEARCHER = srProperSearcher() self.TRAKTSEARCHER = srTraktSearcher() self.SUBTITLESEARCHER = srSubtitleSearcher() # init postprocessor self.AUTOPOSTPROCESSOR = srPostProcessor() # migrate old database file names to new ones if not os.path.exists(main_db.MainDB().filename) and os.path.exists("sickbeard.db"): helpers.moveFile("sickbeard.db", main_db.MainDB().filename) # initialize the main SB database main_db.MainDB().InitialSchema().upgrade() # initialize the cache database cache_db.CacheDB().InitialSchema().upgrade() # initialize the failed downloads database failed_db.FailedDB().InitialSchema().upgrade() # fix up any db problems main_db.MainDB().SanityCheck() # load data for shows from database self.load_shows() if sickrage.srConfig.DEFAULT_PAGE not in ('home', 'schedule', 'history', 'news', 'IRC'): sickrage.srConfig.DEFAULT_PAGE = 'home' if not makeDir(sickrage.srConfig.CACHE_DIR): sickrage.srLogger.error("!!! Creating local cache dir failed") sickrage.srConfig.CACHE_DIR = get_temp_dir() # Check if we need to perform a restore of the cache folder try: restore_dir = os.path.join(sickrage.DATA_DIR, 'restore') if os.path.exists(restore_dir) and os.path.exists(os.path.join(restore_dir, 'cache')): def restore_cache(src_dir, dst_dir): def path_leaf(path): head, tail = os.path.split(path) return tail or os.path.basename(head) try: if os.path.isdir(dst_dir): bak_filename = '{}-{}'.format(path_leaf(dst_dir), datetime.now().strftime('%Y%m%d_%H%M%S')) shutil.move(dst_dir, os.path.join(os.path.dirname(dst_dir), bak_filename)) shutil.move(src_dir, dst_dir) sickrage.srLogger.info("Restore: restoring cache successful") except Exception as E: sickrage.srLogger.error("Restore: restoring cache failed: {}".format(E.message)) restore_cache(os.path.join(restore_dir, 'cache'), sickrage.srConfig.CACHE_DIR) except Exception as e: sickrage.srLogger.error("Restore: restoring cache failed: {}".format(e.message)) finally: if os.path.exists(os.path.join(sickrage.DATA_DIR, 'restore')): try: removetree(os.path.join(sickrage.DATA_DIR, 'restore')) except Exception as e: sickrage.srLogger.error("Restore: Unable to remove the restore directory: {}".format(e.message)) for cleanupDir in ['mako', 'sessions', 'indexers']: try: removetree(os.path.join(sickrage.srConfig.CACHE_DIR, cleanupDir)) except Exception as e: sickrage.srLogger.warning( "Restore: Unable to remove the cache/{} directory: {1}".format(cleanupDir, e)) if sickrage.srConfig.WEB_PORT < 21 or sickrage.srConfig.WEB_PORT > 65535: sickrage.srConfig.WEB_PORT = 8081 if not sickrage.srConfig.WEB_COOKIE_SECRET: sickrage.srConfig.WEB_COOKIE_SECRET = generateCookieSecret() # attempt to help prevent users from breaking links by using a bad url if not sickrage.srConfig.ANON_REDIRECT.endswith('?'): sickrage.srConfig.ANON_REDIRECT = '' if not re.match(r'\d+\|[^|]+(?:\|[^|]+)*', sickrage.srConfig.ROOT_DIRS): sickrage.srConfig.ROOT_DIRS = '' sickrage.srConfig.NAMING_FORCE_FOLDERS = check_force_season_folders() if sickrage.srConfig.NZB_METHOD not in ('blackhole', 'sabnzbd', 'nzbget'): sickrage.srConfig.NZB_METHOD = 'blackhole' if not sickrage.srConfig.PROVIDER_ORDER: sickrage.srConfig.PROVIDER_ORDER = self.providersDict[GenericProvider.NZB].keys() + \ self.providersDict[GenericProvider.TORRENT].keys() if sickrage.srConfig.TORRENT_METHOD not in ( 'blackhole', 'utorrent', 'transmission', 'deluge', 'deluged', 'download_station', 'rtorrent', 'qbittorrent', 'mlnet'): sickrage.srConfig.TORRENT_METHOD = 'blackhole' if sickrage.srConfig.PROPER_SEARCHER_INTERVAL not in ('15m', '45m', '90m', '4h', 'daily'): sickrage.srConfig.PROPER_SEARCHER_INTERVAL = 'daily' if sickrage.srConfig.AUTOPOSTPROCESSOR_FREQ < sickrage.srConfig.MIN_AUTOPOSTPROCESSOR_FREQ: sickrage.srConfig.AUTOPOSTPROCESSOR_FREQ = sickrage.srConfig.MIN_AUTOPOSTPROCESSOR_FREQ if sickrage.srConfig.NAMECACHE_FREQ < sickrage.srConfig.MIN_NAMECACHE_FREQ: sickrage.srConfig.NAMECACHE_FREQ = sickrage.srConfig.MIN_NAMECACHE_FREQ if sickrage.srConfig.DAILY_SEARCHER_FREQ < sickrage.srConfig.MIN_DAILY_SEARCHER_FREQ: sickrage.srConfig.DAILY_SEARCHER_FREQ = sickrage.srConfig.MIN_DAILY_SEARCHER_FREQ sickrage.srConfig.MIN_BACKLOG_SEARCHER_FREQ = get_backlog_cycle_time() if sickrage.srConfig.BACKLOG_SEARCHER_FREQ < sickrage.srConfig.MIN_BACKLOG_SEARCHER_FREQ: sickrage.srConfig.BACKLOG_SEARCHER_FREQ = sickrage.srConfig.MIN_BACKLOG_SEARCHER_FREQ if sickrage.srConfig.VERSION_UPDATER_FREQ < sickrage.srConfig.MIN_VERSION_UPDATER_FREQ: sickrage.srConfig.VERSION_UPDATER_FREQ = sickrage.srConfig.MIN_VERSION_UPDATER_FREQ if sickrage.srConfig.SHOWUPDATE_HOUR > 23: sickrage.srConfig.SHOWUPDATE_HOUR = 0 elif sickrage.srConfig.SHOWUPDATE_HOUR < 0: sickrage.srConfig.SHOWUPDATE_HOUR = 0 if sickrage.srConfig.SUBTITLE_SEARCHER_FREQ < sickrage.srConfig.MIN_SUBTITLE_SEARCHER_FREQ: sickrage.srConfig.SUBTITLE_SEARCHER_FREQ = sickrage.srConfig.MIN_SUBTITLE_SEARCHER_FREQ sickrage.srConfig.NEWS_LATEST = sickrage.srConfig.NEWS_LAST_READ if sickrage.srConfig.SUBTITLES_LANGUAGES[0] == '': sickrage.srConfig.SUBTITLES_LANGUAGES = [] sickrage.srConfig.TIME_PRESET = sickrage.srConfig.TIME_PRESET_W_SECONDS.replace(":%S", "") # initialize metadata_providers self.metadataProviderDict = get_metadata_generator_dict() for cur_metadata_tuple in [(sickrage.srConfig.METADATA_KODI, kodi), (sickrage.srConfig.METADATA_KODI_12PLUS, kodi_12plus), (sickrage.srConfig.METADATA_MEDIABROWSER, mediabrowser), (sickrage.srConfig.METADATA_PS3, ps3), (sickrage.srConfig.METADATA_WDTV, wdtv), (sickrage.srConfig.METADATA_TIVO, tivo), (sickrage.srConfig.METADATA_MEDE8ER, mede8er)]: (cur_metadata_config, cur_metadata_class) = cur_metadata_tuple tmp_provider = cur_metadata_class.metadata_class() tmp_provider.set_config(cur_metadata_config) self.metadataProviderDict[tmp_provider.name] = tmp_provider # add version checker job to scheduler self.SCHEDULER.add_job( self.VERSIONUPDATER.run, srIntervalTrigger( **{'hours': sickrage.srConfig.VERSION_UPDATER_FREQ, 'min': sickrage.srConfig.MIN_VERSION_UPDATER_FREQ}), name="VERSIONUPDATER", id="VERSIONUPDATER", replace_existing=True ) # add network timezones updater job to scheduler self.SCHEDULER.add_job( update_network_dict, srIntervalTrigger(**{'days': 1}), name="TZUPDATER", id="TZUPDATER", replace_existing=True ) # add namecache updater job to scheduler self.SCHEDULER.add_job( self.NAMECACHE.run, srIntervalTrigger(**{'minutes': sickrage.srConfig.NAMECACHE_FREQ, 'min': sickrage.srConfig.MIN_NAMECACHE_FREQ}), name="NAMECACHE", id="NAMECACHE", replace_existing=True ) # add show queue job to scheduler self.SCHEDULER.add_job( self.SHOWQUEUE.run, srIntervalTrigger(**{'seconds': 3}), name="SHOWQUEUE", id="SHOWQUEUE", replace_existing=True ) # add search queue job to scheduler self.SCHEDULER.add_job( self.SEARCHQUEUE.run, srIntervalTrigger(**{'seconds': 1}), name="SEARCHQUEUE", id="SEARCHQUEUE", replace_existing=True ) # add show updater job to scheduler self.SCHEDULER.add_job( self.SHOWUPDATER.run, srIntervalTrigger( **{'hours': 1, 'start_date': datetime.now().replace(hour=sickrage.srConfig.SHOWUPDATE_HOUR)}), name="SHOWUPDATER", id="SHOWUPDATER", replace_existing=True ) # add daily search job to scheduler self.SCHEDULER.add_job( self.DAILYSEARCHER.run, srIntervalTrigger( **{'minutes': sickrage.srConfig.DAILY_SEARCHER_FREQ, 'min': sickrage.srConfig.MIN_DAILY_SEARCHER_FREQ}), name="DAILYSEARCHER", id="DAILYSEARCHER", replace_existing=True ) # add backlog search job to scheduler self.SCHEDULER.add_job( self.BACKLOGSEARCHER.run, srIntervalTrigger( **{'minutes': sickrage.srConfig.BACKLOG_SEARCHER_FREQ, 'min': sickrage.srConfig.MIN_BACKLOG_SEARCHER_FREQ}), name="BACKLOG", id="BACKLOG", replace_existing=True ) # add auto-postprocessing job to scheduler job = self.SCHEDULER.add_job( self.AUTOPOSTPROCESSOR.run, srIntervalTrigger(**{'minutes': sickrage.srConfig.AUTOPOSTPROCESSOR_FREQ, 'min': sickrage.srConfig.MIN_AUTOPOSTPROCESSOR_FREQ}), name="POSTPROCESSOR", id="POSTPROCESSOR", replace_existing=True ) (job.pause, job.resume)[sickrage.srConfig.PROCESS_AUTOMATICALLY]() # add find proper job to scheduler job = self.SCHEDULER.add_job( self.PROPERSEARCHER.run, srIntervalTrigger(**{ 'minutes': {'15m': 15, '45m': 45, '90m': 90, '4h': 4 * 60, 'daily': 24 * 60}[ sickrage.srConfig.PROPER_SEARCHER_INTERVAL]}), name="PROPERSEARCHER", id="PROPERSEARCHER", replace_existing=True ) (job.pause, job.resume)[sickrage.srConfig.DOWNLOAD_PROPERS]() # add trakt.tv checker job to scheduler job = self.SCHEDULER.add_job( self.TRAKTSEARCHER.run, srIntervalTrigger(**{'hours': 1}), name="TRAKTSEARCHER", id="TRAKTSEARCHER", replace_existing=True, ) (job.pause, job.resume)[sickrage.srConfig.USE_TRAKT]() # add subtitles finder job to scheduler job = self.SCHEDULER.add_job( self.SUBTITLESEARCHER.run, srIntervalTrigger(**{'hours': sickrage.srConfig.SUBTITLE_SEARCHER_FREQ}), name="SUBTITLESEARCHER", id="SUBTITLESEARCHER", replace_existing=True ) (job.pause, job.resume)[sickrage.srConfig.USE_SUBTITLES]() # add scheduler callback self.SCHEDULER.start()
def retrieve_exceptions(get_xem=True, get_anidb=True): """ Looks up the exceptions on github, parses them into a dict, and inserts them into the scene_exceptions table in cache.db. Also clears the scene name cache. """ for indexer in sickrage.srCore.INDEXER_API().indexers: indexer_name = sickrage.srCore.INDEXER_API(indexer).name if shouldRefresh(indexer_name): sickrage.srLogger.info( "Checking for SiCKRAGE scene exception updates for {}".format( indexer_name)) loc = sickrage.srCore.INDEXER_API(indexer).config[b'scene_loc'] try: # each exception is on one line with the format indexer_id: 'show name 1', 'show name 2', etc cur_line = None for cur_line in getURL(loc).splitlines(): indexer_id, _, aliases = cur_line.partition( ':') # @UnusedVariable if not aliases: continue # regex out the list of shows, taking \' into account exception_dict[int(indexer_id)] = [{ re.sub(r'\\(.)', r'\1', x): -1 } for x in re.findall(r"'(.*?)(?<!\\)',?", aliases)] if cur_line is None: sickrage.srLogger.debug( "Check scene exceptions update failed. Unable to update from: {}" .format(loc)) continue # refreshed successfully setLastRefresh(indexer_name) except Exception: continue # XEM scene exceptions if get_xem: _xem_exceptions_fetcher() # AniDB scene exceptions if get_anidb: _anidb_exceptions_fetcher() sql_l = [] for cur_indexer_id in exception_dict: sql_ex = cache_db.CacheDB().select( "SELECT * FROM scene_exceptions WHERE indexer_id = ?;", [cur_indexer_id]) existing_exceptions = [x[b"show_name"] for x in sql_ex] if not cur_indexer_id in exception_dict: continue for cur_exception_dict in exception_dict[cur_indexer_id]: for ex in cur_exception_dict.iteritems(): cur_exception, curSeason = ex if cur_exception not in existing_exceptions: sql_l.append([ "INSERT OR IGNORE INTO scene_exceptions (indexer_id, show_name, season) VALUES (?,?,?);", [cur_indexer_id, cur_exception, curSeason] ]) if len(sql_l) > 0: cache_db.CacheDB().mass_action(sql_l) sickrage.srLogger.debug("Updated scene exceptions") del sql_l # cleanup else: sickrage.srLogger.debug("No scene exceptions update needed") # cleanup exception_dict.clear() anidb_exception_dict.clear() xem_exception_dict.clear()
def getSceneSeasons(indexer_id): """get a list of season numbers that have scene exceptions""" seasons = cache_db.CacheDB().select( "SELECT DISTINCT season FROM scene_exceptions WHERE indexer_id = ?", [indexer_id]) return [cur_exception[b"season"] for cur_exception in seasons]