def update_network_dict():
"""Update timezone information from SR repositories"""
url = 'http://sickragetv.github.io/sb_network_timezones/network_timezones.txt'
url_data = getURL(url, session=requests.Session())
if not url_data:
sickrage.LOGGER.warning(
'Updating network timezones failed, this can happen from time to time. URL: %s'
% url)
load_network_dict()
return
d = {}
try:
for line in url_data.splitlines():
(key, val) = line.strip().rsplit(':', 1)
if key is None or val is None:
continue
d[key] = val
except (IOError, OSError):
pass
network_list = dict(
cache_db.CacheDB().select('SELECT * FROM network_timezones;'))
queries = []
for network, timezone in d.iteritems():
existing = network_list.has_key(network)
if not existing:
queries.append([
'INSERT OR IGNORE INTO network_timezones VALUES (?,?);',
[network, timezone]
])
elif network_list[network] is not timezone:
queries.append([
'UPDATE OR IGNORE network_timezones SET timezone = ? WHERE network_name = ?;',
[timezone, network]
])
if existing:
del network_list[network]
if network_list:
purged = [x for x in network_list]
queries.append([
'DELETE FROM network_timezones WHERE network_name IN (%s);' %
','.join(['?'] * len(purged)), purged
])
if queries:
cache_db.CacheDB().mass_action(queries)
load_network_dict()
def get_scene_exceptions(indexer_id, season=-1):
"""
Given a indexer_id, return a list of all the scene exceptions.
"""
exceptionsList = []
if indexer_id not in exceptionsCache or season not in exceptionsCache[
indexer_id]:
exceptions = cache_db.CacheDB().select(
"SELECT show_name FROM scene_exceptions WHERE indexer_id = ? AND season = ?",
[indexer_id, season])
if exceptions:
exceptionsList = list(
set([
cur_exception[b"show_name"] for cur_exception in exceptions
]))
if not indexer_id in exceptionsCache:
exceptionsCache[indexer_id] = {}
exceptionsCache[indexer_id][season] = exceptionsList
else:
exceptionsList = exceptionsCache[indexer_id][season]
if season == 1: # if we where looking for season 1 we can add generic names
exceptionsList += get_scene_exceptions(indexer_id, season=-1)
return exceptionsList
def load_network_dict():
"""
Load network timezones from db into dict network_dict (global dict)
"""
try:
cur_network_list = cache_db.CacheDB().select(
'SELECT * FROM network_timezones;')
if not cur_network_list:
update_network_dict()
cur_network_list = cache_db.CacheDB().select(
'SELECT * FROM network_timezones;')
d = dict(cur_network_list)
except Exception:
d = {}
return d
def saveNameCacheToDb(self):
"""Commit cache to database file"""
for name, indexer_id in self.cache.items():
cache_db.CacheDB().action(
"INSERT OR REPLACE INTO scene_names (indexer_id, name) VALUES (?, ?)",
[indexer_id, name])
def setUp_test_db(force=False):
"""upgrades the db to the latest version
"""
global TESTDB_INITALIZED
if not TESTDB_INITALIZED or force:
# remove old db files
tearDown_test_db()
# upgrade main
main_db.MainDB().InitialSchema().upgrade()
# sanity check main
main_db.MainDB().SanityCheck()
# upgrade cache
cache_db.CacheDB().InitialSchema().upgrade()
# upgrade failed
failed_db.FailedDB().InitialSchema().upgrade()
# populate scene exceiptions table
# retrieve_exceptions(False, False)
TESTDB_INITALIZED = True
def clearCache(self, indexerid=0):
"""
Deletes all "unknown" entries from the cache (names with indexer_id of 0).
"""
cache_db.CacheDB().action("DELETE FROM scene_names WHERE indexer_id = ? OR indexer_id = ?", (indexerid, 0))
toRemove = [key for key, value in self.cache.items() if value == 0 or value == indexerid]
for key in toRemove:
del self.cache[key]
def setLastRefresh(exList):
"""
Update last cache update time for shows in list
:param exList: exception list to set refresh time
"""
cache_db.CacheDB().upsert("scene_exceptions_refresh", {
'last_refreshed':
int(time.mktime(datetime.datetime.today().timetuple()))
}, {'list': exList})
def test_sceneExceptionsResetNameCache(self):
# clear the exceptions
cache_db.CacheDB().action("DELETE FROM scene_exceptions")
# put something in the cache
sickrage.srCore.NAMECACHE.addNameToCache('Cached Name', 0)
# updating should not clear the cache this time since our exceptions didn't change
self.assertEqual(
sickrage.srCore.NAMECACHE.retrieveNameFromCache('Cached Name'), 0)
def update_scene_exceptions(indexer_id, scene_exceptions, season=-1):
"""
Given a indexer_id, and a list of all show scene exceptions, update the db.
"""
cache_db.CacheDB().action(
'DELETE FROM scene_exceptions WHERE indexer_id=? AND season=?',
[indexer_id, season])
sickrage.LOGGER.info("Updating scene exceptions")
# A change has been made to the scene exception list. Let's clear the cache, to make this visible
if indexer_id in exceptionsCache:
exceptionsCache[indexer_id] = {}
exceptionsCache[indexer_id][season] = scene_exceptions
for cur_exception in scene_exceptions:
cache_db.CacheDB().action(
"INSERT INTO scene_exceptions (indexer_id, show_name, season) VALUES (?,?,?)",
[indexer_id, cur_exception, season])
def run(self, force=False):
if self.amActive:
return
self.amActive = True
# set thread name
threading.currentThread().setName(self.name)
update_timestamp = time.mktime(datetime.datetime.now().timetuple())
sqlResult = cache_db.CacheDB().select('SELECT `time` FROM lastUpdate WHERE provider = ?', ['theTVDB'])
if sqlResult:
last_update = sqlResult[0]['time']
else:
last_update = time.mktime(datetime.datetime.min.timetuple())
cache_db.CacheDB().action('INSERT INTO lastUpdate (provider, `time`) VALUES (?, ?)',
['theTVDB', long(last_update)])
if sickrage.srCore.srConfig.USE_FAILED_DOWNLOADS:
FailedHistory.trimHistory()
# get indexer updated show ids
updated_shows = srIndexerApi(1).indexer(**srIndexerApi(1).api_params.copy()).updated(long(last_update))
# start update process
piList = []
for curShow in sickrage.srCore.SHOWLIST:
try:
curShow.nextEpisode()
if curShow.indexerid in set(d["id"] for d in updated_shows or {}):
piList.append(sickrage.srCore.SHOWQUEUE.updateShow(curShow, True))
else:
piList.append(sickrage.srCore.SHOWQUEUE.refreshShow(curShow, False))
except (CantUpdateShowException, CantRefreshShowException) as e:
continue
ProgressIndicators.setIndicator('dailyShowUpdates', QueueProgressIndicator("Daily Show Updates", piList))
cache_db.CacheDB().action('UPDATE lastUpdate SET `time` = ? WHERE provider=?',
[long(update_timestamp), 'theTVDB'])
self.amActive = False
def update_network_dict():
"""Update timezone information from SR repositories"""
url = 'http://sickragetv.github.io/network_timezones/network_timezones.txt'
try:
url_data = sickrage.srCore.srWebSession.get(url).text
except Exception:
sickrage.srCore.srLogger.warning(
'Updating network timezones failed, this can happen from time to time. URL: %s'
% url)
return
d = {}
try:
for line in url_data.splitlines():
(key, val) = line.strip().rsplit(':', 1)
if key is None or val is None:
continue
d[key] = val
except (IOError, OSError):
pass
network_timezones = load_network_dict()
queries = []
for network, timezone in d.items():
existing = network in network_timezones
if not existing:
queries.append([
'INSERT OR IGNORE INTO network_timezones VALUES (?,?);',
[network, timezone]
])
elif network_timezones[network] is not timezone:
queries.append([
'UPDATE OR IGNORE network_timezones SET timezone = ? WHERE network_name = ?;',
[timezone, network]
])
if existing:
del network_timezones[network]
if network_timezones:
purged = [x for x in network_timezones]
queries.append([
'DELETE FROM network_timezones WHERE network_name IN (%s);' %
','.join(['?'] * len(purged)), purged
])
if len(queries) > 0:
cache_db.CacheDB().mass_action(queries)
del queries # cleanup
def addNameToCache(self, name, indexer_id=0):
"""
Adds the show & tvdb id to the scene_names table in cache.db.
:param name: The show name to cache
:param indexer_id: the TVDB id that this show should be cached with (can be None/0 for unknown)
"""
# standardize the name we're using to account for small differences in providers
name = full_sanitizeSceneName(name)
if name not in self.cache:
self.cache[name] = int(indexer_id)
cache_db.CacheDB().action("INSERT OR REPLACE INTO scene_names (indexer_id, name) VALUES (?, ?)",
[indexer_id, name])
def load_network_dict():
"""
Return network timezones from db
"""
try:
cur_network_list = cache_db.CacheDB().select(
'SELECT * FROM network_timezones;')
if cur_network_list:
return dict(cur_network_list)
except Exception:
pass
return {}
def get_scene_exception_by_name_multiple(show_name):
"""
Given a show name, return the indexerid of the exception, None if no exception
is present.
"""
# try the obvious case first
exception_result = cache_db.CacheDB().select(
"SELECT indexer_id, season FROM scene_exceptions WHERE LOWER(show_name) = ? ORDER BY season ASC",
[show_name.lower()])
if exception_result:
return [(int(x["indexer_id"]), int(x["season"]))
for x in exception_result]
out = []
all_exception_results = cache_db.CacheDB().select(
"SELECT show_name, indexer_id, season FROM scene_exceptions")
for cur_exception in all_exception_results:
cur_exception_name = cur_exception["show_name"]
cur_indexer_id = int(cur_exception["indexer_id"])
cur_season = int(cur_exception["season"])
if show_name.lower() in (
cur_exception_name.lower(),
sanitizeSceneName(cur_exception_name).lower().replace(
'.', ' ')):
sickrage.srCore.srLogger.debug(
"Scene exception lookup got indexer id " +
str(cur_indexer_id) + ", using that")
out.append((cur_indexer_id, cur_season))
if out:
return out
return [(None, None)]
def shouldRefresh(exList):
"""
Check if we should refresh cache for items in exList
:param exList: exception list to check if needs a refresh
:return: True if refresh is needed
"""
MAX_REFRESH_AGE_SECS = 86400 # 1 day
rows = cache_db.CacheDB().select(
"SELECT last_refreshed FROM scene_exceptions_refresh WHERE list = ?",
[exList])
if rows:
lastRefresh = int(rows[0][b'last_refreshed'])
return int(time.mktime(datetime.datetime.today().timetuple())
) > lastRefresh + MAX_REFRESH_AGE_SECS
else:
return True
def get_all_scene_exceptions(indexer_id):
"""
Get all scene exceptions for a show ID
:param indexer_id: ID to check
:return: dict of exceptions
"""
exceptionsDict = {}
exceptions = cache_db.CacheDB().select(
"SELECT show_name,season FROM scene_exceptions WHERE indexer_id = ?",
[indexer_id])
if exceptions:
for cur_exception in exceptions:
if not cur_exception[b"season"] in exceptionsDict:
exceptionsDict[cur_exception[b"season"]] = []
exceptionsDict[cur_exception[b"season"]].append(
cur_exception[b"show_name"])
return exceptionsDict
def test_allPossibleShowNames(self):
exceptionsCache[-1] = ['Exception Test']
cache_db.CacheDB().action(
"INSERT INTO scene_exceptions (indexer_id, show_name, season) VALUES (?,?,?)",
[-1, 'Exception Test', -1])
countryList['Full Country Name'] = 'FCN'
self._test_allPossibleShowNames('Show Name', expected=['Show Name'])
self._test_allPossibleShowNames(
'Show Name', -1, expected=['Show Name', 'Exception Test'])
self._test_allPossibleShowNames(
'Show Name FCN',
expected=['Show Name FCN', 'Show Name (Full Country Name)'])
self._test_allPossibleShowNames(
'Show Name (FCN)',
expected=['Show Name (FCN)', 'Show Name (Full Country Name)'])
self._test_allPossibleShowNames(
'Show Name Full Country Name',
expected=['Show Name Full Country Name', 'Show Name (FCN)'])
self._test_allPossibleShowNames(
'Show Name (Full Country Name)',
expected=['Show Name (Full Country Name)', 'Show Name (FCN)'])
def get_scene_seasons(indexer_id):
"""
return a list of season numbers that have scene exceptions
"""
exceptionsSeasonList = []
if indexer_id not in exceptionsSeasonCache:
sqlResults = cache_db.CacheDB().select(
"SELECT DISTINCT(season) AS season FROM scene_exceptions WHERE indexer_id = ?",
[indexer_id])
if sqlResults:
exceptionsSeasonList = list(
set([int(x[b"season"]) for x in sqlResults]))
if not indexer_id in exceptionsSeasonCache:
exceptionsSeasonCache[indexer_id] = {}
exceptionsSeasonCache[indexer_id] = exceptionsSeasonList
else:
exceptionsSeasonList = exceptionsSeasonCache[indexer_id]
return exceptionsSeasonList
def setUp_test_db():
"""upgrades the db to the latest version
"""
global TESTDB_INITALIZED
if not TESTDB_INITALIZED:
# remove old db files
tearDown_test_db()
# upgrading the db
main_db.MainDB().InitialSchema().upgrade()
# fix up any db problems
main_db.MainDB().SanityCheck()
# and for cachedb too
cache_db.CacheDB().InitialSchema().upgrade()
# and for faileddb too
failed_db.FailedDB().InitialSchema().upgrade()
TESTDB_INITALIZED = True
def loadNameCacheFromDB(self):
sqlResults = cache_db.CacheDB(row_type='dict').select(
"SELECT indexer_id, name FROM scene_names")
return dict((row["name"], int(row["indexer_id"])) for row in sqlResults)
def retrieve_exceptions(get_xem=True, get_anidb=True):
"""
Looks up the exceptions on github, parses them into a dict, and inserts them into the
scene_exceptions table in cache.db. Also clears the scene name cache.
"""
for indexer in srIndexerApi().indexers:
indexer_name = srIndexerApi(indexer).name
if shouldRefresh(indexer_name):
sickrage.srCore.srLogger.info(
"Checking for SiCKRAGE scene exception updates on {}".format(
indexer_name))
loc = srIndexerApi(indexer).config['scene_loc']
try:
# each exception is on one line with the format indexer_id: 'show name 1', 'show name 2', etc
cur_line = None
for cur_line in sickrage.srCore.srWebSession.get(
loc).text.splitlines():
indexer_id, _, aliases = cur_line.partition(
':') # @UnusedVariable
if not aliases:
continue
# regex out the list of shows, taking \' into account
exception_dict[int(indexer_id)] = [{
re.sub(r'\\(.)', r'\1', x):
-1
} for x in re.findall(r"'(.*?)(?<!\\)',?", aliases)]
if cur_line is None:
sickrage.srCore.srLogger.debug(
"Check scene exceptions update failed. Unable to update from: {}"
.format(loc))
continue
# refreshed successfully
setLastRefresh(indexer_name)
except Exception:
continue
# XEM scene exceptions
if get_xem:
_xem_exceptions_fetcher()
# AniDB scene exceptions
if get_anidb:
_anidb_exceptions_fetcher()
sql_l = []
for cur_indexer_id in exception_dict:
sql_ex = cache_db.CacheDB().select(
"SELECT * FROM scene_exceptions WHERE indexer_id = ?;",
[cur_indexer_id])
existing_exceptions = [x["show_name"] for x in sql_ex]
if not cur_indexer_id in exception_dict:
continue
for cur_exception_dict in exception_dict[cur_indexer_id]:
for ex in cur_exception_dict.items():
cur_exception, curSeason = ex
if cur_exception not in existing_exceptions:
sql_l.append([
"INSERT OR IGNORE INTO scene_exceptions (indexer_id, show_name, season) VALUES (?,?,?);",
[cur_indexer_id, cur_exception, curSeason]
])
if len(sql_l) > 0:
cache_db.CacheDB().mass_action(sql_l)
sickrage.srCore.srLogger.debug("Updated scene exceptions")
del sql_l # cleanup
else:
sickrage.srCore.srLogger.debug("No scene exceptions update needed")
# cleanup
exception_dict.clear()
anidb_exception_dict.clear()
xem_exception_dict.clear()
def start(self):
self.started = True
# thread name
threading.currentThread().setName('CORE')
# Check if we need to perform a restore first
if os.path.exists(
os.path.abspath(os.path.join(sickrage.DATA_DIR, 'restore'))):
success = restoreSR(
os.path.abspath(os.path.join(sickrage.DATA_DIR, 'restore')),
sickrage.DATA_DIR)
print("Restoring SiCKRAGE backup: %s!\n" %
("FAILED", "SUCCESSFUL")[success])
if success:
shutil.rmtree(os.path.abspath(
os.path.join(sickrage.DATA_DIR, 'restore')),
ignore_errors=True)
# migrate old database file names to new ones
if os.path.isfile(
os.path.abspath(os.path.join(sickrage.DATA_DIR,
'sickbeard.db'))):
if os.path.isfile(os.path.join(sickrage.DATA_DIR, 'sickrage.db')):
helpers.moveFile(
os.path.join(sickrage.DATA_DIR, 'sickrage.db'),
os.path.join(
sickrage.DATA_DIR, '{}.bak-{}'.format(
'sickrage.db',
datetime.datetime.now().strftime(
'%Y%m%d_%H%M%S'))))
helpers.moveFile(
os.path.abspath(os.path.join(sickrage.DATA_DIR,
'sickbeard.db')),
os.path.abspath(os.path.join(sickrage.DATA_DIR,
'sickrage.db')))
# load config
self.srConfig.load()
# set socket timeout
socket.setdefaulttimeout(self.srConfig.SOCKET_TIMEOUT)
# setup logger settings
self.srLogger.logSize = self.srConfig.LOG_SIZE
self.srLogger.logNr = self.srConfig.LOG_NR
self.srLogger.debugLogging = sickrage.DEBUG
self.srLogger.consoleLogging = not sickrage.QUITE
self.srLogger.logFile = self.srConfig.LOG_FILE
# start logger
self.srLogger.start()
# initialize the main SB database
main_db.MainDB().InitialSchema().upgrade()
# initialize the cache database
cache_db.CacheDB().InitialSchema().upgrade()
# initialize the failed downloads database
failed_db.FailedDB().InitialSchema().upgrade()
# fix up any db problems
main_db.MainDB().SanityCheck()
# load data for shows from database
self.load_shows()
if self.srConfig.DEFAULT_PAGE not in ('home', 'schedule', 'history',
'news', 'IRC'):
self.srConfig.DEFAULT_PAGE = 'home'
# cleanup cache folder
for dir in ['mako', 'sessions', 'indexers']:
try:
shutil.rmtree(os.path.join(self.srConfig.CACHE_DIR, dir),
ignore_errors=True)
except Exception:
continue
# init anidb connection
if not self.srConfig.USE_ANIDB:
try:
self.ADBA_CONNECTION = adba.Connection(
keepAlive=True,
log=lambda msg: self.srLogger.debug(
"AniDB: %s " % msg)).auth(self.srConfig.ANIDB_USERNAME,
self.srConfig.ANIDB_PASSWORD)
except Exception as e:
self.srLogger.warning("AniDB exception msg: %r " % repr(e))
if self.srConfig.WEB_PORT < 21 or self.srConfig.WEB_PORT > 65535:
self.srConfig.WEB_PORT = 8081
if not self.srConfig.WEB_COOKIE_SECRET:
self.srConfig.WEB_COOKIE_SECRET = generateCookieSecret()
# attempt to help prevent users from breaking links by using a bad url
if not self.srConfig.ANON_REDIRECT.endswith('?'):
self.srConfig.ANON_REDIRECT = ''
if not re.match(r'\d+\|[^|]+(?:\|[^|]+)*', self.srConfig.ROOT_DIRS):
self.srConfig.ROOT_DIRS = ''
self.srConfig.NAMING_FORCE_FOLDERS = check_force_season_folders()
if self.srConfig.NZB_METHOD not in ('blackhole', 'sabnzbd', 'nzbget'):
self.srConfig.NZB_METHOD = 'blackhole'
if self.srConfig.TORRENT_METHOD not in ('blackhole', 'utorrent',
'transmission', 'deluge',
'deluged', 'download_station',
'rtorrent', 'qbittorrent',
'mlnet', 'putio'):
self.srConfig.TORRENT_METHOD = 'blackhole'
if self.srConfig.PROPER_SEARCHER_INTERVAL not in ('15m', '45m', '90m',
'4h', 'daily'):
self.srConfig.PROPER_SEARCHER_INTERVAL = 'daily'
if self.srConfig.AUTOPOSTPROCESSOR_FREQ < self.srConfig.MIN_AUTOPOSTPROCESSOR_FREQ:
self.srConfig.AUTOPOSTPROCESSOR_FREQ = self.srConfig.MIN_AUTOPOSTPROCESSOR_FREQ
if self.srConfig.NAMECACHE_FREQ < self.srConfig.MIN_NAMECACHE_FREQ:
self.srConfig.NAMECACHE_FREQ = self.srConfig.MIN_NAMECACHE_FREQ
if self.srConfig.DAILY_SEARCHER_FREQ < self.srConfig.MIN_DAILY_SEARCHER_FREQ:
self.srConfig.DAILY_SEARCHER_FREQ = self.srConfig.MIN_DAILY_SEARCHER_FREQ
self.srConfig.MIN_BACKLOG_SEARCHER_FREQ = get_backlog_cycle_time()
if self.srConfig.BACKLOG_SEARCHER_FREQ < self.srConfig.MIN_BACKLOG_SEARCHER_FREQ:
self.srConfig.BACKLOG_SEARCHER_FREQ = self.srConfig.MIN_BACKLOG_SEARCHER_FREQ
if self.srConfig.VERSION_UPDATER_FREQ < self.srConfig.MIN_VERSION_UPDATER_FREQ:
self.srConfig.VERSION_UPDATER_FREQ = self.srConfig.MIN_VERSION_UPDATER_FREQ
if self.srConfig.SHOWUPDATE_HOUR > 23:
self.srConfig.SHOWUPDATE_HOUR = 0
elif self.srConfig.SHOWUPDATE_HOUR < 0:
self.srConfig.SHOWUPDATE_HOUR = 0
if self.srConfig.SUBTITLE_SEARCHER_FREQ < self.srConfig.MIN_SUBTITLE_SEARCHER_FREQ:
self.srConfig.SUBTITLE_SEARCHER_FREQ = self.srConfig.MIN_SUBTITLE_SEARCHER_FREQ
self.srConfig.NEWS_LATEST = self.srConfig.NEWS_LAST_READ
if self.srConfig.SUBTITLES_LANGUAGES[0] == '':
self.srConfig.SUBTITLES_LANGUAGES = []
# initialize metadata_providers
for cur_metadata_tuple in [
(self.srConfig.METADATA_KODI, kodi),
(self.srConfig.METADATA_KODI_12PLUS, kodi_12plus),
(self.srConfig.METADATA_MEDIABROWSER, mediabrowser),
(self.srConfig.METADATA_PS3, ps3),
(self.srConfig.METADATA_WDTV, wdtv),
(self.srConfig.METADATA_TIVO, tivo),
(self.srConfig.METADATA_MEDE8ER, mede8er)
]:
(cur_metadata_config, cur_metadata_class) = cur_metadata_tuple
tmp_provider = cur_metadata_class.metadata_class()
tmp_provider.set_config(cur_metadata_config)
self.metadataProviderDict[tmp_provider.name] = tmp_provider
# add show queue job
self.srScheduler.add_job(self.SHOWQUEUE.run,
srIntervalTrigger(**{'seconds': 1}),
name="SHOWQUEUE",
id="SHOWQUEUE")
# add search queue job
self.srScheduler.add_job(self.SEARCHQUEUE.run,
srIntervalTrigger(**{'seconds': 1}),
name="SEARCHQUEUE",
id="SEARCHQUEUE")
# add version checker job
self.srScheduler.add_job(
self.VERSIONUPDATER.run,
srIntervalTrigger(
**{
'hours': self.srConfig.VERSION_UPDATER_FREQ,
'min': self.srConfig.MIN_VERSION_UPDATER_FREQ
}),
name="VERSIONUPDATER",
id="VERSIONUPDATER")
# add network timezones updater job
self.srScheduler.add_job(update_network_dict,
srIntervalTrigger(**{'days': 1}),
name="TZUPDATER",
id="TZUPDATER")
# add namecache updater job
self.srScheduler.add_job(
self.NAMECACHE.run,
srIntervalTrigger(
**{
'minutes': self.srConfig.NAMECACHE_FREQ,
'min': self.srConfig.MIN_NAMECACHE_FREQ
}),
name="NAMECACHE",
id="NAMECACHE")
# add show updater job
self.srScheduler.add_job(
self.SHOWUPDATER.run,
srIntervalTrigger(
**{
'hours':
1,
'start_date':
datetime.datetime.now().replace(
hour=self.srConfig.SHOWUPDATE_HOUR)
}),
name="SHOWUPDATER",
id="SHOWUPDATER")
# add daily search job
self.srScheduler.add_job(
self.DAILYSEARCHER.run,
srIntervalTrigger(
**{
'minutes': self.srConfig.DAILY_SEARCHER_FREQ,
'min': self.srConfig.MIN_DAILY_SEARCHER_FREQ
}),
name="DAILYSEARCHER",
id="DAILYSEARCHER")
# add backlog search job
self.srScheduler.add_job(
self.BACKLOGSEARCHER.run,
srIntervalTrigger(
**{
'minutes': self.srConfig.BACKLOG_SEARCHER_FREQ,
'min': self.srConfig.MIN_BACKLOG_SEARCHER_FREQ
}),
name="BACKLOG",
id="BACKLOG")
# add auto-postprocessing job
self.srScheduler.add_job(
self.AUTOPOSTPROCESSOR.run,
srIntervalTrigger(
**{
'minutes': self.srConfig.AUTOPOSTPROCESSOR_FREQ,
'min': self.srConfig.MIN_AUTOPOSTPROCESSOR_FREQ
}),
name="POSTPROCESSOR",
id="POSTPROCESSOR")
# add find proper job
self.srScheduler.add_job(
self.PROPERSEARCHER.run,
srIntervalTrigger(
**{
'minutes': {
'15m': 15,
'45m': 45,
'90m': 90,
'4h': 4 * 60,
'daily': 24 * 60
}[self.srConfig.PROPER_SEARCHER_INTERVAL]
}),
name="PROPERSEARCHER",
id="PROPERSEARCHER")
# add trakt.tv checker job
self.srScheduler.add_job(self.TRAKTSEARCHER.run,
srIntervalTrigger(**{'hours': 1}),
name="TRAKTSEARCHER",
id="TRAKTSEARCHER")
# add subtitles finder job
self.srScheduler.add_job(
self.SUBTITLESEARCHER.run,
srIntervalTrigger(
**{'hours': self.srConfig.SUBTITLE_SEARCHER_FREQ}),
name="SUBTITLESEARCHER",
id="SUBTITLESEARCHER")
# start scheduler service
self.srScheduler.start()
# Pause/Resume PROPERSEARCHER job
(self.srScheduler.get_job('PROPERSEARCHER').pause,
self.srScheduler.get_job('PROPERSEARCHER').resume
)[self.srConfig.DOWNLOAD_PROPERS]()
# Pause/Resume TRAKTSEARCHER job
(self.srScheduler.get_job('TRAKTSEARCHER').pause,
self.srScheduler.get_job('TRAKTSEARCHER').resume
)[self.srConfig.USE_TRAKT]()
# Pause/Resume SUBTITLESEARCHER job
(self.srScheduler.get_job('SUBTITLESEARCHER').pause,
self.srScheduler.get_job('SUBTITLESEARCHER').resume
)[self.srConfig.USE_SUBTITLES]()
# Pause/Resume POSTPROCESS job
(self.srScheduler.get_job('POSTPROCESSOR').pause,
self.srScheduler.get_job('POSTPROCESSOR').resume
)[self.srConfig.PROCESS_AUTOMATICALLY]()
# start webserver
self.srWebServer.start()
# start ioloop event handler
IOLoop.instance().start()
def initialize():
if not sickrage.INITIALIZED:
with threading.Lock():
# init encoding
encodingInit()
# Check if we need to perform a restore first
os.chdir(sickrage.DATA_DIR)
restore_dir = os.path.join(sickrage.DATA_DIR, 'restore')
if os.path.exists(restore_dir):
success = restoreDB(restore_dir, sickrage.DATA_DIR)
sickrage.LOGGER.info(
"Restore: restoring DB and config.ini %s!\n" %
("FAILED", "SUCCESSFUL")[success])
# init indexerApi
sickrage.INDEXER_API = indexerApi
# initialize notifiers
sickrage.NOTIFIERS = AttrDict(
libnotify=LibnotifyNotifier(),
kodi_notifier=KODINotifier(),
plex_notifier=PLEXNotifier(),
emby_notifier=EMBYNotifier(),
nmj_notifier=NMJNotifier(),
nmjv2_notifier=NMJv2Notifier(),
synoindex_notifier=synoIndexNotifier(),
synology_notifier=synologyNotifier(),
pytivo_notifier=pyTivoNotifier(),
growl_notifier=GrowlNotifier(),
prowl_notifier=ProwlNotifier(),
libnotify_notifier=LibnotifyNotifier(),
pushover_notifier=PushoverNotifier(),
boxcar_notifier=BoxcarNotifier(),
boxcar2_notifier=Boxcar2Notifier(),
nma_notifier=NMA_Notifier(),
pushalot_notifier=PushalotNotifier(),
pushbullet_notifier=PushbulletNotifier(),
freemobile_notifier=FreeMobileNotifier(),
twitter_notifier=TwitterNotifier(),
trakt_notifier=TraktNotifier(),
email_notifier=EmailNotifier(),
)
sickrage.NAMING_EP_TYPE = (
"%(seasonnumber)dx%(episodenumber)02d",
"s%(seasonnumber)02de%(episodenumber)02d",
"S%(seasonnumber)02dE%(episodenumber)02d",
"%(seasonnumber)02dx%(episodenumber)02d")
sickrage.SPORTS_EP_TYPE = (
"%(seasonnumber)dx%(episodenumber)02d",
"s%(seasonnumber)02de%(episodenumber)02d",
"S%(seasonnumber)02dE%(episodenumber)02d",
"%(seasonnumber)02dx%(episodenumber)02d")
sickrage.NAMING_EP_TYPE_TEXT = ("1x02", "s01e02", "S01E02",
"01x02")
sickrage.NAMING_MULTI_EP_TYPE = {
0: ["-%(episodenumber)02d"] * len(sickrage.NAMING_EP_TYPE),
1: [" - " + x for x in sickrage.NAMING_EP_TYPE],
2: [x + "%(episodenumber)02d" for x in ("x", "e", "E", "x")]
}
sickrage.NAMING_MULTI_EP_TYPE_TEXT = ("extend", "duplicate",
"repeat")
sickrage.NAMING_SEP_TYPE = (" - ", " ")
sickrage.NAMING_SEP_TYPE_TEXT = (" - ", "space")
# migrate old database filenames to new ones
if not os.path.exists(main_db.MainDB().filename
) and os.path.exists("sickbeard.db"):
helpers.moveFile("sickbeard.db", main_db.MainDB().filename)
# init config file
srConfig.load_config(sickrage.CONFIG_FILE, True)
# set socket timeout
socket.setdefaulttimeout(sickrage.SOCKET_TIMEOUT)
# init logger
sickrage.LOGGER = sickrage.LOGGER.__class__(
logFile=sickrage.LOG_FILE,
logSize=sickrage.LOG_SIZE,
logNr=sickrage.LOG_NR,
fileLogging=makeDir(sickrage.LOG_DIR),
debugLogging=sickrage.DEBUG)
# init updater and get current version
sickrage.VERSIONUPDATER = VersionUpdater()
sickrage.VERSION = sickrage.VERSIONUPDATER.updater.get_cur_version
# initialize the main SB database
main_db.MainDB().InitialSchema().upgrade()
# initialize the cache database
cache_db.CacheDB().InitialSchema().upgrade()
# initialize the failed downloads database
failed_db.FailedDB().InitialSchema().upgrade()
# fix up any db problems
main_db.MainDB().SanityCheck()
if sickrage.DEFAULT_PAGE not in ('home', 'schedule', 'history',
'news', 'IRC'):
sickrage.DEFAULT_PAGE = 'home'
if not makeDir(sickrage.CACHE_DIR):
sickrage.LOGGER.error("!!! Creating local cache dir failed")
sickrage.CACHE_DIR = None
# Check if we need to perform a restore of the cache folder
try:
restore_dir = os.path.join(sickrage.DATA_DIR, 'restore')
if os.path.exists(restore_dir) and os.path.exists(
os.path.join(restore_dir, 'cache')):
def restore_cache(srcdir, dstdir):
def path_leaf(path):
head, tail = os.path.split(path)
return tail or os.path.basename(head)
try:
if os.path.isdir(dstdir):
bakfilename = '{}-{1}'.format(
path_leaf(dstdir),
datetime.datetime.strftime(
datetime.date.now(), '%Y%m%d_%H%M%S'))
shutil.move(
dstdir,
os.path.join(os.path.dirname(dstdir),
bakfilename))
shutil.move(srcdir, dstdir)
sickrage.LOGGER.info(
"Restore: restoring cache successful")
except Exception as E:
sickrage.LOGGER.error(
"Restore: restoring cache failed: {}".format(
E))
restore_cache(os.path.join(restore_dir, 'cache'),
sickrage.CACHE_DIR)
except Exception as e:
sickrage.LOGGER.error(
"Restore: restoring cache failed: {}".format(e))
finally:
if os.path.exists(os.path.join(sickrage.DATA_DIR, 'restore')):
try:
removetree(os.path.join(sickrage.DATA_DIR, 'restore'))
except Exception as e:
sickrage.LOGGER.error(
"Restore: Unable to remove the restore directory: {}"
.format(e))
for cleanupDir in ['mako', 'sessions', 'indexers']:
try:
removetree(
os.path.join(sickrage.CACHE_DIR, cleanupDir))
except Exception as e:
sickrage.LOGGER.warning(
"Restore: Unable to remove the cache/{} directory: {1}"
.format(cleanupDir, e))
if sickrage.WEB_PORT < 21 or sickrage.WEB_PORT > 65535:
sickrage.WEB_PORT = 8081
if not sickrage.WEB_COOKIE_SECRET:
sickrage.WEB_COOKIE_SECRET = generateCookieSecret()
# attempt to help prevent users from breaking links by using a bad url
if not sickrage.ANON_REDIRECT.endswith('?'):
sickrage.ANON_REDIRECT = ''
if not re.match(r'\d+\|[^|]+(?:\|[^|]+)*', sickrage.ROOT_DIRS):
sickrage.ROOT_DIRS = ''
sickrage.NAMING_FORCE_FOLDERS = check_force_season_folders()
if sickrage.NZB_METHOD not in ('blackhole', 'sabnzbd', 'nzbget'):
sickrage.NZB_METHOD = 'blackhole'
if not sickrage.PROVIDER_ORDER:
sickrage.PROVIDER_ORDER = sickrage.providersDict[GenericProvider.NZB].keys() + \
sickrage.providersDict[GenericProvider.TORRENT].keys()
if sickrage.TORRENT_METHOD not in ('blackhole', 'utorrent',
'transmission', 'deluge',
'deluged', 'download_station',
'rtorrent', 'qbittorrent',
'mlnet'):
sickrage.TORRENT_METHOD = 'blackhole'
if sickrage.PROPER_SEARCHER_INTERVAL not in ('15m', '45m', '90m',
'4h', 'daily'):
sickrage.PROPER_SEARCHER_INTERVAL = 'daily'
if sickrage.AUTOPOSTPROCESSOR_FREQ < sickrage.MIN_AUTOPOSTPROCESSOR_FREQ:
sickrage.AUTOPOSTPROCESSOR_FREQ = sickrage.MIN_AUTOPOSTPROCESSOR_FREQ
if sickrage.NAMECACHE_FREQ < sickrage.MIN_NAMECACHE_FREQ:
sickrage.NAMECACHE_FREQ = sickrage.MIN_NAMECACHE_FREQ
if sickrage.DAILY_SEARCHER_FREQ < sickrage.MIN_DAILY_SEARCHER_FREQ:
sickrage.DAILY_SEARCHER_FREQ = sickrage.MIN_DAILY_SEARCHER_FREQ
sickrage.MIN_BACKLOG_SEARCHER_FREQ = get_backlog_cycle_time()
if sickrage.BACKLOG_SEARCHER_FREQ < sickrage.MIN_BACKLOG_SEARCHER_FREQ:
sickrage.BACKLOG_SEARCHER_FREQ = sickrage.MIN_BACKLOG_SEARCHER_FREQ
if sickrage.VERSION_UPDATER_FREQ < sickrage.MIN_VERSION_UPDATER_FREQ:
sickrage.VERSION_UPDATER_FREQ = sickrage.MIN_VERSION_UPDATER_FREQ
if sickrage.SHOWUPDATE_HOUR > 23:
sickrage.SHOWUPDATE_HOUR = 0
elif sickrage.SHOWUPDATE_HOUR < 0:
sickrage.SHOWUPDATE_HOUR = 0
if sickrage.SUBTITLE_SEARCHER_FREQ < sickrage.MIN_SUBTITLE_SEARCHER_FREQ:
sickrage.SUBTITLE_SEARCHER_FREQ = sickrage.MIN_SUBTITLE_SEARCHER_FREQ
sickrage.NEWS_LATEST = sickrage.NEWS_LAST_READ
if sickrage.SUBTITLES_LANGUAGES[0] == '':
sickrage.SUBTITLES_LANGUAGES = []
sickrage.TIME_PRESET = sickrage.TIME_PRESET_W_SECONDS.replace(
":%S", "")
# initialize metadata_providers
sickrage.metadataProvideDict = get_metadata_generator_dict()
for cur_metadata_tuple in [
(sickrage.METADATA_KODI, kodi),
(sickrage.METADATA_KODI_12PLUS, kodi_12plus),
(sickrage.METADATA_MEDIABROWSER, mediabrowser),
(sickrage.METADATA_PS3, ps3), (sickrage.METADATA_WDTV, wdtv),
(sickrage.METADATA_TIVO, tivo),
(sickrage.METADATA_MEDE8ER, mede8er)
]:
(cur_metadata_config, cur_metadata_class) = cur_metadata_tuple
tmp_provider = cur_metadata_class.metadata_class()
tmp_provider.set_config(cur_metadata_config)
sickrage.metadataProvideDict[tmp_provider.name] = tmp_provider
# init caches
sickrage.NAMECACHE = nameCache()
# init queues
sickrage.SHOWUPDATER = ShowUpdater()
sickrage.SHOWQUEUE = ShowQueue()
sickrage.SEARCHQUEUE = SearchQueue()
# load data for shows from database
sickrage.showList = load_shows()
# init searchers
sickrage.DAILYSEARCHER = DailySearcher()
sickrage.BACKLOGSEARCHER = BacklogSearcher()
sickrage.PROPERSEARCHER = ProperSearcher()
sickrage.TRAKTSEARCHER = TraktSearcher()
sickrage.SUBTITLESEARCHER = SubtitleSearcher()
# init scheduler
sickrage.Scheduler = Scheduler()
# add version checker job to scheduler
sickrage.Scheduler.add_job(
sickrage.VERSIONUPDATER.run,
SRIntervalTrigger(
**{
'hours': sickrage.VERSION_UPDATER_FREQ,
'min': sickrage.MIN_VERSION_UPDATER_FREQ
}),
name="VERSIONUPDATER",
id="VERSIONUPDATER",
replace_existing=True)
# add network timezones updater job to scheduler
sickrage.Scheduler.add_job(update_network_dict,
SRIntervalTrigger(**{'days': 1}),
name="TZUPDATER",
id="TZUPDATER",
replace_existing=True)
# add namecache updater job to scheduler
sickrage.Scheduler.add_job(
sickrage.NAMECACHE.run,
SRIntervalTrigger(
**{
'minutes': sickrage.NAMECACHE_FREQ,
'min': sickrage.MIN_NAMECACHE_FREQ
}),
name="NAMECACHE",
id="NAMECACHE",
replace_existing=True)
# add show queue job to scheduler
sickrage.Scheduler.add_job(sickrage.SHOWQUEUE.run,
SRIntervalTrigger(**{'seconds': 3}),
name="SHOWQUEUE",
id="SHOWQUEUE",
replace_existing=True)
# add search queue job to scheduler
sickrage.Scheduler.add_job(sickrage.SEARCHQUEUE.run,
SRIntervalTrigger(**{'seconds': 1}),
name="SEARCHQUEUE",
id="SEARCHQUEUE",
replace_existing=True)
# add show updater job to scheduler
sickrage.Scheduler.add_job(
sickrage.SHOWUPDATER.run,
SRIntervalTrigger(
**{
'hours':
1,
'start_date':
datetime.datetime.now().replace(
hour=sickrage.SHOWUPDATE_HOUR)
}),
name="SHOWUPDATER",
id="SHOWUPDATER",
replace_existing=True)
# add daily search job to scheduler
sickrage.Scheduler.add_job(
sickrage.DAILYSEARCHER.run,
SRIntervalTrigger(
**{
'minutes': sickrage.DAILY_SEARCHER_FREQ,
'min': sickrage.MIN_DAILY_SEARCHER_FREQ
}),
name="DAILYSEARCHER",
id="DAILYSEARCHER",
replace_existing=True)
# add backlog search job to scheduler
sickrage.Scheduler.add_job(
sickrage.BACKLOGSEARCHER.run,
SRIntervalTrigger(
**{
'minutes': sickrage.BACKLOG_SEARCHER_FREQ,
'min': sickrage.MIN_BACKLOG_SEARCHER_FREQ
}),
name="BACKLOG",
id="BACKLOG",
replace_existing=True)
# add auto-postprocessing job to scheduler
job = sickrage.Scheduler.add_job(
auto_postprocessor.PostProcessor().run,
SRIntervalTrigger(
**{
'minutes': sickrage.AUTOPOSTPROCESSOR_FREQ,
'min': sickrage.MIN_AUTOPOSTPROCESSOR_FREQ
}),
name="POSTPROCESSOR",
id="POSTPROCESSOR",
replace_existing=True)
(job.pause, job.resume)[sickrage.PROCESS_AUTOMATICALLY]()
# add find propers job to scheduler
job = sickrage.Scheduler.add_job(
sickrage.PROPERSEARCHER.run,
SRIntervalTrigger(
**{
'minutes': {
'15m': 15,
'45m': 45,
'90m': 90,
'4h': 4 * 60,
'daily': 24 * 60
}[sickrage.PROPER_SEARCHER_INTERVAL]
}),
name="PROPERSEARCHER",
id="PROPERSEARCHER",
replace_existing=True)
(job.pause, job.resume)[sickrage.DOWNLOAD_PROPERS]()
# add trakt.tv checker job to scheduler
job = sickrage.Scheduler.add_job(
sickrage.TRAKTSEARCHER.run,
SRIntervalTrigger(**{'hours': 1}),
name="TRAKTSEARCHER",
id="TRAKTSEARCHER",
replace_existing=True,
)
(job.pause, job.resume)[sickrage.USE_TRAKT]()
# add subtitles finder job to scheduler
job = sickrage.Scheduler.add_job(
sickrage.SUBTITLESEARCHER.run,
SRIntervalTrigger(
**{'hours': sickrage.SUBTITLE_SEARCHER_FREQ}),
name="SUBTITLESEARCHER",
id="SUBTITLESEARCHER",
replace_existing=True)
(job.pause, job.resume)[sickrage.USE_SUBTITLES]()
# initialize web server
sickrage.WEB_SERVER = SRWebServer(
**{
'port':
int(sickrage.WEB_PORT),
'host':
sickrage.WEB_HOST,
'data_root':
sickrage.DATA_DIR,
'gui_root':
sickrage.GUI_DIR,
'web_root':
sickrage.WEB_ROOT,
'log_dir':
sickrage.WEB_LOG or sickrage.LOG_DIR,
'username':
sickrage.WEB_USERNAME,
'password':
sickrage.WEB_PASSWORD,
'enable_https':
sickrage.ENABLE_HTTPS,
'handle_reverse_proxy':
sickrage.HANDLE_REVERSE_PROXY,
'https_cert':
os.path.join(sickrage.ROOT_DIR, sickrage.HTTPS_CERT),
'https_key':
os.path.join(sickrage.ROOT_DIR, sickrage.HTTPS_KEY),
'daemonize':
sickrage.DAEMONIZE,
'pidfile':
sickrage.PIDFILE,
'stop_timeout':
3,
'nolaunch':
sickrage.WEB_NOLAUNCH
})
sickrage.LOGGER.info("SiCKRAGE VERSION:[{}] CONFIG:[{}]".format(
sickrage.VERSION, sickrage.CONFIG_FILE))
sickrage.INITIALIZED = True
return True
def retrieve_exceptions():
"""
Looks up the exceptions on github, parses them into a dict, and inserts them into the
scene_exceptions table in cache.db. Also clears the scene name cache.
"""
for indexer in sickrage.INDEXER_API().indexers:
if shouldRefresh(sickrage.INDEXER_API(indexer).name):
sickrage.LOGGER.info("Checking for scene exception updates for " +
sickrage.INDEXER_API(indexer).name + "")
loc = sickrage.INDEXER_API(indexer).config[b'scene_loc']
try:
data = getURL(loc,
session=sickrage.INDEXER_API(indexer).session)
except Exception:
continue
if data is None:
# When data is None, trouble connecting to github, or reading file failed
sickrage.LOGGER.debug(
"Check scene exceptions update failed. Unable to update from: "
+ loc)
continue
setLastRefresh(sickrage.INDEXER_API(indexer).name)
# each exception is on one line with the format indexer_id: 'show name 1', 'show name 2', etc
for cur_line in data.splitlines():
indexer_id, _, aliases = cur_line.partition(
':') # @UnusedVariable
if not aliases:
continue
indexer_id = int(indexer_id)
# regex out the list of shows, taking \' into account
# alias_list = [re.sub(r'\\(.)', r'\1', x) for x in re.findall(r"'(.*?)(?<!\\)',?", aliases)]
alias_list = [{
re.sub(r'\\(.)', r'\1', x): -1
} for x in re.findall(r"'(.*?)(?<!\\)',?", aliases)]
exception_dict[indexer_id] = alias_list
del alias_list
# cleanup
del data
# XEM scene exceptions
_xem_exceptions_fetcher()
for xem_ex in xem_exception_dict:
if xem_ex in exception_dict:
exception_dict[
xem_ex] = exception_dict[xem_ex] + xem_exception_dict[xem_ex]
else:
exception_dict[xem_ex] = xem_exception_dict[xem_ex]
# AniDB scene exceptions
_anidb_exceptions_fetcher()
for anidb_ex in anidb_exception_dict:
if anidb_ex in exception_dict:
exception_dict[anidb_ex] = exception_dict[
anidb_ex] + anidb_exception_dict[anidb_ex]
else:
exception_dict[anidb_ex] = anidb_exception_dict[anidb_ex]
queries = []
for cur_indexer_id in exception_dict:
sql_ex = cache_db.CacheDB().select(
"SELECT * FROM scene_exceptions WHERE indexer_id = ?;",
[cur_indexer_id])
existing_exceptions = [x[b"show_name"] for x in sql_ex]
if not cur_indexer_id in exception_dict:
continue
for cur_exception_dict in exception_dict[cur_indexer_id]:
for ex in cur_exception_dict.iteritems():
cur_exception, curSeason = ex
if cur_exception not in existing_exceptions:
queries.append([
"INSERT OR IGNORE INTO scene_exceptions (indexer_id, show_name, season) VALUES (?,?,?);",
[cur_indexer_id, cur_exception, curSeason]
])
if queries:
cache_db.CacheDB().mass_action(queries)
sickrage.LOGGER.debug("Updated scene exceptions")
else:
sickrage.LOGGER.debug("No scene exceptions update needed")
# cleanup
exception_dict.clear()
anidb_exception_dict.clear()
xem_exception_dict.clear()
def getSceneSeasons(indexer_id):
"""get a list of season numbers that have scene exceptions"""
seasons = cache_db.CacheDB().select(
"SELECT DISTINCT season FROM scene_exceptions WHERE indexer_id = ?",
[indexer_id])
return [cur_exception[b"season"] for cur_exception in seasons]
