class GamesService(WebService):
_name = "Games Service"
_id = "games"
def __init__(self, Routes):
self.scheduler = TornadoScheduler(timezone=utc)
# self.scheduler.add_job(self.score_games)
self.scheduler.add_job(self.score_games, 'interval', hours=1)
self.emitter = ipc.Emitter()
self.listener = ipc.Listener({
"games:getScores": self.score_games
})
super().__init__(Routes)
def score_games(self, date=None):
today = datetime_for_game(date)
print("games for {}".format(str(today)))
games = NBAGame.find_sync({"Date": today})
print("imported {} games today".format(len(games)))
box_results = []
score_results = []
game_count = 0
for game in games:
box_results.append( nba_jobs.box_score.delay(game["fid"]) )
for result in box_results:
score_results.append( nba_jobs.score_players.delay(result.get()) )
for result in score_results:
playerScoring = result.get()
self.emitter.publish("games:scores", playerScoring)
game_count += 1
print("scored {} games".format(game_count))
class SnapshotUtil(object):
instance = None
def __init__(self):
self.scheduler = TornadoScheduler()
@staticmethod
def get_instance():
if not SnapshotUtil.instance:
SnapshotUtil.instance = SnapshotUtil()
return SnapshotUtil.instance
def set_snapshot_interval(self, pos, hours):
self.scheduler.reschedule_job('snapshot_%d' % pos, trigger='interval', hours=hours)
def snapshot_start(self):
self.scheduler.start()
def snapshot_stop(self):
self.scheduler.shutdown()
def snapshot(self, pos, url):
print "snapshot",pos,url
path = os.path.join(os.getcwd(),CAPTURED_DIR)
path = os.path.join(path, str(pos)+"_"+time.strftime("%Y_%m_%d_%H_%M_%S",time.localtime(time.time()))+".jpeg")
command = "sudo ffmpeg -ss 30 -i \'%s\' -y -t 30 -r 1 -f image2 %s" % (url, path)
print command
os.system(command)
time.sleep(5)
add_caputured_image(os.path.basename(path), pos, 1920, 1080)
self.update_image(path, pos)
def update_image(self, path, pos):
datagen, headers = multipart_encode({'file': open(path, "rb"), 'pos': pos, "mac": get_mac_address(),'created_at': time.time()})
url = 'http://%s:%d%s'%(SERVER_WEBSITE, API_PORT, UPLOAD_IMG_URL)
print url
request = urllib2.Request(url, datagen, headers)
res = urllib2.urlopen(request, timeout=30).read()
print "ok"
if res == "ok":
return
else:
return
def remove_snapshot(self,pos):
self.scheduler.remove_job("snapshot_%d"%pos)
def add_snapshot(self, pos, url, hours):
self.scheduler.add_job(self.snapshot, 'interval', args=[ pos, url], seconds=100, id="snapshot_%d"%(pos))
def init_snapshot(self):
positions = get_positions()
for pos in positions:
self.add_snapshot(int(pos['position']), pos['ip_address'], pos['duration'])
def main():
# Use the above instantiated scheduler
# Set Tornado Scheduler
scheduler = TornadoScheduler()
# Use the imported jobs, every 60 minutes
scheduler.add_job(log_theta, 'interval', minutes=60)
scheduler.start()
application.listen(settings["listen.port"])
tornado.ioloop.IOLoop.instance().start()
class Application(tornado.web.Application):
def __init__(self, with_celery, model):
tornado.web.Application.__init__(self, urls, **settings)
self.mongo_client = motor.motor_tornado.MotorClient(
'/usr/local/var/run/mongodb/mongodb-27017.sock')
self.db = self.mongo_client.ubernow
self.redis = redis.StrictRedis(unix_socket_path='/usr/loca/var/run/redis/redis.sock')
self.scheduler = TornadoScheduler(
jobstores=JOBSTORES, executors=EXECUTORS, job_defaults=JOB_DEFAULTS, timezone=utc)
self.scheduler.start()
self.model = model
self.with_celery = with_celery
def __init__(self):
self._jobs = []
tornado.options.parse_command_line()
application = tornado.web.Application(
[
# --- * PROJECT * ---
(r'/', ListProjectsHandler),
(r'/project/(.+)', ProjectDetailsHandler), # project id
# --- * JOB * ---
(r'/exec/([^/]+)/([^/]+)', ExecuteProjectJobHandler), # project id, job id
# View job configuration context. Args: job id
(r'/job/([^/]+)/configuration', JobConfigurationHandler),
# --- * CONFIGURATION * ---
# Change branch
(r'/change/branch', ChangeBranchHandler),
],
template_path = os.path.join(os.path.dirname(__file__), '.', 'templates'),
static_path = os.path.join(os.path.dirname(__file__), '.', 'static')
)
self.http_server = tornado.httpserver.HTTPServer(application)
executors = {
'default': {'type': 'threadpool', 'max_workers': 20},
'processpool': ProcessPoolExecutor(max_workers=5)
}
logger.info("Initializing scheduler (pid: {}, port: '{}')...".format(os.getpid(), settings.HTTP_PORT))
self.scheduler = TornadoScheduler()
self.scheduler.configure(executors = executors)
self.scheduler.start()
def tornado_schedule():
from tornado.ioloop import IOLoop
from apscheduler.schedulers.tornado import TornadoScheduler
def tick():
print('Tick! The time is: %s' % datetime.now())
scheduler = TornadoScheduler()
scheduler.add_job(tick, 'interval', seconds=3)
scheduler.start()
print('Press Ctrl+{0} to exit'.format('Break' if os.name == 'nt' else 'C'))
# Execution will block here until Ctrl+C (Ctrl+Break on Windows) is pressed.
try:
IOLoop.instance().start()
except (KeyboardInterrupt, SystemExit):
pass
def main():
tornado.options.parse_config_file(os.path.join(os.path.dirname(__file__), "config.py"))
tornado.options.parse_command_line()
global app
app = Application()
app.iphone_data = {}
scraper_all()
scheduler = TornadoScheduler()
time_interval = 60
scheduler.add_job(scraper_all, 'interval', seconds=time_interval)
scheduler.add_job(update_data, 'interval', seconds=time_interval)
scheduler.start()
http_server = tornado.httpserver.HTTPServer(app)
http_server.listen(options.port)
tornado.ioloop.IOLoop.instance().start()
def __init__(self):
self._jobs = []
tornado.options.parse_command_line()
application = tornado.web.Application(
[
(r'/', IndexHandler),
# --- * CONFIGURATION * ---
# Context of resource configuration file. Args: resource configuration filename
(r'/config/resource/([^/]+)', ResourceConfigHandler),
# Context of translation configuration file. Args: translation configuration filename
(r'/config/translation/([^/]+)', TranslationConfigHandler),
# --- * DASHBOARD * ---
# Dashboard.
(r'/dashboard', DashboardHandler),
# --- * JOB * ---
# List of jobs.
(r'/jobs', ListJobsHandler),
# List resource slugs for resources in a project. Args: job id
(r'/job/([^/]+)/check/slugs', CheckSlugsHandler),
# Details of a job. Args: job id
(r'/job/([^/]+)/details', JobDetailsHandler),
# Context of most recent log for a job. Args: job id
# --- * LOG * ---
(r'/log/([^/]+)/context', LogContextHandler),
# List of projects.
# --- * PROJECT * ---
(r'/projects', ListProjectsHandler),
# Details of a project. Args: project id
(r'/project/([^/]+)/details', ProjectDetailsHandler),
# List of projects in translation platform (e.g. Transifex projects) Args: translation platform name
# --- * TRANSLATION PLATFORM * ---
(r'/translation/([^/]+)/projects', ListTranslationProjects),
# Details of a project in translation platform. Args: translation platform name, project slag
(r'/translation/([^/]+)/project/([^/]+)/details', TranslationProjectDetails),
# List of all translation strings for a resource of a language. Args: translation platform name, project slug, resource slug, langauge code
(r'/translation/([^/]+)/project/([^/]+)/resource/([^/]+)/translation/([^/]+)/strings', TranslationProjectTranslationStrings),
# Details of a source string. Args: translation platform name, project slug, resource slug
(r'/translation/([^/]+)/project/([^/]+)/resource/([^/]+)/source/strings', TranslationProjectSourceStringDetails)
],
template_path = os.path.join(os.path.dirname(__file__), '.', 'templates'),
static_path = os.path.join(os.path.dirname(__file__), '.', 'static')
)
self.http_server = tornado.httpserver.HTTPServer(application)
executors = {
'default': {'type': 'threadpool', 'max_workers': 20},
'processpool': ProcessPoolExecutor(max_workers=5)
}
logger.info("Initializing scheduler (pid: {}, port: '{}')...".format(os.getpid(), settings.HTTP_PORT))
self.scheduler = TornadoScheduler()
self.scheduler.configure(executors = executors)
self.scheduler.start()
def __init__(self, Routes):
self.scheduler = TornadoScheduler(timezone=utc)
# self.scheduler.add_job(self.score_games)
self.scheduler.add_job(self.score_games, 'interval', hours=1)
self.emitter = ipc.Emitter()
self.listener = ipc.Listener({
"games:getScores": self.score_games
})
super().__init__(Routes)
class ProjectManagerServer():
def __init__(self):
self._jobs = []
tornado.options.parse_command_line()
application = tornado.web.Application(
[
# --- * PROJECT * ---
(r'/', ListProjectsHandler),
(r'/project/(.+)', ProjectDetailsHandler), # project id
# --- * JOB * ---
(r'/exec/([^/]+)/([^/]+)', ExecuteProjectJobHandler), # project id, job id
# View job configuration context. Args: job id
(r'/job/([^/]+)/configuration', JobConfigurationHandler),
# --- * CONFIGURATION * ---
# Change branch
(r'/change/branch', ChangeBranchHandler),
],
template_path = os.path.join(os.path.dirname(__file__), '.', 'templates'),
static_path = os.path.join(os.path.dirname(__file__), '.', 'static')
)
self.http_server = tornado.httpserver.HTTPServer(application)
executors = {
'default': {'type': 'threadpool', 'max_workers': 20},
'processpool': ProcessPoolExecutor(max_workers=5)
}
logger.info("Initializing scheduler (pid: {}, port: '{}')...".format(os.getpid(), settings.HTTP_PORT))
self.scheduler = TornadoScheduler()
self.scheduler.configure(executors = executors)
self.scheduler.start()
# @classmethod
def start(self):
signal.signal(signal.SIGINT, self._signal_handler)
self.http_server.listen(settings.HTTP_PORT)
tornado.ioloop.IOLoop.current().start()
def _signal_handler(self, signal_type, frame):
if signal_type == signal.SIGINT:
logger.info('SIGINT')
else:
logger.warning('Unknown signal')
self.terminate()
# @classmethod
def terminate(self):
logger.info('Stopping console...')
self.scheduler.shutdown()
tornado.ioloop.IOLoop.current().stop()
sys.exit(0)
def __init__(self):
self.started = False
# process id
self.PID = os.getpid()
# cpu count
self.CPU_COUNT = cpu_count()
# generate notifiers dict
self.notifiersDict = AttrDict(libnotify=LibnotifyNotifier(),
kodi_notifier=KODINotifier(),
plex_notifier=PLEXNotifier(),
emby_notifier=EMBYNotifier(),
nmj_notifier=NMJNotifier(),
nmjv2_notifier=NMJv2Notifier(),
synoindex_notifier=synoIndexNotifier(),
synology_notifier=synologyNotifier(),
pytivo_notifier=pyTivoNotifier(),
growl_notifier=GrowlNotifier(),
prowl_notifier=ProwlNotifier(),
libnotify_notifier=LibnotifyNotifier(),
pushover_notifier=PushoverNotifier(),
boxcar_notifier=BoxcarNotifier(),
boxcar2_notifier=Boxcar2Notifier(),
nma_notifier=NMA_Notifier(),
pushalot_notifier=PushalotNotifier(),
pushbullet_notifier=PushbulletNotifier(),
freemobile_notifier=FreeMobileNotifier(),
twitter_notifier=TwitterNotifier(),
trakt_notifier=TraktNotifier(),
email_notifier=EmailNotifier())
# generate metadata providers dict
self.metadataProviderDict = get_metadata_generator_dict()
# generate providers dict
self.providersDict = providersDict()
# init notification queue
self.srNotifications = Notifications()
# init logger
self.srLogger = srLogger()
# init config
self.srConfig = srConfig()
# init scheduler service
self.srScheduler = TornadoScheduler()
# init web server
self.srWebServer = srWebServer()
# init web client session
self.srWebSession = srSession()
# google api
self.googleAuth = googleAuth()
# name cache
self.NAMECACHE = srNameCache()
# queues
self.SHOWQUEUE = srShowQueue()
self.SEARCHQUEUE = srSearchQueue()
# updaters
self.VERSIONUPDATER = srVersionUpdater()
self.SHOWUPDATER = srShowUpdater()
# searchers
self.DAILYSEARCHER = srDailySearcher()
self.BACKLOGSEARCHER = srBacklogSearcher()
self.PROPERSEARCHER = srProperSearcher()
self.TRAKTSEARCHER = srTraktSearcher()
self.SUBTITLESEARCHER = srSubtitleSearcher()
# auto postprocessor
self.AUTOPOSTPROCESSOR = srPostProcessor()
# sickrage version
self.NEWEST_VERSION = None
self.NEWEST_VERSION_STRING = None
# anidb connection
self.ADBA_CONNECTION = None
# show list
self.SHOWLIST = []
def serve():
"""Read configuration and start the server."""
global EMAIL_FROM, SMTP_SETTINGS
jobstores = {'default': SQLAlchemyJobStore(url=JOBS_STORE)}
scheduler = TornadoScheduler(jobstores=jobstores, timezone=pytz.utc)
scheduler.start()
define('port', default=3210, help='run on the given port', type=int)
define('address',
default='',
help='bind the server at the given address',
type=str)
define('ssl_cert',
default=os.path.join(os.path.dirname(__file__), 'ssl',
'diffido_cert.pem'),
help='specify the SSL certificate to use for secure connections')
define('ssl_key',
default=os.path.join(os.path.dirname(__file__), 'ssl',
'diffido_key.pem'),
help='specify the SSL private key to use for secure connections')
define('admin-email',
default='',
help='email address of the site administrator',
type=str)
define('smtp-host',
default='localhost',
help='SMTP server address',
type=str)
define('smtp-port', default=0, help='SMTP server port', type=int)
define('smtp-local-hostname',
default=None,
help='SMTP local hostname',
type=str)
define('smtp-use-ssl',
default=False,
help='Use SSL to connect to the SMTP server',
type=bool)
define('smtp-starttls',
default=False,
help='Use STARTTLS to connect to the SMTP server',
type=bool)
define('smtp-ssl-keyfile', default=None, help='SSL key file', type=str)
define('smtp-ssl-certfile', default=None, help='SSL cert file', type=str)
define('smtp-ssl-context', default=None, help='SSL context', type=str)
define('debug', default=False, help='run in debug mode', type=bool)
define('config',
help='read configuration file',
callback=lambda path: tornado.options.parse_config_file(
path, final=False))
if not options.config and os.path.isfile(DEFAULT_CONF):
tornado.options.parse_config_file(DEFAULT_CONF, final=False)
tornado.options.parse_command_line()
if options.admin_email:
EMAIL_FROM = options.admin_email
for key, value in options.as_dict().items():
if key.startswith('smtp-'):
SMTP_SETTINGS[key] = value
if options.debug:
logger.setLevel(logging.DEBUG)
ssl_options = {}
if os.path.isfile(options.ssl_key) and os.path.isfile(options.ssl_cert):
ssl_options = dict(certfile=options.ssl_cert, keyfile=options.ssl_key)
init_params = dict(listen_port=options.port,
logger=logger,
ssl_options=ssl_options,
scheduler=scheduler)
git_init()
_reset_schedules_path = r'schedules/reset'
_schedule_run_path = r'schedules/(?P<id_>\d+)/run'
_schedules_path = r'schedules/?(?P<id_>\d+)?'
_history_path = r'schedules/?(?P<id_>\d+)/history'
_diff_path = r'schedules/(?P<id_>\d+)/diff/(?P<commit_id>[0-9a-f]+)/?(?P<old_commit_id>[0-9a-f]+)?/?'
application = tornado.web.Application([
(r'/api/%s' % _reset_schedules_path, ResetSchedulesHandler,
init_params),
(r'/api/v%s/%s' % (API_VERSION, _reset_schedules_path),
ResetSchedulesHandler, init_params),
(r'/api/%s' % _schedule_run_path, RunScheduleHandler, init_params),
(r'/api/v%s/%s' %
(API_VERSION, _schedule_run_path), RunScheduleHandler, init_params),
(r'/api/%s' % _history_path, HistoryHandler, init_params),
(r'/api/v%s/%s' %
(API_VERSION, _history_path), HistoryHandler, init_params),
(r'/api/%s' % _diff_path, DiffHandler, init_params),
(r'/api/v%s/%s' % (API_VERSION, _diff_path), DiffHandler, init_params),
(r'/api/%s' % _schedules_path, SchedulesHandler, init_params),
(r'/api/v%s/%s' %
(API_VERSION, _schedules_path), SchedulesHandler, init_params),
(r'/?(.*)', TemplateHandler, init_params),
],
static_path=os.path.join(
os.path.dirname(__file__),
'dist/static'),
template_path=os.path.join(
os.path.dirname(__file__),
'dist/'),
debug=options.debug)
http_server = tornado.httpserver.HTTPServer(application,
ssl_options=ssl_options
or None)
logger.info('Start serving on %s://%s:%d',
'https' if ssl_options else 'http',
options.address if options.address else '127.0.0.1',
options.port)
http_server.listen(options.port, options.address)
try:
IOLoop.instance().start()
except (KeyboardInterrupt, SystemExit):
pass
from apscheduler.schedulers.tornado import TornadoScheduler
import atexit
from datetime import date
# port = 10100
import argparse
parser = argparse.ArgumentParser(description='Run central migration service.')
parser.add_argument('port', help='The port to listen on.')
args = parser.parse_args()
port = args.port
# https://stackoverflow.com/questions/21214270/scheduling-a-function-to-run-every-hour-on-flask
# garbage collection scheduler
def garbage_collect():
print('running on port {}'.format(port))
if __name__ == "__main__":
gc_scheduler = TornadoScheduler()
gc_scheduler.add_job(garbage_collect, 'interval', seconds=3)
gc_scheduler.start()
create_central_migration_service().listen(port)
try:
IOLoop.instance().start()
except (KeyboardInterrupt, SystemExit):
pass
def __init__(self):
self.started = False
self.io_loop = IOLoop.current()
# process id
self.PID = os.getpid()
# generate notifiers dict
self.notifiersDict = notifiersDict()
# generate metadata providers dict
self.metadataProvidersDict = metadataProvidersDict()
# generate providers dict
self.providersDict = providersDict()
# init notification queue
self.srNotifications = Notifications()
# init logger
self.srLogger = srLogger()
# init config
self.srConfig = srConfig()
# init databases
self.mainDB = MainDB()
self.cacheDB = CacheDB()
self.failedDB = FailedDB()
# init scheduler service
self.srScheduler = TornadoScheduler()
# init web server
self.srWebServer = srWebServer()
# init web client session
self.srWebSession = srSession()
# google api
self.googleAuth = googleAuth()
# name cache
self.NAMECACHE = srNameCache()
# queues
self.SHOWQUEUE = srShowQueue()
self.SEARCHQUEUE = srSearchQueue()
# updaters
self.VERSIONUPDATER = srVersionUpdater()
self.SHOWUPDATER = srShowUpdater()
# searchers
self.DAILYSEARCHER = srDailySearcher()
self.BACKLOGSEARCHER = srBacklogSearcher()
self.PROPERSEARCHER = srProperSearcher()
self.TRAKTSEARCHER = srTraktSearcher()
self.SUBTITLESEARCHER = srSubtitleSearcher()
# auto postprocessor
self.AUTOPOSTPROCESSOR = srPostProcessor()
# sickrage version
self.NEWEST_VERSION = None
self.NEWEST_VERSION_STRING = None
# anidb connection
self.ADBA_CONNECTION = None
# show list
self.SHOWLIST = []
class Core(object):
def __init__(self):
self.started = False
self.daemon = None
self.io_loop = None
self.pid = os.getpid()
self.showlist = []
try:
self.tz = tz.tzwinlocal() if tz.tzwinlocal else tz.tzlocal()
except Exception:
self.tz = tz.tzlocal()
self.config_file = None
self.data_dir = None
self.cache_dir = None
self.quiet = None
self.no_launch = None
self.web_port = None
self.developer = None
self.debug = None
self.newest_version_string = None
self.naming_ep_type = ("%(seasonnumber)dx%(episodenumber)02d",
"s%(seasonnumber)02de%(episodenumber)02d",
"S%(seasonnumber)02dE%(episodenumber)02d",
"%(seasonnumber)02dx%(episodenumber)02d",
"S%(seasonnumber)02d E%(episodenumber)02d")
self.sports_ep_type = ("%(seasonnumber)dx%(episodenumber)02d",
"s%(seasonnumber)02de%(episodenumber)02d",
"S%(seasonnumber)02dE%(episodenumber)02d",
"%(seasonnumber)02dx%(episodenumber)02d",
"S%(seasonnumber)02 dE%(episodenumber)02d")
self.naming_ep_type_text = ("1x02", "s01e02", "S01E02", "01x02", "S01 E02",)
self.naming_multi_ep_type = {0: ["-%(episodenumber)02d"] * len(self.naming_ep_type),
1: [" - " + x for x in self.naming_ep_type],
2: [x + "%(episodenumber)02d" for x in ("x", "e", "E", "x")]}
self.naming_multi_ep_type_text = ("extend", "duplicate", "repeat")
self.naming_sep_type = (" - ", " ")
self.naming_sep_type_text = (" - ", "space")
self.user_agent = 'SiCKRAGE.CE.1/({};{};{})'.format(platform.system(), platform.release(), str(uuid.uuid1()))
self.languages = [language for language in os.listdir(sickrage.LOCALE_DIR) if '_' in language]
self.sys_encoding = get_sys_encoding()
self.client_web_urls = {'torrent': '', 'newznab': ''}
self.adba_connection = None
self.notifier_providers = None
self.metadata_providers = {}
self.search_providers = None
self.log = None
self.config = None
self.alerts = None
self.main_db = None
self.cache_db = None
self.scheduler = None
self.wserver = None
self.google_auth = None
self.name_cache = None
self.show_queue = None
self.search_queue = None
self.postprocessor_queue = None
self.event_queue = None
self.version_updater = None
self.show_updater = None
self.tz_updater = None
self.rsscache_updater = None
self.daily_searcher = None
self.backlog_searcher = None
self.proper_searcher = None
self.trakt_searcher = None
self.subtitle_searcher = None
self.auto_postprocessor = None
self.upnp_client = None
self.oidc_client = None
self.quicksearch_cache = None
def start(self):
self.started = True
self.io_loop = IOLoop.current()
# thread name
threading.currentThread().setName('CORE')
# patch modules with encoding kludge
patch_modules()
# init core classes
self.notifier_providers = NotifierProviders()
self.metadata_providers = MetadataProviders()
self.search_providers = SearchProviders()
self.log = Logger()
self.config = Config()
self.alerts = Notifications()
self.main_db = MainDB()
self.cache_db = CacheDB()
self.scheduler = TornadoScheduler()
self.wserver = WebServer()
self.name_cache = NameCache()
self.show_queue = ShowQueue()
self.search_queue = SearchQueue()
self.postprocessor_queue = PostProcessorQueue()
self.event_queue = EventQueue()
self.version_updater = VersionUpdater()
self.show_updater = ShowUpdater()
self.tz_updater = TimeZoneUpdater()
self.rsscache_updater = RSSCacheUpdater()
self.daily_searcher = DailySearcher()
self.failed_snatch_searcher = FailedSnatchSearcher()
self.backlog_searcher = BacklogSearcher()
self.proper_searcher = ProperSearcher()
self.trakt_searcher = TraktSearcher()
self.subtitle_searcher = SubtitleSearcher()
self.auto_postprocessor = AutoPostProcessor()
self.upnp_client = UPNPClient()
self.quicksearch_cache = QuicksearchCache()
# setup oidc client
realm = KeycloakRealm(server_url='https://auth.sickrage.ca', realm_name='sickrage')
self.oidc_client = realm.open_id_connect(client_id='sickrage-app',
client_secret='5d4710b2-ca70-4d39-b5a3-0705e2c5e703')
# Check if we need to perform a restore first
if os.path.exists(os.path.abspath(os.path.join(self.data_dir, 'restore'))):
success = restoreSR(os.path.abspath(os.path.join(self.data_dir, 'restore')), self.data_dir)
self.log.info("Restoring SiCKRAGE backup: {}!".format(("FAILED", "SUCCESSFUL")[success]))
if success:
shutil.rmtree(os.path.abspath(os.path.join(self.data_dir, 'restore')), ignore_errors=True)
# migrate old database file names to new ones
if os.path.isfile(os.path.abspath(os.path.join(self.data_dir, 'sickbeard.db'))):
if os.path.isfile(os.path.join(self.data_dir, 'sickrage.db')):
helpers.move_file(os.path.join(self.data_dir, 'sickrage.db'),
os.path.join(self.data_dir, '{}.bak-{}'
.format('sickrage.db',
datetime.datetime.now().strftime(
'%Y%m%d_%H%M%S'))))
helpers.move_file(os.path.abspath(os.path.join(self.data_dir, 'sickbeard.db')),
os.path.abspath(os.path.join(self.data_dir, 'sickrage.db')))
# load config
self.config.load()
# set language
self.config.change_gui_lang(self.config.gui_lang)
# set socket timeout
socket.setdefaulttimeout(self.config.socket_timeout)
# setup logger settings
self.log.logSize = self.config.log_size
self.log.logNr = self.config.log_nr
self.log.logFile = os.path.join(self.data_dir, 'logs', 'sickrage.log')
self.log.debugLogging = self.config.debug
self.log.consoleLogging = not self.quiet
# start logger
self.log.start()
# user agent
if self.config.random_user_agent:
self.user_agent = UserAgent().random
urlparse.uses_netloc.append('scgi')
urllib.FancyURLopener.version = self.user_agent
# set torrent client web url
torrent_webui_url(True)
# Check available space
try:
total_space, available_space = getFreeSpace(self.data_dir)
if available_space < 100:
self.log.error('Shutting down as SiCKRAGE needs some space to work. You\'ll get corrupted data '
'otherwise. Only %sMB left', available_space)
return
except Exception:
self.log.error('Failed getting disk space: %s', traceback.format_exc())
# perform database startup actions
for db in [self.main_db, self.cache_db]:
# initialize database
db.initialize()
# check integrity of database
db.check_integrity()
# migrate database
db.migrate()
# misc database cleanups
db.cleanup()
# upgrade database
db.upgrade()
# compact main database
if self.config.last_db_compact < time.time() - 604800: # 7 days
self.main_db.compact()
self.config.last_db_compact = int(time.time())
# load name cache
self.name_cache.load()
# load data for shows from database
self.load_shows()
if self.config.default_page not in ('schedule', 'history', 'IRC'):
self.config.default_page = 'home'
# cleanup cache folder
for folder in ['mako', 'sessions', 'indexers']:
try:
shutil.rmtree(os.path.join(sickrage.app.cache_dir, folder), ignore_errors=True)
except Exception:
continue
if self.config.web_port < 21 or self.config.web_port > 65535:
self.config.web_port = 8081
if not self.config.web_cookie_secret:
self.config.web_cookie_secret = generate_secret()
# attempt to help prevent users from breaking links by using a bad url
if not self.config.anon_redirect.endswith('?'):
self.config.anon_redirect = ''
if not re.match(r'\d+\|[^|]+(?:\|[^|]+)*', self.config.root_dirs):
self.config.root_dirs = ''
self.config.naming_force_folders = check_force_season_folders()
if self.config.nzb_method not in ('blackhole', 'sabnzbd', 'nzbget'):
self.config.nzb_method = 'blackhole'
if self.config.torrent_method not in ('blackhole', 'utorrent', 'transmission', 'deluge', 'deluged',
'download_station', 'rtorrent', 'qbittorrent', 'mlnet', 'putio'):
self.config.torrent_method = 'blackhole'
if self.config.autopostprocessor_freq < self.config.min_autopostprocessor_freq:
self.config.autopostprocessor_freq = self.config.min_autopostprocessor_freq
if self.config.daily_searcher_freq < self.config.min_daily_searcher_freq:
self.config.daily_searcher_freq = self.config.min_daily_searcher_freq
if self.config.backlog_searcher_freq < self.config.min_backlog_searcher_freq:
self.config.backlog_searcher_freq = self.config.min_backlog_searcher_freq
if self.config.version_updater_freq < self.config.min_version_updater_freq:
self.config.version_updater_freq = self.config.min_version_updater_freq
if self.config.subtitle_searcher_freq < self.config.min_subtitle_searcher_freq:
self.config.subtitle_searcher_freq = self.config.min_subtitle_searcher_freq
if self.config.failed_snatch_age < self.config.min_failed_snatch_age:
self.config.failed_snatch_age = self.config.min_failed_snatch_age
if self.config.proper_searcher_interval not in ('15m', '45m', '90m', '4h', 'daily'):
self.config.proper_searcher_interval = 'daily'
if self.config.showupdate_hour < 0 or self.config.showupdate_hour > 23:
self.config.showupdate_hour = 0
# add version checker job
self.scheduler.add_job(
self.version_updater.run,
IntervalTrigger(
hours=self.config.version_updater_freq,
),
name=self.version_updater.name,
id=self.version_updater.name
)
# add network timezones updater job
self.scheduler.add_job(
self.tz_updater.run,
IntervalTrigger(
days=1,
),
name=self.tz_updater.name,
id=self.tz_updater.name
)
# add show updater job
self.scheduler.add_job(
self.show_updater.run,
IntervalTrigger(
days=1,
start_date=datetime.datetime.now().replace(hour=self.config.showupdate_hour)
),
name=self.show_updater.name,
id=self.show_updater.name
)
# add rss cache updater job
self.scheduler.add_job(
self.rsscache_updater.run,
IntervalTrigger(
minutes=15,
),
name=self.rsscache_updater.name,
id=self.rsscache_updater.name
)
# add daily search job
self.scheduler.add_job(
self.daily_searcher.run,
IntervalTrigger(
minutes=self.config.daily_searcher_freq,
start_date=datetime.datetime.now() + datetime.timedelta(minutes=4)
),
name=self.daily_searcher.name,
id=self.daily_searcher.name
)
# add failed snatch search job
self.scheduler.add_job(
self.failed_snatch_searcher.run,
IntervalTrigger(
hours=1,
start_date=datetime.datetime.now() + datetime.timedelta(minutes=4)
),
name=self.failed_snatch_searcher.name,
id=self.failed_snatch_searcher.name
)
# add backlog search job
self.scheduler.add_job(
self.backlog_searcher.run,
IntervalTrigger(
minutes=self.config.backlog_searcher_freq,
start_date=datetime.datetime.now() + datetime.timedelta(minutes=30)
),
name=self.backlog_searcher.name,
id=self.backlog_searcher.name
)
# add auto-postprocessing job
self.scheduler.add_job(
self.auto_postprocessor.run,
IntervalTrigger(
minutes=self.config.autopostprocessor_freq
),
name=self.auto_postprocessor.name,
id=self.auto_postprocessor.name
)
# add find proper job
self.scheduler.add_job(
self.proper_searcher.run,
IntervalTrigger(
minutes={
'15m': 15,
'45m': 45,
'90m': 90,
'4h': 4 * 60,
'daily': 24 * 60
}[self.config.proper_searcher_interval]
),
name=self.proper_searcher.name,
id=self.proper_searcher.name
)
# add trakt.tv checker job
self.scheduler.add_job(
self.trakt_searcher.run,
IntervalTrigger(
hours=1
),
name=self.trakt_searcher.name,
id=self.trakt_searcher.name
)
# add subtitles finder job
self.scheduler.add_job(
self.subtitle_searcher.run,
IntervalTrigger(
hours=self.config.subtitle_searcher_freq
),
name=self.subtitle_searcher.name,
id=self.subtitle_searcher.name
)
# add upnp client job
self.scheduler.add_job(
self.upnp_client.run,
IntervalTrigger(
seconds=self.upnp_client._nat_portmap_lifetime
),
name=self.upnp_client.name,
id=self.upnp_client.name
)
# add namecache update job
self.scheduler.add_job(
self.name_cache.build_all,
IntervalTrigger(
days=1,
),
name=self.name_cache.name,
id=self.name_cache.name
)
# start scheduler service
self.scheduler.start()
# start queue's
self.search_queue.start()
self.show_queue.start()
self.postprocessor_queue.start()
self.event_queue.start()
# fire off startup events
self.event_queue.fire_event(self.name_cache.build_all)
self.event_queue.fire_event(self.version_updater.run)
self.event_queue.fire_event(self.tz_updater.run)
# start webserver
self.wserver.start()
# launch browser window
if all([not sickrage.app.no_launch, sickrage.app.config.launch_browser]):
self.event_queue.fire_event(lambda: launch_browser(('http', 'https')[sickrage.app.config.enable_https],
sickrage.app.config.web_host,
sickrage.app.config.web_port))
# start ioloop
self.io_loop.start()
def shutdown(self, restart=False):
if self.started:
self.log.info('SiCKRAGE IS SHUTTING DOWN!!!')
# shutdown webserver
if self.wserver:
self.wserver.shutdown()
# shutdown show queue
if self.show_queue:
self.log.debug("Shutting down show queue")
self.show_queue.shutdown()
del self.show_queue
# shutdown search queue
if self.search_queue:
self.log.debug("Shutting down search queue")
self.search_queue.shutdown()
del self.search_queue
# shutdown post-processor queue
if self.postprocessor_queue:
self.log.debug("Shutting down post-processor queue")
self.postprocessor_queue.shutdown()
del self.postprocessor_queue
# shutdown event queue
if self.event_queue:
self.log.debug("Shutting down event queue")
self.event_queue.shutdown()
del self.event_queue
# log out of ADBA
if self.adba_connection:
self.log.debug("Shutting down ANIDB connection")
self.adba_connection.stop()
# save all show and config settings
self.save_all()
# close databases
for db in [self.main_db, self.cache_db]:
if db.opened:
self.log.debug("Shutting down {} database connection".format(db.name))
db.close()
# shutdown logging
if self.log:
self.log.close()
if restart:
os.execl(sys.executable, sys.executable, *sys.argv)
if sickrage.app.daemon:
sickrage.app.daemon.stop()
self.started = False
if self.io_loop:
self.io_loop.stop()
def save_all(self):
# write all shows
self.log.info("Saving all shows to the database")
for show in self.showlist:
try:
show.save_to_db()
except Exception:
continue
# save config
self.config.save()
def load_shows(self):
"""
Populates the showlist and quicksearch cache with shows and episodes from the database
"""
self.quicksearch_cache.load()
for dbData in self.main_db.all('tv_shows'):
show = TVShow(int(dbData['indexer']), int(dbData['indexer_id']))
try:
self.log.debug("Loading data for show: [{}]".format(show.name))
self.showlist.append(show)
self.quicksearch_cache.add_show(show.indexerid)
except Exception as e:
self.log.debug("Show error in [%s]: %s" % (show.location, str(e)))
from datetime import datetime
import os
from tornado.ioloop import IOLoop
from apscheduler.schedulers.tornado import TornadoScheduler
def tick():
print('Tick! The time is: %s' % datetime.now())
if __name__ == '__main__':
scheduler = TornadoScheduler()
scheduler.add_job(tick, 'interval', seconds=3)
scheduler.start()
print('Press Ctrl+{0} to exit'.format('Break' if os.name == 'nt' else 'C'))
# Execution will block here until Ctrl+C (Ctrl+Break on Windows) is pressed.
try:
IOLoop.instance().start()
except (KeyboardInterrupt, SystemExit):
pass
def __init__(self):##牌类型，牌字
self.m_scheduler = TornadoScheduler()
self.start()
for container in docker_client.containers():
indexer = DockerPackageIndexer(container, docker_client)
indexer.index()
if __name__ == '__main__':
tornado.log.enable_pretty_logging()
tornado.options.parse_command_line()
try:
docker_client = Client(**kwargs_from_env(assert_hostname=False))
docker_client.ping()
except ConnectionError:
logging.error("Unable to connect to Docker. Ensure Docker is running and environment variables are set.")
exit(1)
scheduler = TornadoScheduler()
scheduler.add_job(lambda: index_container_packages(docker_client), 'interval', seconds=tornado.options.options.interval)
scheduler.start()
app = tornado.web.Application([
(r"/", SearchHandler),
], debug=tornado.options.options.reload)
http_server = tornado.httpserver.HTTPServer(app)
http_server.listen(tornado.options.options.port)
try:
tornado.ioloop.IOLoop.current().start()
except (KeyboardInterrupt, SystemExit):
pass
def start(self):
self.started = True
# thread name
threading.currentThread().setName('CORE')
# init sentry
self.init_sentry()
# scheduler
self.scheduler = TornadoScheduler({'apscheduler.timezone': 'UTC'})
# init core classes
self.api = API()
self.config = Config(self.db_type, self.db_prefix, self.db_host, self.db_port, self.db_username, self.db_password)
self.main_db = MainDB(self.db_type, self.db_prefix, self.db_host, self.db_port, self.db_username, self.db_password)
self.cache_db = CacheDB(self.db_type, self.db_prefix, self.db_host, self.db_port, self.db_username, self.db_password)
self.notification_providers = NotificationProviders()
self.metadata_providers = MetadataProviders()
self.search_providers = SearchProviders()
self.series_providers = SeriesProviders()
self.log = Logger()
self.alerts = Notifications()
self.wserver = WebServer()
self.show_queue = ShowQueue()
self.search_queue = SearchQueue()
self.postprocessor_queue = PostProcessorQueue()
self.version_updater = VersionUpdater()
self.show_updater = ShowUpdater()
self.tz_updater = TimeZoneUpdater()
self.rsscache_updater = RSSCacheUpdater()
self.daily_searcher = DailySearcher()
self.failed_snatch_searcher = FailedSnatchSearcher()
self.backlog_searcher = BacklogSearcher()
self.proper_searcher = ProperSearcher()
self.trakt_searcher = TraktSearcher()
self.subtitle_searcher = SubtitleSearcher()
self.auto_postprocessor = AutoPostProcessor()
self.upnp_client = UPNPClient()
self.announcements = Announcements()
self.amqp_consumer = AMQPConsumer()
# authorization sso client
self.auth_server = AuthServer()
# check available space
try:
self.log.info("Performing disk space checks")
total_space, available_space = get_free_space(self.data_dir)
if available_space < 100:
self.log.warning('Shutting down as SiCKRAGE needs some space to work. You\'ll get corrupted data otherwise. Only %sMB left', available_space)
return
except Exception:
self.log.error('Failed getting disk space: %s', traceback.format_exc())
# check if we need to perform a restore first
if os.path.exists(os.path.abspath(os.path.join(self.data_dir, 'restore'))):
self.log.info('Performing restore of backup files')
success = restore_app_data(os.path.abspath(os.path.join(self.data_dir, 'restore')), self.data_dir)
self.log.info("Restoring SiCKRAGE backup: %s!" % ("FAILED", "SUCCESSFUL")[success])
if success:
# remove restore files
shutil.rmtree(os.path.abspath(os.path.join(self.data_dir, 'restore')), ignore_errors=True)
# migrate old database file names to new ones
if os.path.isfile(os.path.abspath(os.path.join(self.data_dir, 'sickbeard.db'))):
if os.path.isfile(os.path.join(self.data_dir, 'sickrage.db')):
helpers.move_file(os.path.join(self.data_dir, 'sickrage.db'),
os.path.join(self.data_dir, '{}.bak-{}'
.format('sickrage.db',
datetime.datetime.now().strftime(
'%Y%m%d_%H%M%S'))))
helpers.move_file(os.path.abspath(os.path.join(self.data_dir, 'sickbeard.db')),
os.path.abspath(os.path.join(self.data_dir, 'sickrage.db')))
# setup databases
self.main_db.setup()
self.config.db.setup()
self.cache_db.setup()
# load config
self.config.load()
# migrate config
self.config.migrate_config_file(self.config_file)
# add server id tag to sentry
sentry_sdk.set_tag('server_id', self.config.general.server_id)
# add user to sentry
sentry_sdk.set_user({
'id': self.config.user.sub_id,
'username': self.config.user.username,
'email': self.config.user.email
})
# config overrides
if self.web_port:
self.config.general.web_port = self.web_port
if self.web_root:
self.config.general.web_root = self.web_root
# set language
change_gui_lang(self.config.gui.gui_lang)
# set socket timeout
socket.setdefaulttimeout(self.config.general.socket_timeout)
# set ssl cert/key filenames
self.https_cert_file = os.path.abspath(os.path.join(self.data_dir, 'server.crt'))
self.https_key_file = os.path.abspath(os.path.join(self.data_dir, 'server.key'))
# setup logger settings
self.log.logSize = self.config.general.log_size
self.log.logNr = self.config.general.log_nr
self.log.logFile = os.path.join(self.data_dir, 'logs', 'sickrage.log')
self.log.debugLogging = self.debug or self.config.general.debug
self.log.consoleLogging = not self.quiet
# start logger
self.log.start()
# user agent
if self.config.general.random_user_agent:
self.user_agent = UserAgent().random
uses_netloc.append('scgi')
FancyURLopener.version = self.user_agent
# set torrent client web url
torrent_webui_url(True)
if self.config.general.default_page not in DefaultHomePage:
self.config.general.default_page = DefaultHomePage.HOME
# attempt to help prevent users from breaking links by using a bad url
if not self.config.general.anon_redirect.endswith('?'):
self.config.general.anon_redirect = ''
if not re.match(r'\d+\|[^|]+(?:\|[^|]+)*', self.config.general.root_dirs):
self.config.general.root_dirs = ''
self.naming_force_folders = check_force_season_folders()
if self.config.general.nzb_method not in NzbMethod:
self.config.general.nzb_method = NzbMethod.BLACKHOLE
if self.config.general.torrent_method not in TorrentMethod:
self.config.general.torrent_method = TorrentMethod.BLACKHOLE
if self.config.general.auto_postprocessor_freq < self.min_auto_postprocessor_freq:
self.config.general.auto_postprocessor_freq = self.min_auto_postprocessor_freq
if self.config.general.daily_searcher_freq < self.min_daily_searcher_freq:
self.config.general.daily_searcher_freq = self.min_daily_searcher_freq
if self.config.general.backlog_searcher_freq < self.min_backlog_searcher_freq:
self.config.general.backlog_searcher_freq = self.min_backlog_searcher_freq
if self.config.general.version_updater_freq < self.min_version_updater_freq:
self.config.general.version_updater_freq = self.min_version_updater_freq
if self.config.general.subtitle_searcher_freq < self.min_subtitle_searcher_freq:
self.config.general.subtitle_searcher_freq = self.min_subtitle_searcher_freq
if self.config.failed_snatches.age < self.min_failed_snatch_age:
self.config.failed_snatches.age = self.min_failed_snatch_age
if self.config.general.proper_searcher_interval not in CheckPropersInterval:
self.config.general.proper_searcher_interval = CheckPropersInterval.DAILY
if self.config.general.show_update_hour < 0 or self.config.general.show_update_hour > 23:
self.config.general.show_update_hour = 0
# add app updater job
self.scheduler.add_job(
self.version_updater.task,
IntervalTrigger(
hours=1,
start_date=datetime.datetime.now() + datetime.timedelta(minutes=4),
timezone='utc'
),
name=self.version_updater.name,
id=self.version_updater.name
)
# add show updater job
self.scheduler.add_job(
self.show_updater.task,
IntervalTrigger(
days=1,
start_date=datetime.datetime.now().replace(hour=self.config.general.show_update_hour),
timezone='utc'
),
name=self.show_updater.name,
id=self.show_updater.name
)
# add rss cache updater job
self.scheduler.add_job(
self.rsscache_updater.task,
IntervalTrigger(
minutes=15,
timezone='utc'
),
name=self.rsscache_updater.name,
id=self.rsscache_updater.name
)
# add daily search job
self.scheduler.add_job(
self.daily_searcher.task,
IntervalTrigger(
minutes=self.config.general.daily_searcher_freq,
start_date=datetime.datetime.now() + datetime.timedelta(minutes=4),
timezone='utc'
),
name=self.daily_searcher.name,
id=self.daily_searcher.name
)
# add failed snatch search job
self.scheduler.add_job(
self.failed_snatch_searcher.task,
IntervalTrigger(
hours=1,
start_date=datetime.datetime.now() + datetime.timedelta(minutes=4),
timezone='utc'
),
name=self.failed_snatch_searcher.name,
id=self.failed_snatch_searcher.name
)
# add backlog search job
self.scheduler.add_job(
self.backlog_searcher.task,
IntervalTrigger(
minutes=self.config.general.backlog_searcher_freq,
start_date=datetime.datetime.now() + datetime.timedelta(minutes=30),
timezone='utc'
),
name=self.backlog_searcher.name,
id=self.backlog_searcher.name
)
# add auto-postprocessing job
self.scheduler.add_job(
self.auto_postprocessor.task,
IntervalTrigger(
minutes=self.config.general.auto_postprocessor_freq,
timezone='utc'
),
name=self.auto_postprocessor.name,
id=self.auto_postprocessor.name
)
# add find proper job
self.scheduler.add_job(
self.proper_searcher.task,
IntervalTrigger(minutes=self.config.general.proper_searcher_interval.value, timezone='utc'),
name=self.proper_searcher.name,
id=self.proper_searcher.name
)
# add trakt.tv checker job
self.scheduler.add_job(
self.trakt_searcher.task,
IntervalTrigger(
hours=1,
timezone='utc'
),
name=self.trakt_searcher.name,
id=self.trakt_searcher.name
)
# add subtitles finder job
self.scheduler.add_job(
self.subtitle_searcher.task,
IntervalTrigger(
hours=self.config.general.subtitle_searcher_freq,
timezone='utc'
),
name=self.subtitle_searcher.name,
id=self.subtitle_searcher.name
)
# add upnp client job
self.scheduler.add_job(
self.upnp_client.task,
IntervalTrigger(
seconds=self.upnp_client._nat_portmap_lifetime,
timezone='utc'
),
name=self.upnp_client.name,
id=self.upnp_client.name
)
# start queues
self.search_queue.start_worker(self.config.general.max_queue_workers)
self.show_queue.start_worker(self.config.general.max_queue_workers)
self.postprocessor_queue.start_worker(self.config.general.max_queue_workers)
# start web server
self.wserver.start()
# start scheduler service
self.scheduler.start()
# perform server checkup
IOLoop.current().add_callback(self.server_checkup)
# load shows
IOLoop.current().add_callback(self.load_shows)
# perform version update check
IOLoop.current().spawn_callback(self.version_updater.check_for_update)
# load network timezones
IOLoop.current().spawn_callback(self.tz_updater.update_network_timezones)
# load search provider urls
IOLoop.current().spawn_callback(self.search_providers.update_urls)
# startup message
IOLoop.current().add_callback(self.startup_message)
# launch browser
IOLoop.current().add_callback(self.launch_browser)
# watch websocket message queue
PeriodicCallback(check_web_socket_queue, 100).start()
# perform server checkups every hour
PeriodicCallback(self.server_checkup, 1 * 60 * 60 * 1000).start()
# perform shutdown trigger check every 5 seconds
PeriodicCallback(self.shutdown_trigger, 5 * 1000).start()
# start ioloop
IOLoop.current().start()
class Core(object):
def __init__(self):
self.started = False
self.loading_shows = False
self.daemon = None
self.pid = os.getpid()
self.gui_static_dir = os.path.join(sickrage.PROG_DIR, 'core', 'webserver', 'static')
self.gui_views_dir = os.path.join(sickrage.PROG_DIR, 'core', 'webserver', 'views')
self.gui_app_dir = os.path.join(sickrage.PROG_DIR, 'core', 'webserver', 'app')
self.https_cert_file = None
self.https_key_file = None
self.trakt_api_key = '5c65f55e11d48c35385d9e8670615763a605fad28374c8ae553a7b7a50651ddd'
self.trakt_api_secret = 'b53e32045ac122a445ef163e6d859403301ffe9b17fb8321d428531b69022a82'
self.trakt_app_id = '4562'
self.fanart_api_key = '9b3afaf26f6241bdb57d6cc6bd798da7'
self.git_remote = "origin"
self.git_remote_url = "https://git.sickrage.ca/SiCKRAGE/sickrage"
self.unrar_tool = rarfile.UNRAR_TOOL
self.naming_force_folders = False
self.min_auto_postprocessor_freq = 1
self.min_daily_searcher_freq = 10
self.min_backlog_searcher_freq = 10
self.min_version_updater_freq = 1
self.min_subtitle_searcher_freq = 1
self.min_failed_snatch_age = 1
try:
self.tz = tz.tzwinlocal() if tz.tzwinlocal else tz.tzlocal()
except Exception:
self.tz = tz.tzlocal()
self.shows = {}
self.shows_recent = deque(maxlen=5)
self.main_db = None
self.cache_db = None
self.config_file = None
self.data_dir = None
self.cache_dir = None
self.quiet = None
self.no_launch = None
self.disable_updates = None
self.web_port = None
self.web_host = None
self.web_root = None
self.developer = None
self.db_type = None
self.db_prefix = None
self.db_host = None
self.db_port = None
self.db_username = None
self.db_password = None
self.debug = None
self.latest_version_string = None
self.naming_ep_type = (
"%(seasonnumber)dx%(episodenumber)02d",
"s%(seasonnumber)02de%(episodenumber)02d",
"S%(seasonnumber)02dE%(episodenumber)02d",
"%(seasonnumber)02dx%(episodenumber)02d",
"S%(seasonnumber)02d E%(episodenumber)02d"
)
self.sports_ep_type = (
"%(seasonnumber)dx%(episodenumber)02d",
"s%(seasonnumber)02de%(episodenumber)02d",
"S%(seasonnumber)02dE%(episodenumber)02d",
"%(seasonnumber)02dx%(episodenumber)02d",
"S%(seasonnumber)02 dE%(episodenumber)02d"
)
self.naming_ep_type_text = (
"1x02",
"s01e02",
"S01E02",
"01x02",
"S01 E02"
)
self.naming_multi_ep_type = {
0: ["-%(episodenumber)02d"] * len(self.naming_ep_type),
1: [" - " + x for x in self.naming_ep_type],
2: [x + "%(episodenumber)02d" for x in ("x", "e", "E", "x")]
}
self.naming_multi_ep_type_text = (
"extend",
"duplicate",
"repeat"
)
self.naming_sep_type = (
" - ",
" "
)
self.naming_sep_type_text = (
" - ",
"space"
)
self.user_agent = 'SiCKRAGE.CE.1/({};{};{})'.format(platform.system(), platform.release(), str(uuid.uuid1()))
self.languages = [language for language in os.listdir(sickrage.LOCALE_DIR) if '_' in language]
self.client_web_urls = {'torrent': '', 'newznab': ''}
self.notification_providers = {}
self.metadata_providers = {}
self.search_providers = {}
self.series_providers = {}
self.adba_connection = None
self.log = None
self.config = None
self.alerts = None
self.scheduler = None
self.wserver = None
self.google_auth = None
self.show_queue = None
self.search_queue = None
self.postprocessor_queue = None
self.version_updater = None
self.show_updater = None
self.tz_updater = None
self.rsscache_updater = None
self.daily_searcher = None
self.failed_snatch_searcher = None
self.backlog_searcher = None
self.proper_searcher = None
self.trakt_searcher = None
self.subtitle_searcher = None
self.auto_postprocessor = None
self.upnp_client = None
self.auth_server = None
self.announcements = None
self.api = None
self.amqp_consumer = None
def start(self):
self.started = True
# thread name
threading.currentThread().setName('CORE')
# init sentry
self.init_sentry()
# scheduler
self.scheduler = TornadoScheduler({'apscheduler.timezone': 'UTC'})
# init core classes
self.api = API()
self.config = Config(self.db_type, self.db_prefix, self.db_host, self.db_port, self.db_username, self.db_password)
self.main_db = MainDB(self.db_type, self.db_prefix, self.db_host, self.db_port, self.db_username, self.db_password)
self.cache_db = CacheDB(self.db_type, self.db_prefix, self.db_host, self.db_port, self.db_username, self.db_password)
self.notification_providers = NotificationProviders()
self.metadata_providers = MetadataProviders()
self.search_providers = SearchProviders()
self.series_providers = SeriesProviders()
self.log = Logger()
self.alerts = Notifications()
self.wserver = WebServer()
self.show_queue = ShowQueue()
self.search_queue = SearchQueue()
self.postprocessor_queue = PostProcessorQueue()
self.version_updater = VersionUpdater()
self.show_updater = ShowUpdater()
self.tz_updater = TimeZoneUpdater()
self.rsscache_updater = RSSCacheUpdater()
self.daily_searcher = DailySearcher()
self.failed_snatch_searcher = FailedSnatchSearcher()
self.backlog_searcher = BacklogSearcher()
self.proper_searcher = ProperSearcher()
self.trakt_searcher = TraktSearcher()
self.subtitle_searcher = SubtitleSearcher()
self.auto_postprocessor = AutoPostProcessor()
self.upnp_client = UPNPClient()
self.announcements = Announcements()
self.amqp_consumer = AMQPConsumer()
# authorization sso client
self.auth_server = AuthServer()
# check available space
try:
self.log.info("Performing disk space checks")
total_space, available_space = get_free_space(self.data_dir)
if available_space < 100:
self.log.warning('Shutting down as SiCKRAGE needs some space to work. You\'ll get corrupted data otherwise. Only %sMB left', available_space)
return
except Exception:
self.log.error('Failed getting disk space: %s', traceback.format_exc())
# check if we need to perform a restore first
if os.path.exists(os.path.abspath(os.path.join(self.data_dir, 'restore'))):
self.log.info('Performing restore of backup files')
success = restore_app_data(os.path.abspath(os.path.join(self.data_dir, 'restore')), self.data_dir)
self.log.info("Restoring SiCKRAGE backup: %s!" % ("FAILED", "SUCCESSFUL")[success])
if success:
# remove restore files
shutil.rmtree(os.path.abspath(os.path.join(self.data_dir, 'restore')), ignore_errors=True)
# migrate old database file names to new ones
if os.path.isfile(os.path.abspath(os.path.join(self.data_dir, 'sickbeard.db'))):
if os.path.isfile(os.path.join(self.data_dir, 'sickrage.db')):
helpers.move_file(os.path.join(self.data_dir, 'sickrage.db'),
os.path.join(self.data_dir, '{}.bak-{}'
.format('sickrage.db',
datetime.datetime.now().strftime(
'%Y%m%d_%H%M%S'))))
helpers.move_file(os.path.abspath(os.path.join(self.data_dir, 'sickbeard.db')),
os.path.abspath(os.path.join(self.data_dir, 'sickrage.db')))
# setup databases
self.main_db.setup()
self.config.db.setup()
self.cache_db.setup()
# load config
self.config.load()
# migrate config
self.config.migrate_config_file(self.config_file)
# add server id tag to sentry
sentry_sdk.set_tag('server_id', self.config.general.server_id)
# add user to sentry
sentry_sdk.set_user({
'id': self.config.user.sub_id,
'username': self.config.user.username,
'email': self.config.user.email
})
# config overrides
if self.web_port:
self.config.general.web_port = self.web_port
if self.web_root:
self.config.general.web_root = self.web_root
# set language
change_gui_lang(self.config.gui.gui_lang)
# set socket timeout
socket.setdefaulttimeout(self.config.general.socket_timeout)
# set ssl cert/key filenames
self.https_cert_file = os.path.abspath(os.path.join(self.data_dir, 'server.crt'))
self.https_key_file = os.path.abspath(os.path.join(self.data_dir, 'server.key'))
# setup logger settings
self.log.logSize = self.config.general.log_size
self.log.logNr = self.config.general.log_nr
self.log.logFile = os.path.join(self.data_dir, 'logs', 'sickrage.log')
self.log.debugLogging = self.debug or self.config.general.debug
self.log.consoleLogging = not self.quiet
# start logger
self.log.start()
# user agent
if self.config.general.random_user_agent:
self.user_agent = UserAgent().random
uses_netloc.append('scgi')
FancyURLopener.version = self.user_agent
# set torrent client web url
torrent_webui_url(True)
if self.config.general.default_page not in DefaultHomePage:
self.config.general.default_page = DefaultHomePage.HOME
# attempt to help prevent users from breaking links by using a bad url
if not self.config.general.anon_redirect.endswith('?'):
self.config.general.anon_redirect = ''
if not re.match(r'\d+\|[^|]+(?:\|[^|]+)*', self.config.general.root_dirs):
self.config.general.root_dirs = ''
self.naming_force_folders = check_force_season_folders()
if self.config.general.nzb_method not in NzbMethod:
self.config.general.nzb_method = NzbMethod.BLACKHOLE
if self.config.general.torrent_method not in TorrentMethod:
self.config.general.torrent_method = TorrentMethod.BLACKHOLE
if self.config.general.auto_postprocessor_freq < self.min_auto_postprocessor_freq:
self.config.general.auto_postprocessor_freq = self.min_auto_postprocessor_freq
if self.config.general.daily_searcher_freq < self.min_daily_searcher_freq:
self.config.general.daily_searcher_freq = self.min_daily_searcher_freq
if self.config.general.backlog_searcher_freq < self.min_backlog_searcher_freq:
self.config.general.backlog_searcher_freq = self.min_backlog_searcher_freq
if self.config.general.version_updater_freq < self.min_version_updater_freq:
self.config.general.version_updater_freq = self.min_version_updater_freq
if self.config.general.subtitle_searcher_freq < self.min_subtitle_searcher_freq:
self.config.general.subtitle_searcher_freq = self.min_subtitle_searcher_freq
if self.config.failed_snatches.age < self.min_failed_snatch_age:
self.config.failed_snatches.age = self.min_failed_snatch_age
if self.config.general.proper_searcher_interval not in CheckPropersInterval:
self.config.general.proper_searcher_interval = CheckPropersInterval.DAILY
if self.config.general.show_update_hour < 0 or self.config.general.show_update_hour > 23:
self.config.general.show_update_hour = 0
# add app updater job
self.scheduler.add_job(
self.version_updater.task,
IntervalTrigger(
hours=1,
start_date=datetime.datetime.now() + datetime.timedelta(minutes=4),
timezone='utc'
),
name=self.version_updater.name,
id=self.version_updater.name
)
# add show updater job
self.scheduler.add_job(
self.show_updater.task,
IntervalTrigger(
days=1,
start_date=datetime.datetime.now().replace(hour=self.config.general.show_update_hour),
timezone='utc'
),
name=self.show_updater.name,
id=self.show_updater.name
)
# add rss cache updater job
self.scheduler.add_job(
self.rsscache_updater.task,
IntervalTrigger(
minutes=15,
timezone='utc'
),
name=self.rsscache_updater.name,
id=self.rsscache_updater.name
)
# add daily search job
self.scheduler.add_job(
self.daily_searcher.task,
IntervalTrigger(
minutes=self.config.general.daily_searcher_freq,
start_date=datetime.datetime.now() + datetime.timedelta(minutes=4),
timezone='utc'
),
name=self.daily_searcher.name,
id=self.daily_searcher.name
)
# add failed snatch search job
self.scheduler.add_job(
self.failed_snatch_searcher.task,
IntervalTrigger(
hours=1,
start_date=datetime.datetime.now() + datetime.timedelta(minutes=4),
timezone='utc'
),
name=self.failed_snatch_searcher.name,
id=self.failed_snatch_searcher.name
)
# add backlog search job
self.scheduler.add_job(
self.backlog_searcher.task,
IntervalTrigger(
minutes=self.config.general.backlog_searcher_freq,
start_date=datetime.datetime.now() + datetime.timedelta(minutes=30),
timezone='utc'
),
name=self.backlog_searcher.name,
id=self.backlog_searcher.name
)
# add auto-postprocessing job
self.scheduler.add_job(
self.auto_postprocessor.task,
IntervalTrigger(
minutes=self.config.general.auto_postprocessor_freq,
timezone='utc'
),
name=self.auto_postprocessor.name,
id=self.auto_postprocessor.name
)
# add find proper job
self.scheduler.add_job(
self.proper_searcher.task,
IntervalTrigger(minutes=self.config.general.proper_searcher_interval.value, timezone='utc'),
name=self.proper_searcher.name,
id=self.proper_searcher.name
)
# add trakt.tv checker job
self.scheduler.add_job(
self.trakt_searcher.task,
IntervalTrigger(
hours=1,
timezone='utc'
),
name=self.trakt_searcher.name,
id=self.trakt_searcher.name
)
# add subtitles finder job
self.scheduler.add_job(
self.subtitle_searcher.task,
IntervalTrigger(
hours=self.config.general.subtitle_searcher_freq,
timezone='utc'
),
name=self.subtitle_searcher.name,
id=self.subtitle_searcher.name
)
# add upnp client job
self.scheduler.add_job(
self.upnp_client.task,
IntervalTrigger(
seconds=self.upnp_client._nat_portmap_lifetime,
timezone='utc'
),
name=self.upnp_client.name,
id=self.upnp_client.name
)
# start queues
self.search_queue.start_worker(self.config.general.max_queue_workers)
self.show_queue.start_worker(self.config.general.max_queue_workers)
self.postprocessor_queue.start_worker(self.config.general.max_queue_workers)
# start web server
self.wserver.start()
# start scheduler service
self.scheduler.start()
# perform server checkup
IOLoop.current().add_callback(self.server_checkup)
# load shows
IOLoop.current().add_callback(self.load_shows)
# perform version update check
IOLoop.current().spawn_callback(self.version_updater.check_for_update)
# load network timezones
IOLoop.current().spawn_callback(self.tz_updater.update_network_timezones)
# load search provider urls
IOLoop.current().spawn_callback(self.search_providers.update_urls)
# startup message
IOLoop.current().add_callback(self.startup_message)
# launch browser
IOLoop.current().add_callback(self.launch_browser)
# watch websocket message queue
PeriodicCallback(check_web_socket_queue, 100).start()
# perform server checkups every hour
PeriodicCallback(self.server_checkup, 1 * 60 * 60 * 1000).start()
# perform shutdown trigger check every 5 seconds
PeriodicCallback(self.shutdown_trigger, 5 * 1000).start()
# start ioloop
IOLoop.current().start()
def init_sentry(self):
# sentry log handler
sentry_logging = LoggingIntegration(
level=logging.INFO, # Capture info and above as breadcrumbs
event_level=logging.ERROR # Send errors as events
)
# init sentry logging
sentry_sdk.init(
dsn="https://[email protected]/2?verify_ssl=0",
integrations=[sentry_logging],
release=sickrage.version(),
environment=('master', 'develop')['dev' in sickrage.version()],
ignore_errors=[
'KeyboardInterrupt',
'PermissionError',
'FileNotFoundError',
'EpisodeNotFoundException'
]
)
# sentry tags
sentry_tags = {
'platform': platform.platform(),
'locale': repr(locale.getdefaultlocale()),
'python': platform.python_version(),
'install_type': sickrage.install_type()
}
# set sentry tags
for tag_key, tag_value in sentry_tags.items():
sentry_sdk.set_tag(tag_key, tag_value)
# set loggers to ignore
ignored_loggers = [
'enzyme.parsers.ebml.core',
'subliminal.core',
'subliminal.utils',
'subliminal.refiners.tvdb',
'subliminal.refiners.metadata',
'subliminal.providers.tvsubtitles',
'pika.connection',
'pika.adapters.base_connection',
'pika.adapters.utils.io_services_utils',
'pika.adapters.utils.connection_workflow',
'pika.adapters.utils.selector_ioloop_adapter'
]
for item in ignored_loggers:
ignore_logger(item)
def server_checkup(self):
if self.config.general.server_id:
server_status = self.api.server.get_status(self.config.general.server_id)
if server_status and not server_status['registered']:
# re-register server
server_id = self.api.server.register_server(
ip_addresses=','.join([get_internal_ip()]),
web_protocol=('http', 'https')[self.config.general.enable_https],
web_port=self.config.general.web_port,
web_root=self.config.general.web_root,
server_version=sickrage.version(),
)
if server_id:
self.log.info('Re-registered SiCKRAGE server with SiCKRAGE API')
sentry_sdk.set_tag('server_id', self.config.general.server_id)
self.config.general.server_id = server_id
self.config.save(mark_dirty=True)
else:
self.log.debug('Updating SiCKRAGE server data on SiCKRAGE API')
# update server information
self.api.server.update_server(
server_id=self.config.general.server_id,
ip_addresses=','.join([get_internal_ip()]),
web_protocol=('http', 'https')[self.config.general.enable_https],
web_port=self.config.general.web_port,
web_root=self.config.general.web_root,
server_version=sickrage.version(),
)
def load_shows(self):
threading.currentThread().setName('CORE')
session = self.main_db.session()
self.log.info('Loading initial shows list')
self.loading_shows = True
self.shows = {}
for query in session.query(MainDB.TVShow).with_entities(MainDB.TVShow.series_id, MainDB.TVShow.series_provider_id, MainDB.TVShow.name,
MainDB.TVShow.location):
try:
# if not os.path.isdir(query.location) and self.config.general.create_missing_show_dirs:
# make_dir(query.location)
self.log.info('Loading show {}'.format(query.name))
self.shows.update({(query.series_id, query.series_provider_id): TVShow(query.series_id, query.series_provider_id)})
except Exception as e:
self.log.debug('There was an error loading show: {}'.format(query.name))
self.loading_shows = False
self.log.info('Loading initial shows list finished')
def startup_message(self):
self.log.info("SiCKRAGE :: STARTED")
self.log.info(f"SiCKRAGE :: APP VERSION:[{sickrage.version()}]")
self.log.info(f"SiCKRAGE :: CONFIG VERSION:[v{self.config.db.version}]")
self.log.info(f"SiCKRAGE :: DATABASE VERSION:[v{self.main_db.version}]")
self.log.info(f"SiCKRAGE :: DATABASE TYPE:[{self.db_type}]")
self.log.info(f"SiCKRAGE :: INSTALL TYPE:[{self.version_updater.updater.type}]")
self.log.info(
f"SiCKRAGE :: URL:[{('http', 'https')[self.config.general.enable_https]}://{(get_internal_ip(), self.web_host)[self.web_host not in ['', '0.0.0.0']]}:{self.config.general.web_port}/{self.config.general.web_root.lstrip('/')}]")
def launch_browser(self):
if not self.no_launch and self.config.general.launch_browser:
launch_browser(protocol=('http', 'https')[self.config.general.enable_https],
host=(get_internal_ip(), self.web_host)[self.web_host != ''],
startport=self.config.general.web_port)
def shutdown(self, restart=False):
if self.started:
self.log.info('SiCKRAGE IS {}!!!'.format(('SHUTTING DOWN', 'RESTARTING')[restart]))
# shutdown scheduler
if self.scheduler:
try:
self.scheduler.shutdown()
except (SchedulerNotRunningError, RuntimeError):
pass
# shutdown webserver
if self.wserver:
self.wserver.shutdown()
# stop queues
self.search_queue.shutdown()
self.show_queue.shutdown()
self.postprocessor_queue.shutdown()
# stop amqp consumer
self.amqp_consumer.stop()
# log out of ADBA
if self.adba_connection:
self.log.debug("Shutting down ANIDB connection")
self.adba_connection.stop()
# save shows
self.log.info('Saving all shows to the database')
for show in self.shows.values():
show.save()
# save settings
self.config.save()
# shutdown logging
if self.log:
self.log.close()
if restart:
os.execl(sys.executable, sys.executable, *sys.argv)
if self.daemon:
self.daemon.stop()
self.started = False
def restart(self):
self.shutdown(restart=True)
def shutdown_trigger(self):
if not self.started:
IOLoop.current().stop()
Demonstrates how to use the Tornado compatible scheduler to schedule a job that executes on 3
second intervals.
"""
from datetime import datetime
import os
from tornado.ioloop import IOLoop, PeriodicCallback
from apscheduler.schedulers.tornado import TornadoScheduler
def tick():
print('Tick! The time is: %s' % datetime.now())
def job():
print('Every Day Job! The time is: %s' % datetime.now())
if __name__ == '__main__':
scheduler = TornadoScheduler()
scheduler.add_job(tick, 'interval', seconds=3)
scheduler.add_job(job, 'cron', hour=10, minute=25)
scheduler.start()
PeriodicCallback(job, 2 * 1000).start()
print('Press Ctrl+{0} to exit'.format('Break' if os.name == 'nt' else 'C'))
try:
IOLoop.instance().start()
except (KeyboardInterrupt, SystemExit):
pass
class MurakamiServer:
"""
*MurakamiServer* is responsible for loading all test runner and result
exporter plugins, as well as managing the main eventloop and loading
WebThings if desired.
####Keyword arguments:
* `port`: port that WebThingsServer listens on
* `hostname`: this WebThingServer's hostname
* `ssl_options`: any TLS options to supply to WebThingServer
* `additonal_routes`: routes to add to the WebThingServer
* `base_path`: path to add to URL where we're listening in case we're
behind a proxy
* `tests_per_day`: number of tests to run in a day
* `location`: string describing physical location of this device
* `network_type`: string describing the network this device is connected to
* `connection_type`: string describing type of connection this device is
using
"""
def __init__(
self,
port=defaults.HTTP_PORT,
hostname=None,
ssl_options=None,
additional_routes=None,
base_path="",
tests_per_day=defaults.TESTS_PER_DAY,
immediate=False,
webthings=False,
location=None,
network_type=None,
connection_type=None,
device_id=None,
config=None,
):
self._runners = {}
self._exporters = {}
self._scheduler = None
self._server = None
self._port = port
self._hostname = hostname
self._ssl_options = ssl_options
self._additional_routes = additional_routes
self._base_path = base_path
self._tests_per_day = tests_per_day
self._immediate = immediate
self._webthings = webthings
self._location = location
self._network_type = network_type
self._connection_type = connection_type
self._device_id = device_id
self._config = config
def _call_runners(self):
for r in self._runners.values():
_logger.info("Running test: %s", r.title)
try:
r.start_test()
except Exception as exc:
_logger.error("Failed to run test %s: %s", r.title, str(exc))
def _call_exporters(self, test_name="", data="", timestamp=None):
for e in self._exporters.values():
_logger.info("Running exporter %s for test %s", e.name, test_name)
try:
e.push(test_name, data, timestamp)
except Exception as exc:
_logger.error("Failed to run exporter %s: %s", e.name,
str(exc))
def _load_runners(self):
trigger = RandomTrigger(tests_per_day=self._tests_per_day,
immediate=self._immediate)
# Load test runners
for entry_point in pkg_resources.iter_entry_points("murakami.runners"):
logging.debug("Loading test runner %s", entry_point.name)
if "tests" not in self._config:
self._config["tests"] = {}
if entry_point.name not in self._config["tests"]:
self._config["tests"][entry_point.name] = {"enabled": False}
self._runners[entry_point.name] = entry_point.load()(
config=self._config["tests"][entry_point.name],
data_cb=self._call_exporters,
location=self._location,
network_type=self._network_type,
connection_type=self._connection_type,
device_id=self._device_id,
)
# Start webthings server if enabled
if self._webthings:
self._server = WebThingServer(
SingleThing(MurakamiThing(self._runners.values())),
port=self._port,
hostname=self._hostname,
ssl_options=self._ssl_options,
additional_routes=self._additional_routes,
base_path=self._base_path,
)
# Start test scheduler if enabled
if self._tests_per_day > 0:
self._scheduler = TornadoScheduler()
self._scheduler.add_job(self._call_runners,
id="runners",
name="Test Runners",
trigger=trigger)
def _load_exporters(self):
self._exporters = {}
# Check if exporters are enabled and load them.
if "exporters" in self._config:
exporters = pkg_resources.get_entry_map("murakami",
group="murakami.exporters")
for name, entry in self._config["exporters"].items():
logging.debug("Loading exporter %s", name)
enabled = True
if "enabled" in entry:
enabled = utils.is_enabled(entry["enabled"])
if enabled:
if "type" in entry:
if entry["type"] in exporters:
self._exporters[name] = exporters[
entry["type"]].load()(
name=name,
location=self._location,
network_type=self._network_type,
connection_type=self._connection_type,
config=entry,
)
else:
logging.error(
"No available exporter type %s, skipping.",
entry["type"],
)
else:
logging.error(
"No type defined for exporter %s, skipping.", name)
else:
logging.debug("Exporter %s disabled, skipping.", name)
def start(self):
"""Start MurakamiServer, including WebThingServer if directed."""
_logger.info("Starting Murakami services.")
self._load_runners()
self._load_exporters()
if self._scheduler is not None:
_logger.info("Starting the job scheduler.")
self._scheduler.start()
if self._server is not None:
_logger.info("Starting the WebThing server.")
self._server.start()
if self._scheduler is not None and self._server is None:
IOLoop.current().start()
def stop(self):
"""Stop MurakamiServer."""
_logger.info("Stopping Murakami services.")
IOLoop.current().stop()
if self._scheduler is not None:
_logger.info("Stopping the job scheduler.")
self._scheduler.shutdown()
if self._server is not None:
_logger.info("Stopping the WebThing server.")
self._server.stop()
_logger.info("Cleaning up test runners.")
for r in self._runners:
self._runners[r].stop_test()
self._runners[r].teardown()
@gen.coroutine
def reload(self, signum, frame, **kwargs):
"""Reload MurakamiServer, to be called as an event handler."""
local_args = dict(locals())
_logger.info("Reloading Murakami services...")
for key, _ in local_args.items():
if key in kwargs:
setattr(self, key, kwargs[key])
yield IOLoop.current().add_callback_from_signal(self.stop)
self.start()
from __future__ import absolute_import, unicode_literals
from datetime import datetime, date, time, timedelta
from apscheduler.schedulers.tornado import TornadoScheduler
from apscheduler.triggers.interval import IntervalTrigger
import tornado.ioloop
from mylogger.mylogger import logger
import logging
def testfunc(a, b, c):
logger.info('dongzhuoyao DOING: %s' % locals())
if __name__ == '__main__':
sched = TornadoScheduler()
sched.add_job(
func=testfunc,
args=(1, 2, 3),
trigger=IntervalTrigger(
start_date=datetime.combine(date.today(), time.max), #today 23:59
end_date=date.today() +
timedelta(days=1e3), #datetime.now() + timedelta(days=1),
days=1,
),
name='testfunc')
sched.start()
try:
tornado.ioloop.IOLoop.current().start()
except (KeyboardInterrupt, SystemExit) as e:
class Core(object):
def __init__(self):
self.started = False
self.daemon = None
self.io_loop = None
self.pid = os.getpid()
self.showlist = []
try:
self.tz = tz.tzwinlocal() if tz.tzwinlocal else tz.tzlocal()
except Exception:
self.tz = tz.tzlocal()
self.config_file = None
self.data_dir = None
self.cache_dir = None
self.quiet = None
self.no_launch = None
self.web_port = None
self.developer = None
self.debug = None
self.newest_version_string = None
self.naming_ep_type = ("%(seasonnumber)dx%(episodenumber)02d",
"s%(seasonnumber)02de%(episodenumber)02d",
"S%(seasonnumber)02dE%(episodenumber)02d",
"%(seasonnumber)02dx%(episodenumber)02d",
"S%(seasonnumber)02d E%(episodenumber)02d")
self.sports_ep_type = ("%(seasonnumber)dx%(episodenumber)02d",
"s%(seasonnumber)02de%(episodenumber)02d",
"S%(seasonnumber)02dE%(episodenumber)02d",
"%(seasonnumber)02dx%(episodenumber)02d",
"S%(seasonnumber)02 dE%(episodenumber)02d")
self.naming_ep_type_text = (
"1x02",
"s01e02",
"S01E02",
"01x02",
"S01 E02",
)
self.naming_multi_ep_type = {
0: ["-%(episodenumber)02d"] * len(self.naming_ep_type),
1: [" - " + x for x in self.naming_ep_type],
2: [x + "%(episodenumber)02d" for x in ("x", "e", "E", "x")]
}
self.naming_multi_ep_type_text = ("extend", "duplicate", "repeat")
self.naming_sep_type = (" - ", " ")
self.naming_sep_type_text = (" - ", "space")
self.user_agent = 'SiCKRAGE.CE.1/({};{};{})'.format(
platform.system(), platform.release(), str(uuid.uuid1()))
self.languages = [
language for language in os.listdir(sickrage.LOCALE_DIR)
if '_' in language
]
self.sys_encoding = get_sys_encoding()
self.client_web_urls = {'torrent': '', 'newznab': ''}
self.adba_connection = None
self.notifier_providers = None
self.metadata_providers = {}
self.search_providers = None
self.log = None
self.config = None
self.alerts = None
self.main_db = None
self.cache_db = None
self.scheduler = None
self.wserver = None
self.google_auth = None
self.name_cache = None
self.show_queue = None
self.search_queue = None
self.postprocessor_queue = None
self.event_queue = None
self.version_updater = None
self.show_updater = None
self.tz_updater = None
self.rsscache_updater = None
self.daily_searcher = None
self.backlog_searcher = None
self.proper_searcher = None
self.trakt_searcher = None
self.subtitle_searcher = None
self.auto_postprocessor = None
self.upnp_client = None
self.oidc_client = None
self.quicksearch_cache = None
def start(self):
self.started = True
self.io_loop = IOLoop.current()
# thread name
threading.currentThread().setName('CORE')
# patch modules with encoding kludge
patch_modules()
# init core classes
self.notifier_providers = NotifierProviders()
self.metadata_providers = MetadataProviders()
self.search_providers = SearchProviders()
self.log = Logger()
self.config = Config()
self.alerts = Notifications()
self.main_db = MainDB()
self.cache_db = CacheDB()
self.scheduler = TornadoScheduler()
self.wserver = WebServer()
self.name_cache = NameCache()
self.show_queue = ShowQueue()
self.search_queue = SearchQueue()
self.postprocessor_queue = PostProcessorQueue()
self.event_queue = EventQueue()
self.version_updater = VersionUpdater()
self.show_updater = ShowUpdater()
self.tz_updater = TimeZoneUpdater()
self.rsscache_updater = RSSCacheUpdater()
self.daily_searcher = DailySearcher()
self.failed_snatch_searcher = FailedSnatchSearcher()
self.backlog_searcher = BacklogSearcher()
self.proper_searcher = ProperSearcher()
self.trakt_searcher = TraktSearcher()
self.subtitle_searcher = SubtitleSearcher()
self.auto_postprocessor = AutoPostProcessor()
self.upnp_client = UPNPClient()
self.quicksearch_cache = QuicksearchCache()
# setup oidc client
realm = KeycloakRealm(server_url='https://auth.sickrage.ca',
realm_name='sickrage')
self.oidc_client = realm.open_id_connect(
client_id='sickrage-app',
client_secret='5d4710b2-ca70-4d39-b5a3-0705e2c5e703')
# Check if we need to perform a restore first
if os.path.exists(
os.path.abspath(os.path.join(self.data_dir, 'restore'))):
success = restoreSR(
os.path.abspath(os.path.join(self.data_dir, 'restore')),
self.data_dir)
print("Restoring SiCKRAGE backup: %s!\n" %
("FAILED", "SUCCESSFUL")[success])
if success:
shutil.rmtree(os.path.abspath(
os.path.join(self.data_dir, 'restore')),
ignore_errors=True)
# migrate old database file names to new ones
if os.path.isfile(
os.path.abspath(os.path.join(self.data_dir, 'sickbeard.db'))):
if os.path.isfile(os.path.join(self.data_dir, 'sickrage.db')):
helpers.move_file(
os.path.join(self.data_dir, 'sickrage.db'),
os.path.join(
self.data_dir, '{}.bak-{}'.format(
'sickrage.db',
datetime.datetime.now().strftime(
'%Y%m%d_%H%M%S'))))
helpers.move_file(
os.path.abspath(os.path.join(self.data_dir, 'sickbeard.db')),
os.path.abspath(os.path.join(self.data_dir, 'sickrage.db')))
# load config
self.config.load()
# set language
self.config.change_gui_lang(self.config.gui_lang)
# set socket timeout
socket.setdefaulttimeout(self.config.socket_timeout)
# setup logger settings
self.log.logSize = self.config.log_size
self.log.logNr = self.config.log_nr
self.log.logFile = os.path.join(self.data_dir, 'logs', 'sickrage.log')
self.log.debugLogging = self.config.debug
self.log.consoleLogging = not self.quiet
# start logger
self.log.start()
# user agent
if self.config.random_user_agent:
self.user_agent = UserAgent().random
urlparse.uses_netloc.append('scgi')
urllib.FancyURLopener.version = self.user_agent
# set torrent client web url
torrent_webui_url(True)
# Check available space
try:
total_space, available_space = getFreeSpace(self.data_dir)
if available_space < 100:
self.log.error(
'Shutting down as SiCKRAGE needs some space to work. You\'ll get corrupted data '
'otherwise. Only %sMB left', available_space)
return
except Exception:
self.log.error('Failed getting disk space: %s',
traceback.format_exc())
# perform database startup actions
for db in [self.main_db, self.cache_db]:
# initialize database
db.initialize()
# check integrity of database
db.check_integrity()
# migrate database
db.migrate()
# misc database cleanups
db.cleanup()
# upgrade database
db.upgrade()
# compact main database
if self.config.last_db_compact < time.time() - 604800: # 7 days
self.main_db.compact()
self.config.last_db_compact = int(time.time())
# load name cache
self.name_cache.load()
# load data for shows from database
self.load_shows()
if self.config.default_page not in ('schedule', 'history', 'IRC'):
self.config.default_page = 'home'
# cleanup cache folder
for folder in ['mako', 'sessions', 'indexers']:
try:
shutil.rmtree(os.path.join(sickrage.app.cache_dir, folder),
ignore_errors=True)
except Exception:
continue
if self.config.web_port < 21 or self.config.web_port > 65535:
self.config.web_port = 8081
if not self.config.web_cookie_secret:
self.config.web_cookie_secret = generate_secret()
# attempt to help prevent users from breaking links by using a bad url
if not self.config.anon_redirect.endswith('?'):
self.config.anon_redirect = ''
if not re.match(r'\d+\|[^|]+(?:\|[^|]+)*', self.config.root_dirs):
self.config.root_dirs = ''
self.config.naming_force_folders = check_force_season_folders()
if self.config.nzb_method not in ('blackhole', 'sabnzbd', 'nzbget'):
self.config.nzb_method = 'blackhole'
if self.config.torrent_method not in ('blackhole', 'utorrent',
'transmission', 'deluge',
'deluged', 'download_station',
'rtorrent', 'qbittorrent',
'mlnet', 'putio'):
self.config.torrent_method = 'blackhole'
if self.config.autopostprocessor_freq < self.config.min_autopostprocessor_freq:
self.config.autopostprocessor_freq = self.config.min_autopostprocessor_freq
if self.config.daily_searcher_freq < self.config.min_daily_searcher_freq:
self.config.daily_searcher_freq = self.config.min_daily_searcher_freq
if self.config.backlog_searcher_freq < self.config.min_backlog_searcher_freq:
self.config.backlog_searcher_freq = self.config.min_backlog_searcher_freq
if self.config.version_updater_freq < self.config.min_version_updater_freq:
self.config.version_updater_freq = self.config.min_version_updater_freq
if self.config.subtitle_searcher_freq < self.config.min_subtitle_searcher_freq:
self.config.subtitle_searcher_freq = self.config.min_subtitle_searcher_freq
if self.config.failed_snatch_age < self.config.min_failed_snatch_age:
self.config.failed_snatch_age = self.config.min_failed_snatch_age
if self.config.proper_searcher_interval not in ('15m', '45m', '90m',
'4h', 'daily'):
self.config.proper_searcher_interval = 'daily'
if self.config.showupdate_hour < 0 or self.config.showupdate_hour > 23:
self.config.showupdate_hour = 0
# add version checker job
self.scheduler.add_job(self.version_updater.run,
IntervalTrigger(
hours=self.config.version_updater_freq, ),
name=self.version_updater.name,
id=self.version_updater.name)
# add network timezones updater job
self.scheduler.add_job(self.tz_updater.run,
IntervalTrigger(days=1, ),
name=self.tz_updater.name,
id=self.tz_updater.name)
# add show updater job
self.scheduler.add_job(self.show_updater.run,
IntervalTrigger(
days=1,
start_date=datetime.datetime.now().replace(
hour=self.config.showupdate_hour)),
name=self.show_updater.name,
id=self.show_updater.name)
# add rss cache updater job
self.scheduler.add_job(self.rsscache_updater.run,
IntervalTrigger(minutes=15, ),
name=self.rsscache_updater.name,
id=self.rsscache_updater.name)
# add daily search job
self.scheduler.add_job(
self.daily_searcher.run,
IntervalTrigger(minutes=self.config.daily_searcher_freq,
start_date=datetime.datetime.now() +
datetime.timedelta(minutes=4)),
name=self.daily_searcher.name,
id=self.daily_searcher.name)
# add failed snatch search job
self.scheduler.add_job(
self.failed_snatch_searcher.run,
IntervalTrigger(hours=1,
start_date=datetime.datetime.now() +
datetime.timedelta(minutes=4)),
name=self.failed_snatch_searcher.name,
id=self.failed_snatch_searcher.name)
# add backlog search job
self.scheduler.add_job(
self.backlog_searcher.run,
IntervalTrigger(minutes=self.config.backlog_searcher_freq,
start_date=datetime.datetime.now() +
datetime.timedelta(minutes=30)),
name=self.backlog_searcher.name,
id=self.backlog_searcher.name)
# add auto-postprocessing job
self.scheduler.add_job(
self.auto_postprocessor.run,
IntervalTrigger(minutes=self.config.autopostprocessor_freq),
name=self.auto_postprocessor.name,
id=self.auto_postprocessor.name)
# add find proper job
self.scheduler.add_job(
self.proper_searcher.run,
IntervalTrigger(minutes={
'15m': 15,
'45m': 45,
'90m': 90,
'4h': 4 * 60,
'daily': 24 * 60
}[self.config.proper_searcher_interval]),
name=self.proper_searcher.name,
id=self.proper_searcher.name)
# add trakt.tv checker job
self.scheduler.add_job(self.trakt_searcher.run,
IntervalTrigger(hours=1),
name=self.trakt_searcher.name,
id=self.trakt_searcher.name)
# add subtitles finder job
self.scheduler.add_job(
self.subtitle_searcher.run,
IntervalTrigger(hours=self.config.subtitle_searcher_freq),
name=self.subtitle_searcher.name,
id=self.subtitle_searcher.name)
# add upnp client job
self.scheduler.add_job(
self.upnp_client.run,
IntervalTrigger(seconds=self.upnp_client._nat_portmap_lifetime),
name=self.upnp_client.name,
id=self.upnp_client.name)
# add namecache update job
self.scheduler.add_job(self.name_cache.build_all,
IntervalTrigger(days=1, ),
name=self.name_cache.name,
id=self.name_cache.name)
# start scheduler service
self.scheduler.start()
# start queue's
self.search_queue.start()
self.show_queue.start()
self.postprocessor_queue.start()
self.event_queue.start()
# fire off startup events
self.event_queue.fire_event(self.name_cache.build_all)
self.event_queue.fire_event(self.version_updater.run)
self.event_queue.fire_event(self.tz_updater.run)
# start webserver
self.wserver.start()
# launch browser window
if all(
[not sickrage.app.no_launch, sickrage.app.config.launch_browser]):
self.event_queue.fire_event(lambda: launch_browser(
('http', 'https')[sickrage.app.config.enable_https], sickrage.
app.config.web_host, sickrage.app.config.web_port))
# start ioloop
self.io_loop.start()
def shutdown(self, restart=False):
if self.started:
self.log.info('SiCKRAGE IS SHUTTING DOWN!!!')
# shutdown webserver
if self.wserver:
self.wserver.shutdown()
# shutdown show queue
if self.show_queue:
self.log.debug("Shutting down show queue")
self.show_queue.shutdown()
del self.show_queue
# shutdown search queue
if self.search_queue:
self.log.debug("Shutting down search queue")
self.search_queue.shutdown()
del self.search_queue
# shutdown post-processor queue
if self.postprocessor_queue:
self.log.debug("Shutting down post-processor queue")
self.postprocessor_queue.shutdown()
del self.postprocessor_queue
# shutdown event queue
if self.event_queue:
self.log.debug("Shutting down event queue")
self.event_queue.shutdown()
del self.event_queue
# log out of ADBA
if self.adba_connection:
self.log.debug("Shutting down ANIDB connection")
self.adba_connection.stop()
# save all show and config settings
self.save_all()
# close databases
for db in [self.main_db, self.cache_db]:
if db.opened:
self.log.debug(
"Shutting down {} database connection".format(db.name))
db.close()
# shutdown logging
if self.log:
self.log.close()
if restart:
os.execl(sys.executable, sys.executable, *sys.argv)
if sickrage.app.daemon:
sickrage.app.daemon.stop()
self.started = False
if self.io_loop:
self.io_loop.stop()
def save_all(self):
# write all shows
self.log.info("Saving all shows to the database")
for show in self.showlist:
try:
show.save_to_db()
except Exception:
continue
# save config
self.config.save()
def load_shows(self):
"""
Populates the showlist and quicksearch cache with shows and episodes from the database
"""
self.quicksearch_cache.load()
for dbData in self.main_db.all('tv_shows'):
show = TVShow(int(dbData['indexer']), int(dbData['indexer_id']))
try:
self.log.debug("Loading data for show: [{}]".format(show.name))
self.showlist.append(show)
self.quicksearch_cache.add_show(show.indexerid)
except Exception as e:
self.log.debug("Show error in [%s]: %s" %
(show.location, str(e)))
class Core(object):
def __init__(self):
self.started = False
self.loading_shows = False
self.daemon = None
self.io_loop = None
self.pid = os.getpid()
try:
self.tz = tz.tzwinlocal() if tz.tzwinlocal else tz.tzlocal()
except Exception:
self.tz = tz.tzlocal()
self.shows = {}
self.private_key = None
self.public_key = None
self.main_db = None
self.cache_db = None
self.config_file = None
self.data_dir = None
self.cache_dir = None
self.quiet = None
self.no_launch = None
self.disable_updates = None
self.web_port = None
self.web_host = None
self.web_root = None
self.developer = None
self.db_type = None
self.db_prefix = None
self.db_host = None
self.db_port = None
self.db_username = None
self.db_password = None
self.debug = None
self.newest_version_string = None
self.oidc_client_id = 'sickrage-app'
self.oidc_client_secret = '5d4710b2-ca70-4d39-b5a3-0705e2c5e703'
self.naming_ep_type = ("%(seasonnumber)dx%(episodenumber)02d",
"s%(seasonnumber)02de%(episodenumber)02d",
"S%(seasonnumber)02dE%(episodenumber)02d",
"%(seasonnumber)02dx%(episodenumber)02d",
"S%(seasonnumber)02d E%(episodenumber)02d")
self.sports_ep_type = ("%(seasonnumber)dx%(episodenumber)02d",
"s%(seasonnumber)02de%(episodenumber)02d",
"S%(seasonnumber)02dE%(episodenumber)02d",
"%(seasonnumber)02dx%(episodenumber)02d",
"S%(seasonnumber)02 dE%(episodenumber)02d")
self.naming_ep_type_text = (
"1x02",
"s01e02",
"S01E02",
"01x02",
"S01 E02",
)
self.naming_multi_ep_type = {
0: ["-%(episodenumber)02d"] * len(self.naming_ep_type),
1: [" - " + x for x in self.naming_ep_type],
2: [x + "%(episodenumber)02d" for x in ("x", "e", "E", "x")]
}
self.naming_multi_ep_type_text = ("extend", "duplicate", "repeat")
self.naming_sep_type = (" - ", " ")
self.naming_sep_type_text = (" - ", "space")
self.user_agent = 'SiCKRAGE.CE.1/({};{};{})'.format(
platform.system(), platform.release(), str(uuid.uuid1()))
self.languages = [
language for language in os.listdir(sickrage.LOCALE_DIR)
if '_' in language
]
self.client_web_urls = {'torrent': '', 'newznab': ''}
self.notifier_providers = {}
self.metadata_providers = {}
self.search_providers = {}
self.adba_connection = None
self.log = None
self.config = None
self.alerts = None
self.scheduler = None
self.wserver = None
self.google_auth = None
self.show_queue = None
self.search_queue = None
self.postprocessor_queue = None
self.version_updater = None
self.show_updater = None
self.tz_updater = None
self.rsscache_updater = None
self.daily_searcher = None
self.failed_snatch_searcher = None
self.backlog_searcher = None
self.proper_searcher = None
self.trakt_searcher = None
self.subtitle_searcher = None
self.auto_postprocessor = None
self.upnp_client = None
self.oidc_client = None
self.quicksearch_cache = None
self.announcements = None
self.api = None
def start(self):
self.started = True
self.io_loop = IOLoop.current()
# thread name
threading.currentThread().setName('CORE')
# init core classes
self.api = API()
self.main_db = MainDB(self.db_type, self.db_prefix, self.db_host,
self.db_port, self.db_username, self.db_password)
self.cache_db = CacheDB(self.db_type, self.db_prefix, self.db_host,
self.db_port, self.db_username,
self.db_password)
self.notifier_providers = NotifierProviders()
self.metadata_providers = MetadataProviders()
self.search_providers = SearchProviders()
self.log = Logger()
self.config = Config()
self.alerts = Notifications()
self.scheduler = TornadoScheduler({'apscheduler.timezone': 'UTC'})
self.wserver = WebServer()
self.show_queue = ShowQueue()
self.search_queue = SearchQueue()
self.postprocessor_queue = PostProcessorQueue()
self.version_updater = VersionUpdater()
self.show_updater = ShowUpdater()
self.tz_updater = TimeZoneUpdater()
self.rsscache_updater = RSSCacheUpdater()
self.daily_searcher = DailySearcher()
self.failed_snatch_searcher = FailedSnatchSearcher()
self.backlog_searcher = BacklogSearcher()
self.proper_searcher = ProperSearcher()
self.trakt_searcher = TraktSearcher()
self.subtitle_searcher = SubtitleSearcher()
self.auto_postprocessor = AutoPostProcessor()
self.upnp_client = UPNPClient()
self.quicksearch_cache = QuicksearchCache()
self.announcements = Announcements()
# setup oidc client
realm = KeycloakRealm(server_url='https://auth.sickrage.ca',
realm_name='sickrage')
self.oidc_client = realm.open_id_connect(
client_id=self.oidc_client_id,
client_secret=self.oidc_client_secret)
# check available space
try:
self.log.info("Performing disk space checks")
total_space, available_space = get_free_space(self.data_dir)
if available_space < 100:
self.log.warning(
'Shutting down as SiCKRAGE needs some space to work. You\'ll get corrupted data otherwise. Only %sMB left',
available_space)
return
except Exception:
self.log.error('Failed getting disk space: %s',
traceback.format_exc())
# check if we need to perform a restore first
if os.path.exists(
os.path.abspath(os.path.join(self.data_dir, 'restore'))):
success = restore_app_data(
os.path.abspath(os.path.join(self.data_dir, 'restore')),
self.data_dir)
self.log.info("Restoring SiCKRAGE backup: %s!" %
("FAILED", "SUCCESSFUL")[success])
if success:
self.main_db = MainDB(self.db_type, self.db_prefix,
self.db_host, self.db_port,
self.db_username, self.db_password)
self.cache_db = CacheDB(self.db_type, self.db_prefix,
self.db_host, self.db_port,
self.db_username, self.db_password)
shutil.rmtree(os.path.abspath(
os.path.join(self.data_dir, 'restore')),
ignore_errors=True)
# migrate old database file names to new ones
if os.path.isfile(
os.path.abspath(os.path.join(self.data_dir, 'sickbeard.db'))):
if os.path.isfile(os.path.join(self.data_dir, 'sickrage.db')):
helpers.move_file(
os.path.join(self.data_dir, 'sickrage.db'),
os.path.join(
self.data_dir, '{}.bak-{}'.format(
'sickrage.db',
datetime.datetime.now().strftime(
'%Y%m%d_%H%M%S'))))
helpers.move_file(
os.path.abspath(os.path.join(self.data_dir, 'sickbeard.db')),
os.path.abspath(os.path.join(self.data_dir, 'sickrage.db')))
# init encryption public and private keys
encryption.initialize()
# load config
self.config.load()
# set language
self.config.change_gui_lang(self.config.gui_lang)
# set socket timeout
socket.setdefaulttimeout(self.config.socket_timeout)
# setup logger settings
self.log.logSize = self.config.log_size
self.log.logNr = self.config.log_nr
self.log.logFile = os.path.join(self.data_dir, 'logs', 'sickrage.log')
self.log.debugLogging = self.config.debug
self.log.consoleLogging = not self.quiet
# start logger
self.log.start()
# perform database startup actions
for db in [self.main_db, self.cache_db]:
# perform integrity check
self.log.info("Performing integrity check on {} database".format(
db.name))
db.integrity_check()
# migrate database
self.log.info("Performing migrations on {} database".format(
db.name))
db.migrate()
# sync database repo
self.log.info("Performing sync on {} database".format(db.name))
db.sync_db_repo()
# cleanup
self.log.info("Performing cleanup on {} database".format(db.name))
db.cleanup()
# user agent
if self.config.random_user_agent:
self.user_agent = UserAgent().random
uses_netloc.append('scgi')
FancyURLopener.version = self.user_agent
# set torrent client web url
torrent_webui_url(True)
# load quicksearch cache
self.quicksearch_cache.load()
if self.config.default_page not in ('schedule', 'history', 'IRC'):
self.config.default_page = 'home'
# cleanup cache folder
for folder in ['mako', 'sessions', 'indexers']:
try:
shutil.rmtree(os.path.join(self.cache_dir, folder),
ignore_errors=True)
except Exception:
continue
if self.config.web_port < 21 or self.config.web_port > 65535:
self.config.web_port = 8081
if not self.config.web_cookie_secret:
self.config.web_cookie_secret = generate_secret()
# attempt to help prevent users from breaking links by using a bad url
if not self.config.anon_redirect.endswith('?'):
self.config.anon_redirect = ''
if not re.match(r'\d+\|[^|]+(?:\|[^|]+)*', self.config.root_dirs):
self.config.root_dirs = ''
self.config.naming_force_folders = check_force_season_folders()
if self.config.nzb_method not in ('blackhole', 'sabnzbd', 'nzbget'):
self.config.nzb_method = 'blackhole'
if self.config.torrent_method not in ('blackhole', 'utorrent',
'transmission', 'deluge',
'deluged', 'download_station',
'rtorrent', 'qbittorrent',
'mlnet', 'putio'):
self.config.torrent_method = 'blackhole'
if self.config.autopostprocessor_freq < self.config.min_autopostprocessor_freq:
self.config.autopostprocessor_freq = self.config.min_autopostprocessor_freq
if self.config.daily_searcher_freq < self.config.min_daily_searcher_freq:
self.config.daily_searcher_freq = self.config.min_daily_searcher_freq
if self.config.backlog_searcher_freq < self.config.min_backlog_searcher_freq:
self.config.backlog_searcher_freq = self.config.min_backlog_searcher_freq
if self.config.version_updater_freq < self.config.min_version_updater_freq:
self.config.version_updater_freq = self.config.min_version_updater_freq
if self.config.subtitle_searcher_freq < self.config.min_subtitle_searcher_freq:
self.config.subtitle_searcher_freq = self.config.min_subtitle_searcher_freq
if self.config.failed_snatch_age < self.config.min_failed_snatch_age:
self.config.failed_snatch_age = self.config.min_failed_snatch_age
if self.config.proper_searcher_interval not in ('15m', '45m', '90m',
'4h', 'daily'):
self.config.proper_searcher_interval = 'daily'
if self.config.showupdate_hour < 0 or self.config.showupdate_hour > 23:
self.config.showupdate_hour = 0
# add version checker job
self.scheduler.add_job(self.version_updater.run,
IntervalTrigger(
hours=self.config.version_updater_freq,
timezone='utc'),
name=self.version_updater.name,
id=self.version_updater.name)
# add network timezones updater job
self.scheduler.add_job(self.tz_updater.run,
IntervalTrigger(days=1, timezone='utc'),
name=self.tz_updater.name,
id=self.tz_updater.name)
# add show updater job
self.scheduler.add_job(self.show_updater.run,
IntervalTrigger(
days=1,
start_date=datetime.datetime.now().replace(
hour=self.config.showupdate_hour),
timezone='utc'),
name=self.show_updater.name,
id=self.show_updater.name)
# add rss cache updater job
self.scheduler.add_job(self.rsscache_updater.run,
IntervalTrigger(minutes=15, timezone='utc'),
name=self.rsscache_updater.name,
id=self.rsscache_updater.name)
# add daily search job
self.scheduler.add_job(
self.daily_searcher.run,
IntervalTrigger(minutes=self.config.daily_searcher_freq,
start_date=datetime.datetime.now() +
datetime.timedelta(minutes=4),
timezone='utc'),
name=self.daily_searcher.name,
id=self.daily_searcher.name)
# add failed snatch search job
self.scheduler.add_job(
self.failed_snatch_searcher.run,
IntervalTrigger(hours=1,
start_date=datetime.datetime.now() +
datetime.timedelta(minutes=4),
timezone='utc'),
name=self.failed_snatch_searcher.name,
id=self.failed_snatch_searcher.name)
# add backlog search job
self.scheduler.add_job(
self.backlog_searcher.run,
IntervalTrigger(minutes=self.config.backlog_searcher_freq,
start_date=datetime.datetime.now() +
datetime.timedelta(minutes=30),
timezone='utc'),
name=self.backlog_searcher.name,
id=self.backlog_searcher.name)
# add auto-postprocessing job
self.scheduler.add_job(self.auto_postprocessor.run,
IntervalTrigger(
minutes=self.config.autopostprocessor_freq,
timezone='utc'),
name=self.auto_postprocessor.name,
id=self.auto_postprocessor.name)
# add find proper job
self.scheduler.add_job(self.proper_searcher.run,
IntervalTrigger(minutes={
'15m': 15,
'45m': 45,
'90m': 90,
'4h': 4 * 60,
'daily': 24 * 60
}[self.config.proper_searcher_interval],
timezone='utc'),
name=self.proper_searcher.name,
id=self.proper_searcher.name)
# add trakt.tv checker job
self.scheduler.add_job(self.trakt_searcher.run,
IntervalTrigger(hours=1, timezone='utc'),
name=self.trakt_searcher.name,
id=self.trakt_searcher.name)
# add subtitles finder job
self.scheduler.add_job(self.subtitle_searcher.run,
IntervalTrigger(
hours=self.config.subtitle_searcher_freq,
timezone='utc'),
name=self.subtitle_searcher.name,
id=self.subtitle_searcher.name)
# add upnp client job
self.scheduler.add_job(
self.upnp_client.run,
IntervalTrigger(seconds=self.upnp_client._nat_portmap_lifetime,
timezone='utc'),
name=self.upnp_client.name,
id=self.upnp_client.name)
# add announcements job
self.scheduler.add_job(self.announcements.run,
IntervalTrigger(minutes=15, timezone='utc'),
name=self.announcements.name,
id=self.announcements.name)
# start queues
self.search_queue.start()
self.show_queue.start()
self.postprocessor_queue.start()
# fire off startup events
self.io_loop.add_callback(self.load_shows)
self.io_loop.add_callback(self.version_updater.run)
self.io_loop.add_callback(self.tz_updater.run)
self.io_loop.add_callback(self.announcements.run)
# start scheduler service
self.scheduler.start()
# start web server
self.wserver.start()
# launch browser window
if all(
[not sickrage.app.no_launch, sickrage.app.config.launch_browser]):
self.io_loop.add_callback(
functools.partial(launch_browser,
('http',
'https')[sickrage.app.config.enable_https],
sickrage.app.config.web_host,
sickrage.app.config.web_port))
def started():
threading.currentThread().setName('CORE')
self.log.info("SiCKRAGE :: STARTED")
self.log.info("SiCKRAGE :: APP VERSION:[{}]".format(
sickrage.version()))
self.log.info("SiCKRAGE :: CONFIG VERSION:[v{}]".format(
self.config.config_version))
self.log.info("SiCKRAGE :: DATABASE VERSION:[v{}]".format(
self.main_db.version))
self.log.info("SiCKRAGE :: DATABASE TYPE:[{}]".format(
self.db_type))
self.log.info("SiCKRAGE :: URL:[{}://{}:{}{}]".format(
('http', 'https')[self.config.enable_https],
self.config.web_host, self.config.web_port,
self.config.web_root))
# start io_loop
self.io_loop.add_callback(started)
self.io_loop.start()
def load_shows(self):
threading.currentThread().setName('CORE')
session = self.main_db.session()
self.log.info('Loading initial shows list')
self.loading_shows = True
self.shows = {}
for query in session.query(MainDB.TVShow).with_entities(
MainDB.TVShow.indexer_id, MainDB.TVShow.indexer,
MainDB.TVShow.name):
try:
self.log.info('Loading show {} and building caches'.format(
query.name))
self.shows.update({
(query.indexer_id, query.indexer):
TVShow(query.indexer_id, query.indexer)
})
self.quicksearch_cache.add_show(query.indexer_id)
except Exception as e:
self.log.debug('There was an error loading show: {}'.format(
query.name))
self.loading_shows = False
self.log.info('Loading initial shows list finished')
def shutdown(self, restart=False):
if self.started:
self.log.info('SiCKRAGE IS SHUTTING DOWN!!!')
# shutdown webserver
if self.wserver:
self.wserver.shutdown()
# log out of ADBA
if self.adba_connection:
self.log.debug("Shutting down ANIDB connection")
self.adba_connection.stop()
# save shows
self.log.info('Saving all shows to the database')
for show in self.shows.values():
show.save()
# save settings
self.config.save()
# shutdown logging
if self.log:
self.log.close()
if restart:
os.execl(sys.executable, sys.executable, *sys.argv)
if sickrage.app.daemon:
sickrage.app.daemon.stop()
self.started = False
if self.io_loop:
self.io_loop.stop()
def on_tick():
tick_logger = logging.getLogger(__name__)
try:
last = snapshotter.snap_last_trade('BTC-USD')
rxt = datetime.datetime.now(pytz.utc)
px = float(last['price'])
qty = float(last['size'])
last_row = [{'index': rxt, 'price': px, 'qty': qty}]
tick_lib.write('BTC-USD', last_row, metadata={'source': 'CoinbasePro'})
tick_logger.info("wrote latest trade to Arctic: {} @ {}".format(
qty, px))
except CoinbaseAPIError:
tick_logger.info(
"ignoring transient error from Coinbase Pro API; will retry")
if __name__ == '__main__':
scheduler = TornadoScheduler()
scheduler.add_jobstore(MemoryJobStore())
scheduler.add_executor(TornadoExecutor())
scheduler.add_job(on_tick, 'interval', minutes=1)
scheduler.start()
# Execution will block here until Ctrl+C (Ctrl+Break on Windows) is pressed.
try:
logger.info("starting Tornado")
IOLoop.instance().start()
except (KeyboardInterrupt, SystemExit):
pass
def __init__(self, sqlalchemy_engine):
self.scheduler = TornadoScheduler()
self.scheduler.add_jobstore("sqlalchemy", engine=sqlalchemy_engine)
def tick():
global number
number+=1
print('Tick! The number is {0}'.format(number))
def make_app():
return tornado.web.Application([
(r"/", MainHandler),
])
if __name__ == "__main__":
scheduler = TornadoScheduler()
jobstores = {
'default': MemoryJobStore()
}
executors = {
'default': TornadoExecutor(max_workers=5)
}
job_defaults = {
'coalesce': False,
'max_instances': 1
}
scheduler.configure(jobstores=jobstores, executors=executors, job_defaults=job_defaults, timezone=utc)
scheduler.add_job(tick, 'interval', seconds=3)
def start(self):
self.started = True
self.io_loop = IOLoop.current()
# thread name
threading.currentThread().setName('CORE')
# patch modules with encoding kludge
patch_modules()
# init core classes
self.notifier_providers = NotifierProviders()
self.metadata_providers = MetadataProviders()
self.search_providers = SearchProviders()
self.log = Logger()
self.config = Config()
self.alerts = Notifications()
self.main_db = MainDB()
self.cache_db = CacheDB()
self.scheduler = TornadoScheduler()
self.wserver = WebServer()
self.name_cache = NameCache()
self.show_queue = ShowQueue()
self.search_queue = SearchQueue()
self.postprocessor_queue = PostProcessorQueue()
self.event_queue = EventQueue()
self.version_updater = VersionUpdater()
self.show_updater = ShowUpdater()
self.tz_updater = TimeZoneUpdater()
self.rsscache_updater = RSSCacheUpdater()
self.daily_searcher = DailySearcher()
self.failed_snatch_searcher = FailedSnatchSearcher()
self.backlog_searcher = BacklogSearcher()
self.proper_searcher = ProperSearcher()
self.trakt_searcher = TraktSearcher()
self.subtitle_searcher = SubtitleSearcher()
self.auto_postprocessor = AutoPostProcessor()
self.upnp_client = UPNPClient()
self.quicksearch_cache = QuicksearchCache()
# setup oidc client
realm = KeycloakRealm(server_url='https://auth.sickrage.ca', realm_name='sickrage')
self.oidc_client = realm.open_id_connect(client_id='sickrage-app',
client_secret='5d4710b2-ca70-4d39-b5a3-0705e2c5e703')
# Check if we need to perform a restore first
if os.path.exists(os.path.abspath(os.path.join(self.data_dir, 'restore'))):
success = restoreSR(os.path.abspath(os.path.join(self.data_dir, 'restore')), self.data_dir)
self.log.info("Restoring SiCKRAGE backup: {}!".format(("FAILED", "SUCCESSFUL")[success]))
if success:
shutil.rmtree(os.path.abspath(os.path.join(self.data_dir, 'restore')), ignore_errors=True)
# migrate old database file names to new ones
if os.path.isfile(os.path.abspath(os.path.join(self.data_dir, 'sickbeard.db'))):
if os.path.isfile(os.path.join(self.data_dir, 'sickrage.db')):
helpers.move_file(os.path.join(self.data_dir, 'sickrage.db'),
os.path.join(self.data_dir, '{}.bak-{}'
.format('sickrage.db',
datetime.datetime.now().strftime(
'%Y%m%d_%H%M%S'))))
helpers.move_file(os.path.abspath(os.path.join(self.data_dir, 'sickbeard.db')),
os.path.abspath(os.path.join(self.data_dir, 'sickrage.db')))
# load config
self.config.load()
# set language
self.config.change_gui_lang(self.config.gui_lang)
# set socket timeout
socket.setdefaulttimeout(self.config.socket_timeout)
# setup logger settings
self.log.logSize = self.config.log_size
self.log.logNr = self.config.log_nr
self.log.logFile = os.path.join(self.data_dir, 'logs', 'sickrage.log')
self.log.debugLogging = self.config.debug
self.log.consoleLogging = not self.quiet
# start logger
self.log.start()
# user agent
if self.config.random_user_agent:
self.user_agent = UserAgent().random
urlparse.uses_netloc.append('scgi')
urllib.FancyURLopener.version = self.user_agent
# set torrent client web url
torrent_webui_url(True)
# Check available space
try:
total_space, available_space = getFreeSpace(self.data_dir)
if available_space < 100:
self.log.error('Shutting down as SiCKRAGE needs some space to work. You\'ll get corrupted data '
'otherwise. Only %sMB left', available_space)
return
except Exception:
self.log.error('Failed getting disk space: %s', traceback.format_exc())
# perform database startup actions
for db in [self.main_db, self.cache_db]:
# initialize database
db.initialize()
# check integrity of database
db.check_integrity()
# migrate database
db.migrate()
# misc database cleanups
db.cleanup()
# upgrade database
db.upgrade()
# compact main database
if self.config.last_db_compact < time.time() - 604800: # 7 days
self.main_db.compact()
self.config.last_db_compact = int(time.time())
# load name cache
self.name_cache.load()
# load data for shows from database
self.load_shows()
if self.config.default_page not in ('schedule', 'history', 'IRC'):
self.config.default_page = 'home'
# cleanup cache folder
for folder in ['mako', 'sessions', 'indexers']:
try:
shutil.rmtree(os.path.join(sickrage.app.cache_dir, folder), ignore_errors=True)
except Exception:
continue
if self.config.web_port < 21 or self.config.web_port > 65535:
self.config.web_port = 8081
if not self.config.web_cookie_secret:
self.config.web_cookie_secret = generate_secret()
# attempt to help prevent users from breaking links by using a bad url
if not self.config.anon_redirect.endswith('?'):
self.config.anon_redirect = ''
if not re.match(r'\d+\|[^|]+(?:\|[^|]+)*', self.config.root_dirs):
self.config.root_dirs = ''
self.config.naming_force_folders = check_force_season_folders()
if self.config.nzb_method not in ('blackhole', 'sabnzbd', 'nzbget'):
self.config.nzb_method = 'blackhole'
if self.config.torrent_method not in ('blackhole', 'utorrent', 'transmission', 'deluge', 'deluged',
'download_station', 'rtorrent', 'qbittorrent', 'mlnet', 'putio'):
self.config.torrent_method = 'blackhole'
if self.config.autopostprocessor_freq < self.config.min_autopostprocessor_freq:
self.config.autopostprocessor_freq = self.config.min_autopostprocessor_freq
if self.config.daily_searcher_freq < self.config.min_daily_searcher_freq:
self.config.daily_searcher_freq = self.config.min_daily_searcher_freq
if self.config.backlog_searcher_freq < self.config.min_backlog_searcher_freq:
self.config.backlog_searcher_freq = self.config.min_backlog_searcher_freq
if self.config.version_updater_freq < self.config.min_version_updater_freq:
self.config.version_updater_freq = self.config.min_version_updater_freq
if self.config.subtitle_searcher_freq < self.config.min_subtitle_searcher_freq:
self.config.subtitle_searcher_freq = self.config.min_subtitle_searcher_freq
if self.config.failed_snatch_age < self.config.min_failed_snatch_age:
self.config.failed_snatch_age = self.config.min_failed_snatch_age
if self.config.proper_searcher_interval not in ('15m', '45m', '90m', '4h', 'daily'):
self.config.proper_searcher_interval = 'daily'
if self.config.showupdate_hour < 0 or self.config.showupdate_hour > 23:
self.config.showupdate_hour = 0
# add version checker job
self.scheduler.add_job(
self.version_updater.run,
IntervalTrigger(
hours=self.config.version_updater_freq,
),
name=self.version_updater.name,
id=self.version_updater.name
)
# add network timezones updater job
self.scheduler.add_job(
self.tz_updater.run,
IntervalTrigger(
days=1,
),
name=self.tz_updater.name,
id=self.tz_updater.name
)
# add show updater job
self.scheduler.add_job(
self.show_updater.run,
IntervalTrigger(
days=1,
start_date=datetime.datetime.now().replace(hour=self.config.showupdate_hour)
),
name=self.show_updater.name,
id=self.show_updater.name
)
# add rss cache updater job
self.scheduler.add_job(
self.rsscache_updater.run,
IntervalTrigger(
minutes=15,
),
name=self.rsscache_updater.name,
id=self.rsscache_updater.name
)
# add daily search job
self.scheduler.add_job(
self.daily_searcher.run,
IntervalTrigger(
minutes=self.config.daily_searcher_freq,
start_date=datetime.datetime.now() + datetime.timedelta(minutes=4)
),
name=self.daily_searcher.name,
id=self.daily_searcher.name
)
# add failed snatch search job
self.scheduler.add_job(
self.failed_snatch_searcher.run,
IntervalTrigger(
hours=1,
start_date=datetime.datetime.now() + datetime.timedelta(minutes=4)
),
name=self.failed_snatch_searcher.name,
id=self.failed_snatch_searcher.name
)
# add backlog search job
self.scheduler.add_job(
self.backlog_searcher.run,
IntervalTrigger(
minutes=self.config.backlog_searcher_freq,
start_date=datetime.datetime.now() + datetime.timedelta(minutes=30)
),
name=self.backlog_searcher.name,
id=self.backlog_searcher.name
)
# add auto-postprocessing job
self.scheduler.add_job(
self.auto_postprocessor.run,
IntervalTrigger(
minutes=self.config.autopostprocessor_freq
),
name=self.auto_postprocessor.name,
id=self.auto_postprocessor.name
)
# add find proper job
self.scheduler.add_job(
self.proper_searcher.run,
IntervalTrigger(
minutes={
'15m': 15,
'45m': 45,
'90m': 90,
'4h': 4 * 60,
'daily': 24 * 60
}[self.config.proper_searcher_interval]
),
name=self.proper_searcher.name,
id=self.proper_searcher.name
)
# add trakt.tv checker job
self.scheduler.add_job(
self.trakt_searcher.run,
IntervalTrigger(
hours=1
),
name=self.trakt_searcher.name,
id=self.trakt_searcher.name
)
# add subtitles finder job
self.scheduler.add_job(
self.subtitle_searcher.run,
IntervalTrigger(
hours=self.config.subtitle_searcher_freq
),
name=self.subtitle_searcher.name,
id=self.subtitle_searcher.name
)
# add upnp client job
self.scheduler.add_job(
self.upnp_client.run,
IntervalTrigger(
seconds=self.upnp_client._nat_portmap_lifetime
),
name=self.upnp_client.name,
id=self.upnp_client.name
)
# add namecache update job
self.scheduler.add_job(
self.name_cache.build_all,
IntervalTrigger(
days=1,
),
name=self.name_cache.name,
id=self.name_cache.name
)
# start scheduler service
self.scheduler.start()
# start queue's
self.search_queue.start()
self.show_queue.start()
self.postprocessor_queue.start()
self.event_queue.start()
# fire off startup events
self.event_queue.fire_event(self.name_cache.build_all)
self.event_queue.fire_event(self.version_updater.run)
self.event_queue.fire_event(self.tz_updater.run)
# start webserver
self.wserver.start()
# launch browser window
if all([not sickrage.app.no_launch, sickrage.app.config.launch_browser]):
self.event_queue.fire_event(lambda: launch_browser(('http', 'https')[sickrage.app.config.enable_https],
sickrage.app.config.web_host,
sickrage.app.config.web_port))
# start ioloop
self.io_loop.start()
#import logging
from tornado.options import options,define
define("port",default=8082,help="server listen port", type=int)
define("dbinit",default=0,help="if need to init db, 0:No; 1:Yes", type=int)
APP_USER_COOKIE_NAME = "_user__"
UPLOAD_FOLDER = 'uploads'
ALLOWED_EXTENSIONS = set(['png', 'jpg', 'jpeg', 'gif','doc','docx','pdf','txt','xlsx'])
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
log_file_path = os.path.join(BASE_DIR, 'logs/auth.log')
#err_file_path = os.path.join(BASE_DIR, 'logs/err.log')
sched = TornadoScheduler()
# logger.remove()
logger.add(sys.stderr, colorize=True, format="<green>{time}</green> <level>{message}</level>", filter="auth")
logger.add(log_file_path, rotation="1 days", colorize=True, format="<green>{time}</green> <level>{message}</level>", encoding='utf-8', filter="auth", level="DEBUG")
#logger.add(err_file_path, rotation="1 days", colorize=True, format="<green>{time}</green> <level>{message}</level>", encoding='utf-8', filter="auth", level="INFO")
def allowed_file(filename):
return '.' in filename and filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS
class BaseHandler(tornado.web.RequestHandler):
async def prepare(self):
# get_current_user cannot be a coroutine, so set
# self.current_user in prepare instead.
executors = {
'default': ThreadPoolExecutor(1),
'processpool': ProcessPoolExecutor(1)
}
job_defaults = {
'coalesce': False,
'max_instances': 1,
'misfire_grace_time': 60 * 60 * 20,
}
db_info = settings.db
jobstores = {
"default":
SQLAlchemyJobStore(url="mysql://%s:%s@%s/%s" %
(db_info["user"], db_info["password"], db_info["host"],
db_info["db_name"]))
}
scheduler = TornadoScheduler(jobstores=jobstores,
executors=executors,
job_defaults=job_defaults)
# scheduler = BlockingScheduler(jobstores=jobstores, executors=executors, job_defaults=job_defaults)
# print "scheduler starting"
# scheduler.start()
# print "scheduler started"
# scheduler.add_job(print_datetime, 'interval', id="xh-job-1", seconds=2, replace_existing=True)
# jobstores = {
# "default": SQLAlchemyJobStore(DbStore)
# }
executors = {
"default": ThreadPoolExecutor(10), # 线程池
# "processpool": ProcessPoolExecutor(5) # 进程池
}
job_defaults = {
"coalesce": True, # 只执行一次
"max_instances": 1 # 最大实例数量
}
# 实例化任务调度器TornadoScheduler
scheduler = TornadoScheduler(
daemonic=False, # 不使用守护进程
# jobstores=jobstores, #默认使用内存
executors=executors,
job_defaults=job_defaults)
# 全局可用
AutoReportGlobals.Scheduler = scheduler
AutoReportGlobals.EmailIds = {}
def onListener(event):
if event.exception:
logger.warn("The job crashed :( " + str(event))
# print("The job crashed :(", event.alias, event.code, event.exception,
# event.job_id, event.jobstore, event.retval, event.scheduled_run_time)
class SchedulerManager():
def __init__(self, config=None, syncobj=None):
if config is None:
config = Config()
self.config = config
executors = {
'default': ThreadPoolExecutor(20),
'processpool': ProcessPoolExecutor(5)
}
self.scheduler = TornadoScheduler(executors=executors)
self.task_queue = Queue()
self.poll_task_queue_callback = None
self.pool_task_queue_interval = 10
self.ioloop = IOLoop.instance()
self.poll_task_queue_callback = PeriodicCallback(
self.poll_task_queue, self.pool_task_queue_interval * 1000)
self.clear_finished_jobs_callback = PeriodicCallback(
self.clear_finished_jobs, 60 * 1000)
self.reset_timeout_job_callback = PeriodicCallback(
self.reset_timeout_job, 10 * 1000)
self.sync_obj = syncobj
if syncobj is not None:
self.sync_obj.set_on_remove_schedule_job(
self.on_cluster_remove_scheduling_job)
self.sync_obj.set_on_add_schedule_job(
self.on_cluster_add_scheduling_job)
def init(self):
session = Session()
# move completed jobs into history
for job in session.query(SpiderExecutionQueue).filter(
SpiderExecutionQueue.status.in_((2, 3))):
historical_job = HistoricalJob()
historical_job.id = job.id
historical_job.spider_id = job.spider_id
historical_job.project_name = job.project_name
historical_job.spider_name = job.spider_name
historical_job.fire_time = job.fire_time
historical_job.start_time = job.start_time
historical_job.complete_time = job.update_time
historical_job.status = job.status
session.delete(job)
session.add(historical_job)
session.commit()
# init triggers
triggers = session.query(Trigger)
for trigger in triggers:
try:
self.add_job(trigger.id, trigger.cron_pattern)
except InvalidCronExpression:
logger.warning('Trigger %d,%s cannot be added ' %
(trigger.id, trigger.cron_pattern))
session.close()
self.scheduler.start()
self.poll_task_queue_callback.start()
self.clear_finished_jobs_callback.start()
self.reset_timeout_job_callback.start()
def poll_task_queue(self):
if self.task_queue.empty():
with session_scope() as session:
tasks_to_run = session.query(SpiderExecutionQueue).filter_by(
status=0).order_by(SpiderExecutionQueue.update_time).slice(
0, 10)
for task in tasks_to_run:
self.task_queue.put(task)
def build_cron_trigger(self, cron):
cron_parts = cron.split(' ')
if len(cron_parts) != 5:
raise InvalidCronExpression()
try:
crontrigger = CronTrigger(
minute=cron_parts[0],
hour=cron_parts[1],
day=cron_parts[2],
month=cron_parts[3],
day_of_week=cron_parts[4],
)
return crontrigger
except ValueError:
raise InvalidCronExpression()
def add_job(self, trigger_id, cron):
logger.debug('adding trigger %s %s' % (trigger_id, cron))
crontrigger = self.build_cron_trigger(cron)
job = self.scheduler.add_job(func=self.trigger_fired,
trigger=crontrigger,
kwargs={'trigger_id': trigger_id},
id=str(trigger_id),
replace_existing=True)
if self.sync_obj:
self.ioloop.call_later(0, self.sync_obj.add_schedule_job,
trigger_id)
#self.sync_obj.add_schedule_job(trigger_id)
def on_cluster_remove_scheduling_job(self, job_id):
logger.debug('on_cluster_remove_scheduling_job')
if self.scheduler.get_job(job_id):
self.scheduler.remove_job(job_id)
def on_cluster_add_scheduling_job(self, trigger_id):
logger.debug('on_cluster_add_scheduling_job')
with session_scope() as session:
trigger = session.query(Trigger).filter_by(id=trigger_id).first()
if trigger is None:
return
crontrigger = self.build_cron_trigger(trigger.cron_pattern)
job = self.scheduler.add_job(func=self.trigger_fired,
trigger=crontrigger,
kwargs={'trigger_id': trigger_id},
id=str(trigger_id),
replace_existing=True)
def trigger_fired(self, trigger_id):
with session_scope() as session:
trigger = session.query(Trigger).filter_by(id=trigger_id).first()
if not trigger:
logger.error('Trigger %s not found.' % trigger_id)
return
spider = session.query(Spider).filter_by(
id=trigger.spider_id).first()
project = session.query(Project).filter_by(
id=spider.project_id).first()
executing = session.query(SpiderExecutionQueue).filter(
SpiderExecutionQueue.spider_id == spider.id,
SpiderExecutionQueue.status.in_([0, 1]))
concurrency_setting = session.query(SpiderSettings).filter_by(
spider_id=spider.id, setting_key='concurrency').first()
concurrency = int(
concurrency_setting.value) if concurrency_setting else 1
executing_slots = [
executing_job.slot for executing_job in executing
]
free_slots = [
x for x in range(1, concurrency + 1)
if x not in executing_slots
]
if not free_slots:
logger.warning(
'spider %s-%s is configured as %d concurency, and %d in queue, skipping'
% (project.name, spider.name, concurrency,
len(executing_slots)))
return
executing = SpiderExecutionQueue()
executing.id = generate_job_id()
executing.spider_id = spider.id
executing.project_name = project.name
executing.spider_name = spider.name
executing.fire_time = datetime.datetime.now()
executing.update_time = datetime.datetime.now()
executing.slot = free_slots[0]
session.add(executing)
try:
session.commit()
except (Exception, IntegrityError) as e:
logger.warning(e)
session.close()
return
def add_schedule(self, project, spider, cron):
with session_scope() as session:
triggers = session.query(Trigger).filter(
Trigger.spider_id == spider.id)
found = False
for trigger in triggers:
if trigger.cron_pattern == cron:
found = True
break
if not found:
# create a cron_trigger for just validating
cron_trigger = self.build_cron_trigger(cron)
trigger = Trigger()
trigger.spider_id = spider.id
trigger.cron_pattern = cron
session.add(trigger)
session.commit()
self.add_job(trigger.id, cron)
def add_task(self, project_name, spider_name):
session = Session()
project = session.query(Project).filter(
Project.name == project_name).first()
spider = session.query(Spider).filter(
Spider.name == spider_name,
Spider.project_id == project.id).first()
try:
existing = list(
session.query(SpiderExecutionQueue).filter(
SpiderExecutionQueue.spider_id == spider.id,
SpiderExecutionQueue.status.in_([0, 1])))
if existing:
logger.warning('job %s_%s is running, ignoring schedule' %
(project.name, spider.name))
raise JobRunning(existing[0].id)
executing = SpiderExecutionQueue()
jobid = generate_job_id()
executing.id = jobid
executing.spider_id = spider.id
executing.project_name = project.name
executing.spider_name = spider.name
executing.fire_time = datetime.datetime.now()
executing.update_time = datetime.datetime.now()
session.add(executing)
session.commit()
session.refresh(executing)
return executing
finally:
session.close()
def on_node_expired(self, node_id):
session = Session()
for job in session.query(SpiderExecutionQueue).filter(
SpiderExecutionQueue.node_id == node_id,
SpiderExecutionQueue.status == 1):
job.status = 0
job.update_time = datetime.datetime.now()
job.start_time = None
job.pid = None
job.node_id = None
session.add(job)
session.commit()
session.close()
def jobs(self):
session = Session()
pending = list(
session.query(SpiderExecutionQueue).filter(
SpiderExecutionQueue.status == 0))
running = list(
session.query(SpiderExecutionQueue).filter(
SpiderExecutionQueue.status == 1))
finished = list(session.query(HistoricalJob).slice(0, 100))
session.close()
return pending, running, finished
def job_start(self, jobid, pid):
with session_scope() as session:
job = session.query(SpiderExecutionQueue).filter_by(
id=jobid).first()
if job.start_time is None:
job.start_time = datetime.datetime.now()
job.update_time = datetime.datetime.now()
if job.pid is None and pid:
job.pid = pid
session.add(job)
session.commit()
session.close()
def get_next_task(self, node_id):
if not self.task_queue.empty():
session = Session()
try:
next_task = self.task_queue.get_nowait()
except Empty:
return None
next_task = session.query(SpiderExecutionQueue).filter(
SpiderExecutionQueue.id == next_task.id,
SpiderExecutionQueue.status == 0).first()
if not next_task:
return None
next_task.start_time = datetime.datetime.now()
next_task.update_time = datetime.datetime.now()
next_task.node_id = node_id
next_task.status = 1
session.add(next_task)
session.commit()
session.refresh(next_task)
session.close()
return next_task
return None
def has_task(self):
return not self.task_queue.empty()
def jobs_running(self, node_id, job_ids):
'''
:param node_id:
:param job_ids:
:return:(job_id) to kill
'''
with session_scope() as session:
for job_id in job_ids:
job = session.query(SpiderExecutionQueue).filter(
SpiderExecutionQueue.id == job_id).first()
if job:
if job.node_id is None:
job.node_id = node_id
if job.node_id != node_id or \
job.status != 1:
yield job.id
else:
job.update_time = datetime.datetime.now()
session.add(job)
else:
yield job_id
session.commit()
def job_finished(self, job, log_file=None, items_file=None):
session = Session()
if job.status not in (2, 3):
raise Exception('Invliad status.')
job_status = job.status
job = session.query(SpiderExecutionQueue).filter_by(id=job.id).first()
job.status = job_status
job.update_time = datetime.datetime.now()
historical_job = HistoricalJob()
historical_job.id = job.id
historical_job.spider_id = job.spider_id
historical_job.project_name = job.project_name
historical_job.spider_name = job.spider_name
historical_job.fire_time = job.fire_time
historical_job.start_time = job.start_time
historical_job.complete_time = job.update_time
historical_job.status = job.status
if log_file:
historical_job.log_file = log_file
import re
items_crawled_pattern = re.compile(
"\'item_scraped_count\': (\d+),")
error_log_pattern = re.compile("\'log_count/ERROR\': (\d+),")
warning_log_pattern = re.compile("\'log_count/WARNING\': (\d+),")
with open(log_file, 'r') as f:
log_content = f.read()
m = items_crawled_pattern.search(log_content)
if m:
historical_job.items_count = int(m.group(1))
m = error_log_pattern.search(log_content)
if m and historical_job.status == JOB_STATUS_SUCCESS:
historical_job.status = JOB_STATUS_FAIL
m = warning_log_pattern.search(log_content)
if m and historical_job.status == JOB_STATUS_SUCCESS:
historical_job.status = JOB_STATUS_WARNING
if items_file:
historical_job.items_file = items_file
session.delete(job)
session.add(historical_job)
session.commit()
session.refresh(historical_job)
# send mail
if historical_job.status == JOB_STATUS_FAIL:
self.try_send_job_failed_mail(historical_job)
session.close()
return historical_job
def try_send_job_failed_mail(self, job):
logger.debug('try_send_job_failed_mail')
job_fail_send_mail = self.config.getboolean('job_fail_send_mail')
if job_fail_send_mail:
try:
mail_sender = MailSender(self.config)
subject = 'scrapydd job failed'
to_address = self.config.get('job_fail_mail_receiver')
content = 'bot:%s \r\nspider:%s \r\n job_id:%s \r\n' % (
job.spider.project.name, job.spider_name, job.id)
mail_sender.send(to_addresses=to_address,
subject=subject,
content=content)
except Exception as e:
logger.error('Error when sending job_fail mail %s' % e)
def clear_finished_jobs(self):
job_history_limit_each_spider = 100
with session_scope() as session:
spiders = list(session.query(distinct(HistoricalJob.spider_id)))
for row in spiders:
spider_id = row[0]
with session_scope() as session:
over_limitation_jobs = list(session.query(HistoricalJob)\
.filter(HistoricalJob.spider_id==spider_id)\
.order_by(desc(HistoricalJob.complete_time))\
.slice(job_history_limit_each_spider, 1000)\
.all())
for over_limitation_job in over_limitation_jobs:
self._remove_histical_job(over_limitation_job)
def _clear_running_jobs(self):
with session_scope() as session:
jobs = list(
session.query(SpiderExecutionQueue).filter(
SpiderExecutionQueue.status.in_([0, 1])))
for job in jobs:
self._remove_histical_job(job)
def reset_timeout_job(self):
with session_scope() as session:
timeout_time = datetime.datetime.now() - datetime.timedelta(
minutes=1)
for job in session.query(SpiderExecutionQueue).filter(
SpiderExecutionQueue.status == 1):
spider = session.query(Spider).filter_by(
id=job.spider_id).first()
job_timeout_setting = session.query(SpiderSettings).filter_by(
spider_id=spider.id, setting_key='timeout').first()
job_timeout = int(
job_timeout_setting.value) if job_timeout_setting else 3600
logger.debug((job.update_time - job.start_time).seconds)
if job.update_time < timeout_time:
# job is not refresh as expected, node might be died, reset the status to PENDING
job.status = 0
job.pid = None
job.node_id = None
job.update_time = datetime.datetime.now()
session.add(job)
logger.info('Job %s is timeout, reseting.' % job.id)
elif (job.update_time - job.start_time).seconds > job_timeout:
# job is running too long, should be killed
historical_job = HistoricalJob()
historical_job.id = job.id
historical_job.spider_id = job.spider_id
historical_job.project_name = job.project_name
historical_job.spider_name = job.spider_name
historical_job.fire_time = job.fire_time
historical_job.start_time = job.start_time
historical_job.complete_time = job.update_time
historical_job.status = 3
session.delete(job)
session.add(historical_job)
logger.info('Job %s is timeout, killed.' % job.id)
session.commit()
def _remove_histical_job(self, job):
'''
@type job: HistoricalJob
'''
with session_scope() as session:
job = session.query(HistoricalJob).filter(
HistoricalJob.id == job.id).first()
if job.items_file:
try:
os.remove(job.items_file)
except Exception as e:
logger.warning(e.message)
if job.log_file:
try:
os.remove(job.log_file)
except Exception as e:
logger.warning(e.message)
original_log_file = os.path.join('logs', job.project_name,
job.spider_name,
'%s.log' % job.id)
if os.path.exists(original_log_file):
os.remove(original_log_file)
original_items_file = os.path.join('items', job.project_name,
job.spider_name,
'%s.jl' % job.id)
if os.path.exists(original_items_file):
os.remove(original_items_file)
session.delete(job)
def remove_schedule(self, project_name, spider_name, trigger_id):
with session_scope() as session:
project = session.query(Project).filter(
Project.name == project_name).first()
spider = session.query(Spider).filter(
Spider.project_id == project.id,
Spider.name == spider_name).first()
trigger = session.query(Trigger).filter(
Trigger.spider_id == spider.id,
Trigger.id == trigger_id).first()
session.delete(trigger)
if self.scheduler.get_job(str(trigger_id)):
self.scheduler.remove_job(str(trigger.id))
if self.sync_obj:
logger.info('remove_schedule')
self.sync_obj.remove_schedule_job(trigger.id)
sys.path.append(os.path.join(root_path, 'api'))
sys.path.append(root_path)
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "alpha.settings.prod")
django.setup()
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s [%(levelname)s] %(name)s:%(lineno)d %(message)s',
datefmt='%Y-%m-%d %H:%M:%S')
logging.info((' Init Django with profile: %s ' %
os.environ.get('DJANGO_SETTINGS_MODULE')).center(80, '-'))
if __name__ == '__main__':
init_project()
scheduler = TornadoScheduler()
# 设置 apscheduler 的 logging
class AddedFilter(logging.Filter):
def filter(self, record):
return not record.msg.startswith(
'Adding') and not record.msg.startswith('Added')
logging.getLogger("apscheduler.scheduler").addFilter(AddedFilter())
if platform().startswith('Windows'):
from wind_spider import WindSpider
# 命令行解析万德账号
import argparse
parser = argparse.ArgumentParser(description='运行爬虫')
import time
import tornado.ioloop
from apscheduler.triggers.date import DateTrigger
from apscheduler.schedulers.tornado import TornadoScheduler
sched = TornadoScheduler()
def child_job():
"""创建一个执行时间为 60 s 的任务"""
print("start")
time.sleep(60)
print("end")
def main_job():
# sched.add_job(child_job, trigger=DateTrigger(), id="123")
sched.add_job(child_job, max_instances=10, trigger=DateTrigger(), id="123")
if __name__ == "__main__":
# 每 5 s 执行一次任务
sched.add_job(main_job, 'interval', seconds=5)
sched.start()
tornado.ioloop.IOLoop.instance().start()
'''问题复现：
python sche1.py
start
WARNING:apscheduler.scheduler:Execution of job "child_job (trigger: date[2020-05-07 11:15:49 CST], next run at: 2020-05-07 11:15:49 CST)" skipped: maximum number of running instances reached (1)
WARNING:apscheduler.scheduler:Execution of job "child_job (trigger: date[2020-05-07 11:15:54 CST], next run at: 2020-05-07 11:15:54 CST)" skipped: maximum number of running instances reached (1)
class Core(object):
def __init__(self):
self.started = False
self.io_loop = IOLoop.current()
# process id
self.PID = os.getpid()
# generate notifiers dict
self.notifiersDict = notifiersDict()
# generate metadata providers dict
self.metadataProvidersDict = metadataProvidersDict()
# generate providers dict
self.providersDict = providersDict()
# init notification queue
self.srNotifications = Notifications()
# init logger
self.srLogger = srLogger()
# init config
self.srConfig = srConfig()
# init databases
self.mainDB = MainDB()
self.cacheDB = CacheDB()
self.failedDB = FailedDB()
# init scheduler service
self.srScheduler = TornadoScheduler()
# init web server
self.srWebServer = srWebServer()
# init web client session
self.srWebSession = srSession()
# google api
self.googleAuth = googleAuth()
# name cache
self.NAMECACHE = srNameCache()
# queues
self.SHOWQUEUE = srShowQueue()
self.SEARCHQUEUE = srSearchQueue()
# updaters
self.VERSIONUPDATER = srVersionUpdater()
self.SHOWUPDATER = srShowUpdater()
# searchers
self.DAILYSEARCHER = srDailySearcher()
self.BACKLOGSEARCHER = srBacklogSearcher()
self.PROPERSEARCHER = srProperSearcher()
self.TRAKTSEARCHER = srTraktSearcher()
self.SUBTITLESEARCHER = srSubtitleSearcher()
# auto postprocessor
self.AUTOPOSTPROCESSOR = srPostProcessor()
# sickrage version
self.NEWEST_VERSION = None
self.NEWEST_VERSION_STRING = None
# anidb connection
self.ADBA_CONNECTION = None
# show list
self.SHOWLIST = []
def start(self):
self.started = True
# thread name
threading.currentThread().setName('CORE')
# Check if we need to perform a restore first
if os.path.exists(
os.path.abspath(os.path.join(sickrage.DATA_DIR, 'restore'))):
success = restoreSR(
os.path.abspath(os.path.join(sickrage.DATA_DIR, 'restore')),
sickrage.DATA_DIR)
print("Restoring SiCKRAGE backup: %s!\n" %
("FAILED", "SUCCESSFUL")[success])
if success:
shutil.rmtree(os.path.abspath(
os.path.join(sickrage.DATA_DIR, 'restore')),
ignore_errors=True)
# migrate old database file names to new ones
if os.path.isfile(
os.path.abspath(os.path.join(sickrage.DATA_DIR,
'sickbeard.db'))):
if os.path.isfile(os.path.join(sickrage.DATA_DIR, 'sickrage.db')):
helpers.moveFile(
os.path.join(sickrage.DATA_DIR, 'sickrage.db'),
os.path.join(
sickrage.DATA_DIR, '{}.bak-{}'.format(
'sickrage.db',
datetime.datetime.now().strftime(
'%Y%m%d_%H%M%S'))))
helpers.moveFile(
os.path.abspath(os.path.join(sickrage.DATA_DIR,
'sickbeard.db')),
os.path.abspath(os.path.join(sickrage.DATA_DIR,
'sickrage.db')))
# load config
self.srConfig.load()
# set socket timeout
socket.setdefaulttimeout(self.srConfig.SOCKET_TIMEOUT)
# setup logger settings
self.srLogger.logSize = self.srConfig.LOG_SIZE
self.srLogger.logNr = self.srConfig.LOG_NR
self.srLogger.logFile = self.srConfig.LOG_FILE
self.srLogger.debugLogging = sickrage.DEBUG
self.srLogger.consoleLogging = not sickrage.QUITE
# start logger
self.srLogger.start()
# Check available space
try:
total_space, available_space = getFreeSpace(sickrage.DATA_DIR)
if available_space < 100:
self.srLogger.error(
'Shutting down as SiCKRAGE needs some space to work. You\'ll get corrupted data otherwise. Only %sMB left',
available_space)
sickrage.restart = False
return
except:
self.srLogger.error('Failed getting diskspace: %s',
traceback.format_exc())
# perform database startup actions
for db in [self.mainDB, self.cacheDB, self.failedDB]:
# initialize database
db.initialize()
# check integrity of database
db.check_integrity()
# migrate database
db.migrate()
# misc database cleanups
db.cleanup()
# compact main database
if not sickrage.DEVELOPER and self.srConfig.LAST_DB_COMPACT < time.time(
) - 604800: # 7 days
self.mainDB.compact()
self.srConfig.LAST_DB_COMPACT = int(time.time())
# load data for shows from database
self.load_shows()
if self.srConfig.DEFAULT_PAGE not in ('home', 'schedule', 'history',
'news', 'IRC'):
self.srConfig.DEFAULT_PAGE = 'home'
# cleanup cache folder
for folder in ['mako', 'sessions', 'indexers']:
try:
shutil.rmtree(os.path.join(sickrage.CACHE_DIR, folder),
ignore_errors=True)
except Exception:
continue
# init anidb connection
if self.srConfig.USE_ANIDB:
try:
self.ADBA_CONNECTION = adba.Connection(
keepAlive=True,
log=lambda msg: self.srLogger.debug(
"AniDB: %s " % msg)).auth(self.srConfig.ANIDB_USERNAME,
self.srConfig.ANIDB_PASSWORD)
except Exception as e:
self.srLogger.warning("AniDB exception msg: %r " % repr(e))
if self.srConfig.WEB_PORT < 21 or self.srConfig.WEB_PORT > 65535:
self.srConfig.WEB_PORT = 8081
if not self.srConfig.WEB_COOKIE_SECRET:
self.srConfig.WEB_COOKIE_SECRET = generateCookieSecret()
# attempt to help prevent users from breaking links by using a bad url
if not self.srConfig.ANON_REDIRECT.endswith('?'):
self.srConfig.ANON_REDIRECT = ''
if not re.match(r'\d+\|[^|]+(?:\|[^|]+)*', self.srConfig.ROOT_DIRS):
self.srConfig.ROOT_DIRS = ''
self.srConfig.NAMING_FORCE_FOLDERS = check_force_season_folders()
if self.srConfig.NZB_METHOD not in ('blackhole', 'sabnzbd', 'nzbget'):
self.srConfig.NZB_METHOD = 'blackhole'
if self.srConfig.TORRENT_METHOD not in ('blackhole', 'utorrent',
'transmission', 'deluge',
'deluged', 'download_station',
'rtorrent', 'qbittorrent',
'mlnet', 'putio'):
self.srConfig.TORRENT_METHOD = 'blackhole'
if self.srConfig.PROPER_SEARCHER_INTERVAL not in ('15m', '45m', '90m',
'4h', 'daily'):
self.srConfig.PROPER_SEARCHER_INTERVAL = 'daily'
if self.srConfig.AUTOPOSTPROCESSOR_FREQ < self.srConfig.MIN_AUTOPOSTPROCESSOR_FREQ:
self.srConfig.AUTOPOSTPROCESSOR_FREQ = self.srConfig.MIN_AUTOPOSTPROCESSOR_FREQ
if self.srConfig.NAMECACHE_FREQ < self.srConfig.MIN_NAMECACHE_FREQ:
self.srConfig.NAMECACHE_FREQ = self.srConfig.MIN_NAMECACHE_FREQ
if self.srConfig.DAILY_SEARCHER_FREQ < self.srConfig.MIN_DAILY_SEARCHER_FREQ:
self.srConfig.DAILY_SEARCHER_FREQ = self.srConfig.MIN_DAILY_SEARCHER_FREQ
self.srConfig.MIN_BACKLOG_SEARCHER_FREQ = self.BACKLOGSEARCHER.get_backlog_cycle_time(
)
if self.srConfig.BACKLOG_SEARCHER_FREQ < self.srConfig.MIN_BACKLOG_SEARCHER_FREQ:
self.srConfig.BACKLOG_SEARCHER_FREQ = self.srConfig.MIN_BACKLOG_SEARCHER_FREQ
if self.srConfig.VERSION_UPDATER_FREQ < self.srConfig.MIN_VERSION_UPDATER_FREQ:
self.srConfig.VERSION_UPDATER_FREQ = self.srConfig.MIN_VERSION_UPDATER_FREQ
if self.srConfig.SHOWUPDATE_HOUR > 23:
self.srConfig.SHOWUPDATE_HOUR = 0
elif self.srConfig.SHOWUPDATE_HOUR < 0:
self.srConfig.SHOWUPDATE_HOUR = 0
if self.srConfig.SUBTITLE_SEARCHER_FREQ < self.srConfig.MIN_SUBTITLE_SEARCHER_FREQ:
self.srConfig.SUBTITLE_SEARCHER_FREQ = self.srConfig.MIN_SUBTITLE_SEARCHER_FREQ
if self.srConfig.SUBTITLES_LANGUAGES[0] == '':
self.srConfig.SUBTITLES_LANGUAGES = []
# add version checker job
self.srScheduler.add_job(
self.VERSIONUPDATER.run,
srIntervalTrigger(
**{
'hours': self.srConfig.VERSION_UPDATER_FREQ,
'min': self.srConfig.MIN_VERSION_UPDATER_FREQ
}),
name="VERSIONUPDATER",
id="VERSIONUPDATER")
# add network timezones updater job
self.srScheduler.add_job(update_network_dict,
srIntervalTrigger(**{'days': 1}),
name="TZUPDATER",
id="TZUPDATER")
# add show updater job
self.srScheduler.add_job(
self.SHOWUPDATER.run,
srIntervalTrigger(
**{
'days':
1,
'start_date':
datetime.datetime.now().replace(
hour=self.srConfig.SHOWUPDATE_HOUR)
}),
name="SHOWUPDATER",
id="SHOWUPDATER")
# add show next episode job
self.srScheduler.add_job(self.SHOWUPDATER.nextEpisode,
srIntervalTrigger(**{'hours': 1}),
name="SHOWNEXTEP",
id="SHOWNEXTEP")
# add daily search job
self.srScheduler.add_job(self.DAILYSEARCHER.run,
srIntervalTrigger(
**{
'minutes':
self.srConfig.DAILY_SEARCHER_FREQ,
'min':
self.srConfig.MIN_DAILY_SEARCHER_FREQ,
'start_date':
datetime.datetime.now() +
datetime.timedelta(minutes=4)
}),
name="DAILYSEARCHER",
id="DAILYSEARCHER")
# add backlog search job
self.srScheduler.add_job(
self.BACKLOGSEARCHER.run,
srIntervalTrigger(
**{
'minutes':
self.srConfig.BACKLOG_SEARCHER_FREQ,
'min':
self.srConfig.MIN_BACKLOG_SEARCHER_FREQ,
'start_date':
datetime.datetime.now() + datetime.timedelta(minutes=30)
}),
name="BACKLOG",
id="BACKLOG")
# add auto-postprocessing job
self.srScheduler.add_job(
self.AUTOPOSTPROCESSOR.run,
srIntervalTrigger(
**{
'minutes': self.srConfig.AUTOPOSTPROCESSOR_FREQ,
'min': self.srConfig.MIN_AUTOPOSTPROCESSOR_FREQ
}),
name="POSTPROCESSOR",
id="POSTPROCESSOR")
# add find proper job
self.srScheduler.add_job(
self.PROPERSEARCHER.run,
srIntervalTrigger(
**{
'minutes': {
'15m': 15,
'45m': 45,
'90m': 90,
'4h': 4 * 60,
'daily': 24 * 60
}[self.srConfig.PROPER_SEARCHER_INTERVAL]
}),
name="PROPERSEARCHER",
id="PROPERSEARCHER")
# add trakt.tv checker job
self.srScheduler.add_job(self.TRAKTSEARCHER.run,
srIntervalTrigger(**{'hours': 1}),
name="TRAKTSEARCHER",
id="TRAKTSEARCHER")
# add subtitles finder job
self.srScheduler.add_job(
self.SUBTITLESEARCHER.run,
srIntervalTrigger(
**{'hours': self.srConfig.SUBTITLE_SEARCHER_FREQ}),
name="SUBTITLESEARCHER",
id="SUBTITLESEARCHER")
# start scheduler service
self.srScheduler.start()
# Pause/Resume PROPERSEARCHER job
(self.srScheduler.get_job('PROPERSEARCHER').pause,
self.srScheduler.get_job('PROPERSEARCHER').resume
)[self.srConfig.DOWNLOAD_PROPERS]()
# Pause/Resume TRAKTSEARCHER job
(self.srScheduler.get_job('TRAKTSEARCHER').pause,
self.srScheduler.get_job('TRAKTSEARCHER').resume
)[self.srConfig.USE_TRAKT]()
# Pause/Resume SUBTITLESEARCHER job
(self.srScheduler.get_job('SUBTITLESEARCHER').pause,
self.srScheduler.get_job('SUBTITLESEARCHER').resume
)[self.srConfig.USE_SUBTITLES]()
# Pause/Resume POSTPROCESS job
(self.srScheduler.get_job('POSTPROCESSOR').pause,
self.srScheduler.get_job('POSTPROCESSOR').resume
)[self.srConfig.PROCESS_AUTOMATICALLY]()
# start queue's
self.SEARCHQUEUE.start()
self.SHOWQUEUE.start()
# start webserver
self.srWebServer.start()
self.srLogger.info("SiCKRAGE :: STARTED")
self.srLogger.info("SiCKRAGE :: VERSION:[{}]".format(
self.VERSIONUPDATER.version))
self.srLogger.info("SiCKRAGE :: CONFIG:[{}] [v{}]".format(
sickrage.CONFIG_FILE, self.srConfig.CONFIG_VERSION))
self.srLogger.info("SiCKRAGE :: URL:[{}://{}:{}/]".format(
('http', 'https')[self.srConfig.ENABLE_HTTPS],
self.srConfig.WEB_HOST, self.srConfig.WEB_PORT))
# launch browser window
if all(
[not sickrage.NOLAUNCH, sickrage.srCore.srConfig.LAUNCH_BROWSER]):
threading.Thread(
None, lambda: launch_browser(
('http', 'https')[sickrage.srCore.srConfig.ENABLE_HTTPS],
self.srConfig.WEB_HOST, sickrage.srCore.srConfig.WEB_PORT)
).start()
# start ioloop event handler
self.io_loop.start()
def shutdown(self):
if self.started:
self.started = False
self.srLogger.info('SiCKRAGE IS SHUTTING DOWN!!!')
# shutdown/restart webserver
self.srWebServer.shutdown()
# shutdown show queue
if self.SHOWQUEUE:
self.srLogger.debug("Shutting down show queue")
self.SHOWQUEUE.shutdown()
# shutdown search queue
if self.SEARCHQUEUE:
self.srLogger.debug("Shutting down search queue")
self.SEARCHQUEUE.shutdown()
# log out of ADBA
if sickrage.srCore.ADBA_CONNECTION:
self.srLogger.debug("Logging out ANIDB connection")
sickrage.srCore.ADBA_CONNECTION.logout()
# save all show and config settings
self.save_all()
# close databases
for db in [self.mainDB, self.cacheDB, self.failedDB]:
db.close()
# shutdown logging
self.srLogger.close()
# stop daemon process
if not sickrage.restart and sickrage.daemon: sickrage.daemon.stop()
def save_all(self):
# write all shows
self.srLogger.info("Saving all shows to the database")
for SHOW in self.SHOWLIST:
try:
SHOW.saveToDB()
except:
continue
# save config
self.srConfig.save()
def load_shows(self):
"""
Populates the showlist with shows from the database
"""
self.NAMECACHE.load()
for dbData in [
x['doc'] for x in self.mainDB.db.all('tv_shows', with_doc=True)
]:
try:
self.srLogger.debug("Loading data for show: [%s]",
dbData['show_name'])
show = TVShow(int(dbData['indexer']),
int(dbData['indexer_id']))
if not sickrage.DEVELOPER:
show.nextEpisode()
self.NAMECACHE.build(show)
self.SHOWLIST += [show]
except Exception as e:
self.srLogger.error("Show error in [%s]: %s" %
(dbData['location'], e.message))
return not record.msg.startswith(
'Adding') and not record.msg.startswith('Added')
logging.getLogger("apscheduler.scheduler").addFilter(AddedFilter())
import future_spider, spot_spider
all_crawlers = []
for spider in [future_spider, spot_spider]:
submodules = inspect.getmembers(spider, inspect.ismodule)
for submodule_name, submodule in submodules:
crawlers = inspect.getmembers(
submodule, lambda i: inspect.isfunction(i) and i.__name__.
startswith('crawl_'))
all_crawlers.extend(map(lambda i: i[1], crawlers))
scheduler = TornadoScheduler()
logging.info(
'爬取周期配置默认60分钟爬取一次, 如要指定1小时爬一次请在爬虫方法名最后加上 __60m, 目前支持 m 为单位的周期爬取, 储存、覆盖逻辑由爬虫自己实现'
)
for crawler in all_crawlers:
configuration = list(
filter(lambda i: re.match(r'(\d+)(m)', i),
crawler.__name__.split('__')))
if not configuration:
try:
crawler()
except:
pass
scheduler.add_job(spider_wrapper(crawler),
'interval',
class Core(object):
def __init__(self):
self.started = False
# process id
self.PID = os.getpid()
# cpu count
self.CPU_COUNT = cpu_count()
# generate notifiers dict
self.notifiersDict = AttrDict(libnotify=LibnotifyNotifier(),
kodi_notifier=KODINotifier(),
plex_notifier=PLEXNotifier(),
emby_notifier=EMBYNotifier(),
nmj_notifier=NMJNotifier(),
nmjv2_notifier=NMJv2Notifier(),
synoindex_notifier=synoIndexNotifier(),
synology_notifier=synologyNotifier(),
pytivo_notifier=pyTivoNotifier(),
growl_notifier=GrowlNotifier(),
prowl_notifier=ProwlNotifier(),
libnotify_notifier=LibnotifyNotifier(),
pushover_notifier=PushoverNotifier(),
boxcar_notifier=BoxcarNotifier(),
boxcar2_notifier=Boxcar2Notifier(),
nma_notifier=NMA_Notifier(),
pushalot_notifier=PushalotNotifier(),
pushbullet_notifier=PushbulletNotifier(),
freemobile_notifier=FreeMobileNotifier(),
twitter_notifier=TwitterNotifier(),
trakt_notifier=TraktNotifier(),
email_notifier=EmailNotifier())
# generate metadata providers dict
self.metadataProviderDict = get_metadata_generator_dict()
# generate providers dict
self.providersDict = providersDict()
# init notification queue
self.srNotifications = Notifications()
# init logger
self.srLogger = srLogger()
# init config
self.srConfig = srConfig()
# init scheduler service
self.srScheduler = TornadoScheduler()
# init web server
self.srWebServer = srWebServer()
# init web client session
self.srWebSession = srSession()
# google api
self.googleAuth = googleAuth()
# name cache
self.NAMECACHE = srNameCache()
# queues
self.SHOWQUEUE = srShowQueue()
self.SEARCHQUEUE = srSearchQueue()
# updaters
self.VERSIONUPDATER = srVersionUpdater()
self.SHOWUPDATER = srShowUpdater()
# searchers
self.DAILYSEARCHER = srDailySearcher()
self.BACKLOGSEARCHER = srBacklogSearcher()
self.PROPERSEARCHER = srProperSearcher()
self.TRAKTSEARCHER = srTraktSearcher()
self.SUBTITLESEARCHER = srSubtitleSearcher()
# auto postprocessor
self.AUTOPOSTPROCESSOR = srPostProcessor()
# sickrage version
self.NEWEST_VERSION = None
self.NEWEST_VERSION_STRING = None
# anidb connection
self.ADBA_CONNECTION = None
# show list
self.SHOWLIST = []
def start(self):
self.started = True
# thread name
threading.currentThread().setName('CORE')
# Check if we need to perform a restore first
if os.path.exists(
os.path.abspath(os.path.join(sickrage.DATA_DIR, 'restore'))):
success = restoreSR(
os.path.abspath(os.path.join(sickrage.DATA_DIR, 'restore')),
sickrage.DATA_DIR)
print("Restoring SiCKRAGE backup: %s!\n" %
("FAILED", "SUCCESSFUL")[success])
if success:
shutil.rmtree(os.path.abspath(
os.path.join(sickrage.DATA_DIR, 'restore')),
ignore_errors=True)
# migrate old database file names to new ones
if os.path.isfile(
os.path.abspath(os.path.join(sickrage.DATA_DIR,
'sickbeard.db'))):
if os.path.isfile(os.path.join(sickrage.DATA_DIR, 'sickrage.db')):
helpers.moveFile(
os.path.join(sickrage.DATA_DIR, 'sickrage.db'),
os.path.join(
sickrage.DATA_DIR, '{}.bak-{}'.format(
'sickrage.db',
datetime.datetime.now().strftime(
'%Y%m%d_%H%M%S'))))
helpers.moveFile(
os.path.abspath(os.path.join(sickrage.DATA_DIR,
'sickbeard.db')),
os.path.abspath(os.path.join(sickrage.DATA_DIR,
'sickrage.db')))
# load config
self.srConfig.load()
# set socket timeout
socket.setdefaulttimeout(self.srConfig.SOCKET_TIMEOUT)
# setup logger settings
self.srLogger.logSize = self.srConfig.LOG_SIZE
self.srLogger.logNr = self.srConfig.LOG_NR
self.srLogger.debugLogging = sickrage.DEBUG
self.srLogger.consoleLogging = not sickrage.QUITE
self.srLogger.logFile = self.srConfig.LOG_FILE
# start logger
self.srLogger.start()
# Check available space
try:
total_space, available_space = getFreeSpace(sickrage.DATA_DIR)
if available_space < 100:
self.srLogger.error(
'Shutting down as SiCKRAGE needs some space to work. You\'ll get corrupted data otherwise. Only %sMB left',
available_space)
sickrage.restart = False
return
except:
self.srLogger.error('Failed getting diskspace: %s',
traceback.format_exc())
# perform database startup actions
for db in [MainDB, CacheDB, FailedDB]:
# initialize the database
db().initialize()
# migrate the database
db().migrate()
# compact the main database
db().compact()
# load data for shows from database
self.load_shows()
# build name cache
self.NAMECACHE.build()
if self.srConfig.DEFAULT_PAGE not in ('home', 'schedule', 'history',
'news', 'IRC'):
self.srConfig.DEFAULT_PAGE = 'home'
# cleanup cache folder
for folder in ['mako', 'sessions', 'indexers']:
try:
shutil.rmtree(os.path.join(self.srConfig.CACHE_DIR, folder),
ignore_errors=True)
except Exception:
continue
# init anidb connection
if not self.srConfig.USE_ANIDB:
try:
self.ADBA_CONNECTION = adba.Connection(
keepAlive=True,
log=lambda msg: self.srLogger.debug(
"AniDB: %s " % msg)).auth(self.srConfig.ANIDB_USERNAME,
self.srConfig.ANIDB_PASSWORD)
except Exception as e:
self.srLogger.warning("AniDB exception msg: %r " % repr(e))
if self.srConfig.WEB_PORT < 21 or self.srConfig.WEB_PORT > 65535:
self.srConfig.WEB_PORT = 8081
if not self.srConfig.WEB_COOKIE_SECRET:
self.srConfig.WEB_COOKIE_SECRET = generateCookieSecret()
# attempt to help prevent users from breaking links by using a bad url
if not self.srConfig.ANON_REDIRECT.endswith('?'):
self.srConfig.ANON_REDIRECT = ''
if not re.match(r'\d+\|[^|]+(?:\|[^|]+)*', self.srConfig.ROOT_DIRS):
self.srConfig.ROOT_DIRS = ''
self.srConfig.NAMING_FORCE_FOLDERS = check_force_season_folders()
if self.srConfig.NZB_METHOD not in ('blackhole', 'sabnzbd', 'nzbget'):
self.srConfig.NZB_METHOD = 'blackhole'
if self.srConfig.TORRENT_METHOD not in ('blackhole', 'utorrent',
'transmission', 'deluge',
'deluged', 'download_station',
'rtorrent', 'qbittorrent',
'mlnet', 'putio'):
self.srConfig.TORRENT_METHOD = 'blackhole'
if self.srConfig.PROPER_SEARCHER_INTERVAL not in ('15m', '45m', '90m',
'4h', 'daily'):
self.srConfig.PROPER_SEARCHER_INTERVAL = 'daily'
if self.srConfig.AUTOPOSTPROCESSOR_FREQ < self.srConfig.MIN_AUTOPOSTPROCESSOR_FREQ:
self.srConfig.AUTOPOSTPROCESSOR_FREQ = self.srConfig.MIN_AUTOPOSTPROCESSOR_FREQ
if self.srConfig.NAMECACHE_FREQ < self.srConfig.MIN_NAMECACHE_FREQ:
self.srConfig.NAMECACHE_FREQ = self.srConfig.MIN_NAMECACHE_FREQ
if self.srConfig.DAILY_SEARCHER_FREQ < self.srConfig.MIN_DAILY_SEARCHER_FREQ:
self.srConfig.DAILY_SEARCHER_FREQ = self.srConfig.MIN_DAILY_SEARCHER_FREQ
self.srConfig.MIN_BACKLOG_SEARCHER_FREQ = get_backlog_cycle_time()
if self.srConfig.BACKLOG_SEARCHER_FREQ < self.srConfig.MIN_BACKLOG_SEARCHER_FREQ:
self.srConfig.BACKLOG_SEARCHER_FREQ = self.srConfig.MIN_BACKLOG_SEARCHER_FREQ
if self.srConfig.VERSION_UPDATER_FREQ < self.srConfig.MIN_VERSION_UPDATER_FREQ:
self.srConfig.VERSION_UPDATER_FREQ = self.srConfig.MIN_VERSION_UPDATER_FREQ
if self.srConfig.SHOWUPDATE_HOUR > 23:
self.srConfig.SHOWUPDATE_HOUR = 0
elif self.srConfig.SHOWUPDATE_HOUR < 0:
self.srConfig.SHOWUPDATE_HOUR = 0
if self.srConfig.SUBTITLE_SEARCHER_FREQ < self.srConfig.MIN_SUBTITLE_SEARCHER_FREQ:
self.srConfig.SUBTITLE_SEARCHER_FREQ = self.srConfig.MIN_SUBTITLE_SEARCHER_FREQ
self.srConfig.NEWS_LATEST = self.srConfig.NEWS_LAST_READ
if self.srConfig.SUBTITLES_LANGUAGES[0] == '':
self.srConfig.SUBTITLES_LANGUAGES = []
# initialize metadata_providers
for cur_metadata_tuple in [
(self.srConfig.METADATA_KODI, kodi),
(self.srConfig.METADATA_KODI_12PLUS, kodi_12plus),
(self.srConfig.METADATA_MEDIABROWSER, mediabrowser),
(self.srConfig.METADATA_PS3, ps3),
(self.srConfig.METADATA_WDTV, wdtv),
(self.srConfig.METADATA_TIVO, tivo),
(self.srConfig.METADATA_MEDE8ER, mede8er)
]:
(cur_metadata_config, cur_metadata_class) = cur_metadata_tuple
tmp_provider = cur_metadata_class.metadata_class()
tmp_provider.set_config(cur_metadata_config)
self.metadataProviderDict[tmp_provider.name] = tmp_provider
# add version checker job
self.srScheduler.add_job(
self.VERSIONUPDATER.run,
srIntervalTrigger(
**{
'hours': self.srConfig.VERSION_UPDATER_FREQ,
'min': self.srConfig.MIN_VERSION_UPDATER_FREQ
}),
name="VERSIONUPDATER",
id="VERSIONUPDATER")
# add network timezones updater job
self.srScheduler.add_job(update_network_dict,
srIntervalTrigger(**{'days': 1}),
name="TZUPDATER",
id="TZUPDATER")
# add namecache updater job
self.srScheduler.add_job(
self.NAMECACHE.run,
srIntervalTrigger(
**{
'minutes': self.srConfig.NAMECACHE_FREQ,
'min': self.srConfig.MIN_NAMECACHE_FREQ
}),
name="NAMECACHE",
id="NAMECACHE")
# add show updater job
self.srScheduler.add_job(
self.SHOWUPDATER.run,
srIntervalTrigger(
**{
'hours':
1,
'start_date':
datetime.datetime.now().replace(
hour=self.srConfig.SHOWUPDATE_HOUR)
}),
name="SHOWUPDATER",
id="SHOWUPDATER")
# add daily search job
self.srScheduler.add_job(
self.DAILYSEARCHER.run,
srIntervalTrigger(
**{
'minutes': self.srConfig.DAILY_SEARCHER_FREQ,
'min': self.srConfig.MIN_DAILY_SEARCHER_FREQ
}),
name="DAILYSEARCHER",
id="DAILYSEARCHER")
# add backlog search job
self.srScheduler.add_job(
self.BACKLOGSEARCHER.run,
srIntervalTrigger(
**{
'minutes': self.srConfig.BACKLOG_SEARCHER_FREQ,
'min': self.srConfig.MIN_BACKLOG_SEARCHER_FREQ
}),
name="BACKLOG",
id="BACKLOG")
# add auto-postprocessing job
self.srScheduler.add_job(
self.AUTOPOSTPROCESSOR.run,
srIntervalTrigger(
**{
'minutes': self.srConfig.AUTOPOSTPROCESSOR_FREQ,
'min': self.srConfig.MIN_AUTOPOSTPROCESSOR_FREQ
}),
name="POSTPROCESSOR",
id="POSTPROCESSOR")
# add find proper job
self.srScheduler.add_job(
self.PROPERSEARCHER.run,
srIntervalTrigger(
**{
'minutes': {
'15m': 15,
'45m': 45,
'90m': 90,
'4h': 4 * 60,
'daily': 24 * 60
}[self.srConfig.PROPER_SEARCHER_INTERVAL]
}),
name="PROPERSEARCHER",
id="PROPERSEARCHER")
# add trakt.tv checker job
self.srScheduler.add_job(self.TRAKTSEARCHER.run,
srIntervalTrigger(**{'hours': 1}),
name="TRAKTSEARCHER",
id="TRAKTSEARCHER")
# add subtitles finder job
self.srScheduler.add_job(
self.SUBTITLESEARCHER.run,
srIntervalTrigger(
**{'hours': self.srConfig.SUBTITLE_SEARCHER_FREQ}),
name="SUBTITLESEARCHER",
id="SUBTITLESEARCHER")
# start scheduler service
self.srScheduler.start()
# Pause/Resume PROPERSEARCHER job
(self.srScheduler.get_job('PROPERSEARCHER').pause,
self.srScheduler.get_job('PROPERSEARCHER').resume
)[self.srConfig.DOWNLOAD_PROPERS]()
# Pause/Resume TRAKTSEARCHER job
(self.srScheduler.get_job('TRAKTSEARCHER').pause,
self.srScheduler.get_job('TRAKTSEARCHER').resume
)[self.srConfig.USE_TRAKT]()
# Pause/Resume SUBTITLESEARCHER job
(self.srScheduler.get_job('SUBTITLESEARCHER').pause,
self.srScheduler.get_job('SUBTITLESEARCHER').resume
)[self.srConfig.USE_SUBTITLES]()
# Pause/Resume POSTPROCESS job
(self.srScheduler.get_job('POSTPROCESSOR').pause,
self.srScheduler.get_job('POSTPROCESSOR').resume
)[self.srConfig.PROCESS_AUTOMATICALLY]()
# start queue's
self.SEARCHQUEUE.start()
self.SHOWQUEUE.start()
# start webserver
self.srWebServer.start()
# start ioloop event handler
IOLoop.current().start()
def shutdown(self):
if self.started:
self.started = False
self.srLogger.info('SiCKRAGE IS SHUTTING DOWN!!!')
# shutdown/restart webserver
self.srWebServer.shutdown()
# shutdown scheduler
self.srLogger.info("Shutting down scheduler")
self.srScheduler.shutdown()
# shutdown show queue
if self.SHOWQUEUE:
self.srLogger.info("Shutting down show queue")
self.SHOWQUEUE.shutdown()
# shutdown search queue
if self.SEARCHQUEUE:
self.srLogger.info("Shutting down search queue")
self.SEARCHQUEUE.shutdown()
# log out of ADBA
if sickrage.srCore.ADBA_CONNECTION:
self.srLogger.info("Logging out ANIDB connection")
sickrage.srCore.ADBA_CONNECTION.logout()
# save all show and config settings
self.save_all()
# shutdown logging
self.srLogger.shutdown()
# delete pid file
if sickrage.DAEMONIZE:
sickrage.delpid(sickrage.PID_FILE)
def save_all(self):
# write all shows
self.srLogger.info("Saving all shows to the database")
for SHOW in self.SHOWLIST:
try:
SHOW.saveToDB()
except:
continue
# save config
self.srConfig.save()
def load_shows(self):
"""
Populates the showlist with shows from the database
"""
for dbData in [
x['doc'] for x in MainDB().db.all('tv_shows', with_doc=True)
]:
try:
self.srLogger.debug("Loading data for show: [%s]",
dbData['show_name'])
show = TVShow(int(dbData['indexer']),
int(dbData['indexer_id']))
show.nextEpisode()
self.SHOWLIST += [show]
except Exception as e:
self.srLogger.error("Show error in [%s]: %s" %
(dbData['location'], e.message))
from apscheduler.schedulers.tornado import TornadoScheduler
from tornado import websocket, gen, httpclient, ioloop, web
from alphabot import help
from alphabot import memory
DEFAULT_SCRIPT_DIR = 'default-scripts'
DEBUG_CHANNEL = os.getenv('DEBUG_CHANNEL', 'alphabot')
WEB_PORT = int(os.getenv('WEB_PORT', 8000))
WEB_PORT_SSL = int(os.getenv('WEB_PORT_SSL', 8443))
log = logging.getLogger(__name__)
log_level = logging.getLevelName(os.getenv('LOG_LEVEL', 'INFO'))
log.setLevel(log_level)
scheduler = TornadoScheduler()
scheduler.start()
class AlphaBotException(Exception):
"""Top of hierarchy for all alphabot failures."""
class CoreException(AlphaBotException):
"""Used to signify a failure in the robot's core."""
class InvalidOptions(AlphaBotException):
"""Robot failed because input options were somehow broken."""
def tornado_scheduler(io_loop):
scheduler = TornadoScheduler(io_loop=io_loop)
scheduler.start(paused=True)
yield scheduler
scheduler.shutdown(False)
# get all routes which belong to user or profile
# TODO - include employees
self.write( {"lol":"lmao"} )
@Route(r"/signals")
class SimpleHandler6(tornado.web.RequestHandler):
def get(self):
# TODO - get all routes which belong to
self.write( {"lol":"lmao"} )
app = tornado.web.Application(Route.routes() + [
(r'/send_message', SendMessageHandler)
] + sockjs.tornado.SockJSRouter(MessageHandler, '/sockjs').urls)
if __name__ == "__main__":
app.listen(8988)
#app.listen(8000)
#app.listen(5000)
#tornado.ioloop.IOLoop.current().add_callback(print_changes)
tornado.ioloop.IOLoop.current().add_callback(company_name_to_domain_changes)
tornado.ioloop.IOLoop.current().add_callback(trigger_changes)
scheduler = TornadoScheduler()
scheduler.add_job(AsyncCompanyNameResearch().start, 'interval', seconds=1)
scheduler.add_job(AsyncCompanyResearch().start_company_info_research, 'interval', seconds=1)
scheduler.add_job(AsyncCompanyResearch().start_employee_research, 'interval', seconds=1)
scheduler.add_job(AsyncCompanyResearch().start_email_pattern_research, 'interval', seconds=1)
scheduler.start()
tornado.ioloop.IOLoop.current().start()
def start(self):
self.started = True
self.io_loop = IOLoop.current()
# thread name
threading.currentThread().setName('CORE')
# patch modules with encoding kludge
patch_modules()
# init core classes
self.notifier_providers = NotifierProviders()
self.metadata_providers = MetadataProviders()
self.search_providers = SearchProviders()
self.log = Logger()
self.config = Config()
self.alerts = Notifications()
self.main_db = MainDB()
self.cache_db = CacheDB()
self.scheduler = TornadoScheduler()
self.wserver = WebServer()
self.name_cache = NameCache()
self.show_queue = ShowQueue()
self.search_queue = SearchQueue()
self.postprocessor_queue = PostProcessorQueue()
self.event_queue = EventQueue()
self.version_updater = VersionUpdater()
self.show_updater = ShowUpdater()
self.tz_updater = TimeZoneUpdater()
self.rsscache_updater = RSSCacheUpdater()
self.daily_searcher = DailySearcher()
self.failed_snatch_searcher = FailedSnatchSearcher()
self.backlog_searcher = BacklogSearcher()
self.proper_searcher = ProperSearcher()
self.trakt_searcher = TraktSearcher()
self.subtitle_searcher = SubtitleSearcher()
self.auto_postprocessor = AutoPostProcessor()
self.upnp_client = UPNPClient()
self.quicksearch_cache = QuicksearchCache()
# setup oidc client
realm = KeycloakRealm(server_url='https://auth.sickrage.ca',
realm_name='sickrage')
self.oidc_client = realm.open_id_connect(
client_id='sickrage-app',
client_secret='5d4710b2-ca70-4d39-b5a3-0705e2c5e703')
# Check if we need to perform a restore first
if os.path.exists(
os.path.abspath(os.path.join(self.data_dir, 'restore'))):
success = restoreSR(
os.path.abspath(os.path.join(self.data_dir, 'restore')),
self.data_dir)
print("Restoring SiCKRAGE backup: %s!\n" %
("FAILED", "SUCCESSFUL")[success])
if success:
shutil.rmtree(os.path.abspath(
os.path.join(self.data_dir, 'restore')),
ignore_errors=True)
# migrate old database file names to new ones
if os.path.isfile(
os.path.abspath(os.path.join(self.data_dir, 'sickbeard.db'))):
if os.path.isfile(os.path.join(self.data_dir, 'sickrage.db')):
helpers.move_file(
os.path.join(self.data_dir, 'sickrage.db'),
os.path.join(
self.data_dir, '{}.bak-{}'.format(
'sickrage.db',
datetime.datetime.now().strftime(
'%Y%m%d_%H%M%S'))))
helpers.move_file(
os.path.abspath(os.path.join(self.data_dir, 'sickbeard.db')),
os.path.abspath(os.path.join(self.data_dir, 'sickrage.db')))
# load config
self.config.load()
# set language
self.config.change_gui_lang(self.config.gui_lang)
# set socket timeout
socket.setdefaulttimeout(self.config.socket_timeout)
# setup logger settings
self.log.logSize = self.config.log_size
self.log.logNr = self.config.log_nr
self.log.logFile = os.path.join(self.data_dir, 'logs', 'sickrage.log')
self.log.debugLogging = self.config.debug
self.log.consoleLogging = not self.quiet
# start logger
self.log.start()
# user agent
if self.config.random_user_agent:
self.user_agent = UserAgent().random
urlparse.uses_netloc.append('scgi')
urllib.FancyURLopener.version = self.user_agent
# set torrent client web url
torrent_webui_url(True)
# Check available space
try:
total_space, available_space = getFreeSpace(self.data_dir)
if available_space < 100:
self.log.error(
'Shutting down as SiCKRAGE needs some space to work. You\'ll get corrupted data '
'otherwise. Only %sMB left', available_space)
return
except Exception:
self.log.error('Failed getting disk space: %s',
traceback.format_exc())
# perform database startup actions
for db in [self.main_db, self.cache_db]:
# initialize database
db.initialize()
# check integrity of database
db.check_integrity()
# migrate database
db.migrate()
# misc database cleanups
db.cleanup()
# upgrade database
db.upgrade()
# compact main database
if self.config.last_db_compact < time.time() - 604800: # 7 days
self.main_db.compact()
self.config.last_db_compact = int(time.time())
# load name cache
self.name_cache.load()
# load data for shows from database
self.load_shows()
if self.config.default_page not in ('schedule', 'history', 'IRC'):
self.config.default_page = 'home'
# cleanup cache folder
for folder in ['mako', 'sessions', 'indexers']:
try:
shutil.rmtree(os.path.join(sickrage.app.cache_dir, folder),
ignore_errors=True)
except Exception:
continue
if self.config.web_port < 21 or self.config.web_port > 65535:
self.config.web_port = 8081
if not self.config.web_cookie_secret:
self.config.web_cookie_secret = generate_secret()
# attempt to help prevent users from breaking links by using a bad url
if not self.config.anon_redirect.endswith('?'):
self.config.anon_redirect = ''
if not re.match(r'\d+\|[^|]+(?:\|[^|]+)*', self.config.root_dirs):
self.config.root_dirs = ''
self.config.naming_force_folders = check_force_season_folders()
if self.config.nzb_method not in ('blackhole', 'sabnzbd', 'nzbget'):
self.config.nzb_method = 'blackhole'
if self.config.torrent_method not in ('blackhole', 'utorrent',
'transmission', 'deluge',
'deluged', 'download_station',
'rtorrent', 'qbittorrent',
'mlnet', 'putio'):
self.config.torrent_method = 'blackhole'
if self.config.autopostprocessor_freq < self.config.min_autopostprocessor_freq:
self.config.autopostprocessor_freq = self.config.min_autopostprocessor_freq
if self.config.daily_searcher_freq < self.config.min_daily_searcher_freq:
self.config.daily_searcher_freq = self.config.min_daily_searcher_freq
if self.config.backlog_searcher_freq < self.config.min_backlog_searcher_freq:
self.config.backlog_searcher_freq = self.config.min_backlog_searcher_freq
if self.config.version_updater_freq < self.config.min_version_updater_freq:
self.config.version_updater_freq = self.config.min_version_updater_freq
if self.config.subtitle_searcher_freq < self.config.min_subtitle_searcher_freq:
self.config.subtitle_searcher_freq = self.config.min_subtitle_searcher_freq
if self.config.failed_snatch_age < self.config.min_failed_snatch_age:
self.config.failed_snatch_age = self.config.min_failed_snatch_age
if self.config.proper_searcher_interval not in ('15m', '45m', '90m',
'4h', 'daily'):
self.config.proper_searcher_interval = 'daily'
if self.config.showupdate_hour < 0 or self.config.showupdate_hour > 23:
self.config.showupdate_hour = 0
# add version checker job
self.scheduler.add_job(self.version_updater.run,
IntervalTrigger(
hours=self.config.version_updater_freq, ),
name=self.version_updater.name,
id=self.version_updater.name)
# add network timezones updater job
self.scheduler.add_job(self.tz_updater.run,
IntervalTrigger(days=1, ),
name=self.tz_updater.name,
id=self.tz_updater.name)
# add show updater job
self.scheduler.add_job(self.show_updater.run,
IntervalTrigger(
days=1,
start_date=datetime.datetime.now().replace(
hour=self.config.showupdate_hour)),
name=self.show_updater.name,
id=self.show_updater.name)
# add rss cache updater job
self.scheduler.add_job(self.rsscache_updater.run,
IntervalTrigger(minutes=15, ),
name=self.rsscache_updater.name,
id=self.rsscache_updater.name)
# add daily search job
self.scheduler.add_job(
self.daily_searcher.run,
IntervalTrigger(minutes=self.config.daily_searcher_freq,
start_date=datetime.datetime.now() +
datetime.timedelta(minutes=4)),
name=self.daily_searcher.name,
id=self.daily_searcher.name)
# add failed snatch search job
self.scheduler.add_job(
self.failed_snatch_searcher.run,
IntervalTrigger(hours=1,
start_date=datetime.datetime.now() +
datetime.timedelta(minutes=4)),
name=self.failed_snatch_searcher.name,
id=self.failed_snatch_searcher.name)
# add backlog search job
self.scheduler.add_job(
self.backlog_searcher.run,
IntervalTrigger(minutes=self.config.backlog_searcher_freq,
start_date=datetime.datetime.now() +
datetime.timedelta(minutes=30)),
name=self.backlog_searcher.name,
id=self.backlog_searcher.name)
# add auto-postprocessing job
self.scheduler.add_job(
self.auto_postprocessor.run,
IntervalTrigger(minutes=self.config.autopostprocessor_freq),
name=self.auto_postprocessor.name,
id=self.auto_postprocessor.name)
# add find proper job
self.scheduler.add_job(
self.proper_searcher.run,
IntervalTrigger(minutes={
'15m': 15,
'45m': 45,
'90m': 90,
'4h': 4 * 60,
'daily': 24 * 60
}[self.config.proper_searcher_interval]),
name=self.proper_searcher.name,
id=self.proper_searcher.name)
# add trakt.tv checker job
self.scheduler.add_job(self.trakt_searcher.run,
IntervalTrigger(hours=1),
name=self.trakt_searcher.name,
id=self.trakt_searcher.name)
# add subtitles finder job
self.scheduler.add_job(
self.subtitle_searcher.run,
IntervalTrigger(hours=self.config.subtitle_searcher_freq),
name=self.subtitle_searcher.name,
id=self.subtitle_searcher.name)
# add upnp client job
self.scheduler.add_job(
self.upnp_client.run,
IntervalTrigger(seconds=self.upnp_client._nat_portmap_lifetime),
name=self.upnp_client.name,
id=self.upnp_client.name)
# add namecache update job
self.scheduler.add_job(self.name_cache.build_all,
IntervalTrigger(days=1, ),
name=self.name_cache.name,
id=self.name_cache.name)
# start scheduler service
self.scheduler.start()
# start queue's
self.search_queue.start()
self.show_queue.start()
self.postprocessor_queue.start()
self.event_queue.start()
# fire off startup events
self.event_queue.fire_event(self.name_cache.build_all)
self.event_queue.fire_event(self.version_updater.run)
self.event_queue.fire_event(self.tz_updater.run)
# start webserver
self.wserver.start()
# launch browser window
if all(
[not sickrage.app.no_launch, sickrage.app.config.launch_browser]):
self.event_queue.fire_event(lambda: launch_browser(
('http', 'https')[sickrage.app.config.enable_https], sickrage.
app.config.web_host, sickrage.app.config.web_port))
# start ioloop
self.io_loop.start()
def main():
# Running APScheduler
aps = TornadoScheduler()
aps.add_jobstore('mongodb', collection='example_jobs')
aps.remove_all_jobs()
aps.add_job(tick, 'interval', seconds=3)
aps.add_job(tick, 'interval', seconds=3)
aps.add_job(tick, 'interval', seconds=3)
aps.start()
# Running server
app = TornadoApplication()
app.listen(options.port)
tornado.ioloop.IOLoop.current().start()
from tornado.ioloop import IOLoop
from tornado.web import Application, StaticFileHandler
from tornado.httpserver import HTTPServer
from cfg import service
from controller.action import *
from controller.open_action import *
from controller.pan_manage_action import *
from controller.user_action import *
from controller.pro_action import ProductHandler
from controller.middle_ware import get_middleware
from controller.async_action import AsyncHandler
from controller.main_action import MainHandler
from controller.wx.wxget import WXAppGet
from controller.wx.wxput import WXAppPut
from utils import log as logger
scheduler = TornadoScheduler()
scheduler.start()
guest_user = None
context = dict(guest=None)
def update_sys_cfg(release=True):
try:
open_service.sync_cfg()
open_service.sync_tags()
context['guest'] = open_service.guest_user()
except Exception as e:
print("update_sys_cfg err:", e)
pass
def main():
application = tornado.web.Application([
(r'/', IndexHandler),
(r'/dash', DashHandler),
(r'/test', TestHandler),
(r'/google', AnalyticsHandler),
(r'/assets/(.*)', tornado.web.StaticFileHandler, {'path': './assets'},),
(r'/ws/', WebSocketHandler)
])
parse_command_line()
application.listen(options.port)
sched = TornadoScheduler(daemon=True)
atexit.register(lambda: sched.shutdown())
sched.add_job(social_media_fetcher.instagram_counts, 'cron', minute="*/1")
sched.add_job(social_media_fetcher.twitter_counts, 'cron', minute="*/1")
sched.add_job(social_media_fetcher.pinterest_counts, 'cron', minute="*/5")
sched.add_job(social_media_fetcher.youtube_counts, 'cron', minute="*/5")
sched.add_job(social_media_fetcher.facebook_counts, 'cron', minute="*/1")
# todo reinstate when keys inserted
# sched.add_job(social_media_fetcher.linkedin_count, 'cron', minute="*/5")
# Google Analytics importer
sched.add_job(analytic_fetcher.get_results, 'cron', hour="1", minute="1")
sched.start()
tornado.ioloop.IOLoop.instance().start()
from apscheduler.schedulers.tornado import TornadoScheduler
from tornado import websocket, gen, httpclient, ioloop, web
from alphabot import help
from alphabot import memory
DEFAULT_SCRIPT_DIR = 'default-scripts'
DEBUG_CHANNEL = os.getenv('DEBUG_CHANNEL', 'alphabot')
WEB_PORT = int(os.getenv('WEB_PORT', 8000))
WEB_PORT_SSL = int(os.getenv('WEB_PORT_SSL', 8443))
log = logging.getLogger(__name__)
log_level = logging.getLevelName(os.getenv('LOG_LEVEL', 'INFO'))
log.setLevel(log_level)
scheduler = TornadoScheduler()
scheduler.start()
class AlphaBotException(Exception):
"""Top of hierarchy for all alphabot failures."""
class CoreException(AlphaBotException):
"""Used to signify a failure in the robot's core."""
class InvalidOptions(AlphaBotException):
"""Robot failed because input options were somehow broken."""
def __init__(self):
self.started = False
# process id
self.PID = os.getpid()
# generate notifiers dict
self.notifiersDict = AttrDict(
libnotify=LibnotifyNotifier(),
kodi_notifier=KODINotifier(),
plex_notifier=PLEXNotifier(),
emby_notifier=EMBYNotifier(),
nmj_notifier=NMJNotifier(),
nmjv2_notifier=NMJv2Notifier(),
synoindex_notifier=synoIndexNotifier(),
synology_notifier=synologyNotifier(),
pytivo_notifier=pyTivoNotifier(),
growl_notifier=GrowlNotifier(),
prowl_notifier=ProwlNotifier(),
libnotify_notifier=LibnotifyNotifier(),
pushover_notifier=PushoverNotifier(),
boxcar_notifier=BoxcarNotifier(),
boxcar2_notifier=Boxcar2Notifier(),
nma_notifier=NMA_Notifier(),
pushalot_notifier=PushalotNotifier(),
pushbullet_notifier=PushbulletNotifier(),
freemobile_notifier=FreeMobileNotifier(),
twitter_notifier=TwitterNotifier(),
trakt_notifier=TraktNotifier(),
email_notifier=EmailNotifier()
)
# generate metadata providers dict
self.metadataProviderDict = get_metadata_generator_dict()
# generate providers dict
self.providersDict = providersDict()
# init notification queue
self.srNotifications = Notifications()
# init logger
self.srLogger = srLogger()
# init config
self.srConfig = srConfig()
# init scheduler service
self.srScheduler = TornadoScheduler()
# init web server
self.srWebServer = srWebServer()
# init web client session
self.srWebSession = srSession()
# google api
self.googleAuth = googleAuth()
# name cache
self.NAMECACHE = srNameCache()
# queues
self.SHOWQUEUE = srShowQueue()
self.SEARCHQUEUE = srSearchQueue()
# updaters
self.VERSIONUPDATER = srVersionUpdater()
self.SHOWUPDATER = srShowUpdater()
# searchers
self.DAILYSEARCHER = srDailySearcher()
self.BACKLOGSEARCHER = srBacklogSearcher()
self.PROPERSEARCHER = srProperSearcher()
self.TRAKTSEARCHER = srTraktSearcher()
self.SUBTITLESEARCHER = srSubtitleSearcher()
# auto postprocessor
self.AUTOPOSTPROCESSOR = srPostProcessor()
# sickrage version
self.NEWEST_VERSION = None
self.NEWEST_VERSION_STRING = None
# anidb connection
self.ADBA_CONNECTION = None
# show list
self.SHOWLIST = []
]
)
PORT = 8888
if __name__ == "__main__":
import logging
logFormat = "%(asctime)s:%(levelname)s:%(funcName)s:%(message)s"
logging.basicConfig(format=logFormat)
# options.options['log_file_prefix'].set('/opt/logs/my_app.log')
# options.parse_command_line() # read the command-line to get log_file_prefix from there
_log = logging.getLogger("tornado.application")
_log.critical("\n--------------------------------------------------------------------------\nApplication starting")
_log.setLevel(LOG_LEVEL) # 50 critical, 40 error, 30 warning, 20 info, 10 debug
_log.critical("Current log-level : %s", logging.getLevelName(_log.getEffectiveLevel()))
# _setupsql() # Prepare sqls
# Setup the server
application.listen(PORT)
_log.info("Listening on port %d" % PORT)
_dbHelper = DBHelperSQLITE()
_scheduler = TornadoScheduler()
_scheduler.start()
ioloop = IOLoop.instance()
_bgtask = TaskRunner(ioloop)
TaskRunner.bgsetup(_bgtask)
_log.info("Background task is set up, starting ioloop")
ioloop.start()
SRC_DIR, log_file),
shell=True)
def crawl_guba():
# 股吧爬虫
log_file = os.path.join(LOG_DIR, 'guba.log')
subprocess.call(
'cd {0}/websitespider && scrapy crawl guba --logfile={1}'.format(
SRC_DIR, log_file),
shell=True)
if __name__ == '__main__':
# -c scray_wechat
scheduler = TornadoScheduler()
# cnmn 中国有色网 的爬虫 一分钟
scheduler.add_job(crawl_cnmn, 'cron', minute="1-59", hour="*")
# wechat 微信公众号 的爬虫 一分钟
scheduler.add_job(crawl_wechat, 'cron', minute="1", hour="*")
# 上海有色网 的接口 30s
scheduler.add_job(get_smm_interface, 'interval', seconds=30)
# 华尔街见闻 的接口 30s
scheduler.add_job(get_wallstreet_interface, 'interval', seconds=30)
# 财联社直播 每分钟
scheduler.add_job(get_cailian_interface, 'cron', minute="*", hour="*")
# 股吧
scheduler.add_job(crawl_guba, 'cron', minute="5", hour="*")
# 华尔街见闻新闻
scheduler.add_job(crawl_wallstreet, 'cron', minute="4", hour="*")
# 金融界咨询
def __init__(self):
self.scheduler = TornadoScheduler()
job_defaults = {'coalesce': False, 'max_instances': 3}
class MainHandler(tornado.web.RequestHandler):
def get(self):
self.write("Hello World")
def make_app():
return tornado.web.Application([
(r"/", MainHandler),
])
if __name__ == '__main__':
scheduler = TornadoScheduler()
# alarm_time = datetime.now() + timedelta(seconds=10)
scheduler.add_job(tick, 'interval', seconds=10)
# scheduler.add_job(sensorTimerA, 'interval', seconds=19)
print('To clear the alarms, delete the example.sqlite file.')
print('Press Ctrl+{0} to exit'.format('Break' if os.name == 'nt' else 'C'))
scheduler.configure(jobstores=jobstores,
executors=executors,
job_defaults=job_defaults,
timezone=utc)
scheduler.start()
app = make_app()
app.listen(8888)
tornado.ioloop.IOLoop.current().start()
"""
Demonstrates how to use the Tornado compatible scheduler to schedule a job that executes on 3 second intervals.
"""
from datetime import datetime
import os
from tornado.ioloop import IOLoop
from apscheduler.schedulers.tornado import TornadoScheduler
def tick():
print('Tick! The time is: %s' % datetime.now())
if __name__ == '__main__':
scheduler = TornadoScheduler()
scheduler.add_job(tick, 'interval', seconds=3)
scheduler.start()
print('Press Ctrl+{0} to exit'.format('Break' if os.name == 'nt' else 'C'))
# Execution will block here until Ctrl+C (Ctrl+Break on Windows) is pressed.
try:
IOLoop.instance().start()
except (KeyboardInterrupt, SystemExit):
pass
class ScheduleServer():
def __init__(self):
tornado.options.parse_command_line()
application = tornado.web.Application(
[
(r'/', IndexPageHandler),
# --- PROJECT --- #
# List of project summary.
(r'/api/v0/projects', apih.ListProjectSummaryHandler),
# Details of a project. Args: project id
(r'/api/v0/project/([^/]+)/details', apih.ProjectDetailsHandler),
# List of details of resources. Args: project id
(r'/api/v0/project/([^/]+)/resource/details', apih.ProjectResourceDetailsHandler),
# List of language status. Args: project id
(r'/api/v0/project/([^/]+)/translation/status', apih.ProjectTranslationStatusHandler),
# --- JOB --- #
# List of jobs.
(r'/api/v0/jobs', apih.ListJobSummaryHandler),
# Summary of a job. Args: job id
(r'/api/v0/job/([^/]+)', apih.JobSummaryHandler),
# Execute a job. Args: job id
(r'/api/v0/job/([^/]+)/exec', apih.JobExecutionHandler),
# Details of a job. Args: job id
(r'/api/v0/job/([^/]+)/details', apih.JobDetailsHandler),
# List of resource file path and slug. Args: job id
(r'/api/v0/job/([^/]+)/resource/slugs', apih.JobResourceSlugsHandler),
# List of resource name (in translation platform) and slug. Args: job id
(r'/api/v0/job/([^/]+)/translation/slugs', apih.JobTranslationSlugsHandler),
# Sync status (only for ResourceUploaderJob or TranslationUploaderJob). Args: job id
(r'/api/v0/job/([^/]+)/sync/status', apih.JobSyncStatusHandler),
# Job execution status. Args: job id
(r'/api/v0/job/([^/]+)/exec/status', apih.JobExecStatusHandler),
# maybe /job/(^/]+)/log/context/3 (limit = 3) might be useful
# --- CONFIGURATION --- #
# not using now but keep for a while
# (r'/api/v0/config/([^/]+)/([^/]+)', apih.ConfigurationHandler), # job id, 'key' in config file
# Contents of project configuration file. Args: project id
# (r'/api/v0/config/project/([^/]+)', apih.ProjectConfigurationHandler),
# Contents of job configuration file. Args: job id
# (r'/api/v0/config/job/([^/]+)', apih.JobConfigurationHandler),
# Contents of resource configuration file. Args: resource configuration file name
(r'/api/v0/config/resource/([^/]+)', apih.ResourceConfigurationHandler),
# Contents of translation configuration file. Args: translation configuration file name
(r'/api/v0/config/translation/([^/]+)', apih.TranslationConfigurationHandler),
#--- LOG (JOB EXECUTION LOG) --- #
# Log context. Args: log path
(r'/api/v0/log/([^/]+)/context', apih.LogContextHandler),
# --- LOCAL FILES --- #
# List local repositories (git repository).
(r'/api/v0/local/repositories', apih.ListLocalRepositoriesHandler),
# List branches of specified local git repository. Args: repository name
(r'/api/v0/local/repository/([^/]+)/branches', apih.ListLocalRepositoryBranchesHandler),
# List local files under specified directory. Args: repository name, relative path in the repository
(r'/api/v0/local/repository/([^/]+)/files/([^/]+)', apih.ListLocalRepositoryFilesHandler),
# --- RESOURCE REPOSITORY --- #
# List of repositories. Args: platform name
#(r'/api/v0/resource/([^/]+)/repositories', apih.ListResourceRepositoriessHandler),
#--- TRANSLATION REPOSITORY --- #
# List of projects. Args: platform name
(r'/api/v0/translation/([^/]+)/projects', apih.ListTranslationProjectsHandler),
# Project details. Args: platform name, project id
(r'/api/v0/translation/([^/]+)/project/([^/]+)/details', apih.TranslationProjectDetailsHandler),
# Resource details. Args: platform name, project id, resource id
(r'/api/v0/translation/([^/]+)/project/([^/]+)/resource/([^/]+)/details', apih.TranslationResourceDetailsHandler),
# All strings for a language. Args: platform name, project id, resource id, language code
(r'/api/v0/translation/([^/]+)/project/([^/]+)/resource/([^/]+)/translation/([^/]+)/strings', apih.TranslationTranslationStringsHandler),
# Details for a translation string. Args: platform name, project id, resource id, source string id
# (r'/api/v0/translation/([^/]+)/project/([^/]+)/resource/([^/]+)/translation/([^/]+)/details', apih.TranslationTranslationStringDetailsHandler),
# Details for a source string. Args: platform name, project id, resource id, source string id (key)
(r'/api/v0/translation/([^/]+)/project/([^/]+)/resource/([^/]+)/source/([^/]+)/details', apih.TranslationSourceStringDetailsHandler)
],
template_path = os.path.join(os.path.dirname(__file__), 'templates'),
static_path = os.path.join(os.path.dirname(__file__), 'static')
)
self.http_server = tornado.httpserver.HTTPServer(application)
executors = {
'default': {'type': 'threadpool', 'max_workers': 20},
'processpool': ProcessPoolExecutor(max_workers=5)
}
logger.info("Initializing scheduler (pid: {}, port: '{}')...".format(os.getpid(), settings.HTTP_PORT))
self.scheduler = TornadoScheduler()
self.scheduler.configure(executors = executors)
self._restore_jobs()
self.scheduler.start()
logger.info(self.scheduler.print_jobs())
def _restore_jobs(self):
global job
configs = job.get_configuration(status='active')
total = 0
for c in configs:
o = SchedulerJob(c)
self.scheduler.add_job(o.execute, 'cron', month=c.month, day=c.day, day_of_week=c.day_of_week, hour=c.hour, minute=c.minute, name=c.name, id=c.id, misfire_grace_time = 600)
total += 1
logger.info("Restored '{}' jobs.".format(total))
# @classmethod
def start(self):
signal.signal(signal.SIGINT, self._signal_handler)
self.http_server.listen(settings.HTTP_PORT)
tornado.ioloop.IOLoop.current().start()
def _signal_handler(self, signal_type, frame):
if signal_type == signal.SIGINT:
logger.info('SIGINT')
else:
logger.warning('Unknown signal')
self.terminate()
# @classmethod
def terminate(self):
logger.info('Stopping scheduler...')
self.scheduler.shutdown()
tornado.ioloop.IOLoop.current().stop()
sys.exit(0)
class Core(object):
def __init__(self):
self.started = False
# process id
self.PID = os.getpid()
# generate notifiers dict
self.notifiersDict = AttrDict(
libnotify=LibnotifyNotifier(),
kodi_notifier=KODINotifier(),
plex_notifier=PLEXNotifier(),
emby_notifier=EMBYNotifier(),
nmj_notifier=NMJNotifier(),
nmjv2_notifier=NMJv2Notifier(),
synoindex_notifier=synoIndexNotifier(),
synology_notifier=synologyNotifier(),
pytivo_notifier=pyTivoNotifier(),
growl_notifier=GrowlNotifier(),
prowl_notifier=ProwlNotifier(),
libnotify_notifier=LibnotifyNotifier(),
pushover_notifier=PushoverNotifier(),
boxcar_notifier=BoxcarNotifier(),
boxcar2_notifier=Boxcar2Notifier(),
nma_notifier=NMA_Notifier(),
pushalot_notifier=PushalotNotifier(),
pushbullet_notifier=PushbulletNotifier(),
freemobile_notifier=FreeMobileNotifier(),
twitter_notifier=TwitterNotifier(),
trakt_notifier=TraktNotifier(),
email_notifier=EmailNotifier()
)
# generate metadata providers dict
self.metadataProviderDict = get_metadata_generator_dict()
# generate providers dict
self.providersDict = providersDict()
# init notification queue
self.srNotifications = Notifications()
# init logger
self.srLogger = srLogger()
# init config
self.srConfig = srConfig()
# init scheduler service
self.srScheduler = TornadoScheduler()
# init web server
self.srWebServer = srWebServer()
# init web client session
self.srWebSession = srSession()
# google api
self.googleAuth = googleAuth()
# name cache
self.NAMECACHE = srNameCache()
# queues
self.SHOWQUEUE = srShowQueue()
self.SEARCHQUEUE = srSearchQueue()
# updaters
self.VERSIONUPDATER = srVersionUpdater()
self.SHOWUPDATER = srShowUpdater()
# searchers
self.DAILYSEARCHER = srDailySearcher()
self.BACKLOGSEARCHER = srBacklogSearcher()
self.PROPERSEARCHER = srProperSearcher()
self.TRAKTSEARCHER = srTraktSearcher()
self.SUBTITLESEARCHER = srSubtitleSearcher()
# auto postprocessor
self.AUTOPOSTPROCESSOR = srPostProcessor()
# sickrage version
self.NEWEST_VERSION = None
self.NEWEST_VERSION_STRING = None
# anidb connection
self.ADBA_CONNECTION = None
# show list
self.SHOWLIST = []
def start(self):
self.started = True
# thread name
threading.currentThread().setName('CORE')
# Check if we need to perform a restore first
if os.path.exists(os.path.abspath(os.path.join(sickrage.DATA_DIR, 'restore'))):
success = restoreSR(os.path.abspath(os.path.join(sickrage.DATA_DIR, 'restore')), sickrage.DATA_DIR)
print("Restoring SiCKRAGE backup: %s!\n" % ("FAILED", "SUCCESSFUL")[success])
if success:
shutil.rmtree(os.path.abspath(os.path.join(sickrage.DATA_DIR, 'restore')), ignore_errors=True)
# migrate old database file names to new ones
if os.path.isfile(os.path.abspath(os.path.join(sickrage.DATA_DIR, 'sickbeard.db'))):
if os.path.isfile(os.path.join(sickrage.DATA_DIR, 'sickrage.db')):
helpers.moveFile(os.path.join(sickrage.DATA_DIR, 'sickrage.db'),
os.path.join(sickrage.DATA_DIR, '{}.bak-{}'
.format('sickrage.db',
datetime.datetime.now().strftime(
'%Y%m%d_%H%M%S'))))
helpers.moveFile(os.path.abspath(os.path.join(sickrage.DATA_DIR, 'sickbeard.db')), os.path.abspath(os.path.join(sickrage.DATA_DIR, 'sickrage.db')))
# load config
self.srConfig.load()
# set socket timeout
socket.setdefaulttimeout(self.srConfig.SOCKET_TIMEOUT)
# setup logger settings
self.srLogger.logSize = self.srConfig.LOG_SIZE
self.srLogger.logNr = self.srConfig.LOG_NR
self.srLogger.debugLogging = sickrage.DEBUG
self.srLogger.consoleLogging = not sickrage.QUITE
self.srLogger.logFile = self.srConfig.LOG_FILE
# start logger
self.srLogger.start()
# initialize the main SB database
main_db.MainDB().InitialSchema().upgrade()
# initialize the cache database
cache_db.CacheDB().InitialSchema().upgrade()
# initialize the failed downloads database
failed_db.FailedDB().InitialSchema().upgrade()
# fix up any db problems
main_db.MainDB().SanityCheck()
# load data for shows from database
self.load_shows()
if self.srConfig.DEFAULT_PAGE not in ('home', 'schedule', 'history', 'news', 'IRC'):
self.srConfig.DEFAULT_PAGE = 'home'
# cleanup cache folder
for dir in ['mako', 'sessions', 'indexers']:
try:
shutil.rmtree(os.path.join(self.srConfig.CACHE_DIR, dir), ignore_errors=True)
except Exception:
continue
# init anidb connection
if not self.srConfig.USE_ANIDB:
try:
self.ADBA_CONNECTION = adba.Connection(keepAlive=True, log=lambda msg: self.srLogger.debug(
"AniDB: %s " % msg)).auth(self.srConfig.ANIDB_USERNAME, self.srConfig.ANIDB_PASSWORD)
except Exception as e:
self.srLogger.warning("AniDB exception msg: %r " % repr(e))
if self.srConfig.WEB_PORT < 21 or self.srConfig.WEB_PORT > 65535:
self.srConfig.WEB_PORT = 8081
if not self.srConfig.WEB_COOKIE_SECRET:
self.srConfig.WEB_COOKIE_SECRET = generateCookieSecret()
# attempt to help prevent users from breaking links by using a bad url
if not self.srConfig.ANON_REDIRECT.endswith('?'):
self.srConfig.ANON_REDIRECT = ''
if not re.match(r'\d+\|[^|]+(?:\|[^|]+)*', self.srConfig.ROOT_DIRS):
self.srConfig.ROOT_DIRS = ''
self.srConfig.NAMING_FORCE_FOLDERS = check_force_season_folders()
if self.srConfig.NZB_METHOD not in ('blackhole', 'sabnzbd', 'nzbget'):
self.srConfig.NZB_METHOD = 'blackhole'
if self.srConfig.TORRENT_METHOD not in ('blackhole',
'utorrent',
'transmission',
'deluge',
'deluged',
'download_station',
'rtorrent',
'qbittorrent',
'mlnet',
'putio'): self.srConfig.TORRENT_METHOD = 'blackhole'
if self.srConfig.PROPER_SEARCHER_INTERVAL not in ('15m', '45m', '90m', '4h', 'daily'):
self.srConfig.PROPER_SEARCHER_INTERVAL = 'daily'
if self.srConfig.AUTOPOSTPROCESSOR_FREQ < self.srConfig.MIN_AUTOPOSTPROCESSOR_FREQ:
self.srConfig.AUTOPOSTPROCESSOR_FREQ = self.srConfig.MIN_AUTOPOSTPROCESSOR_FREQ
if self.srConfig.NAMECACHE_FREQ < self.srConfig.MIN_NAMECACHE_FREQ:
self.srConfig.NAMECACHE_FREQ = self.srConfig.MIN_NAMECACHE_FREQ
if self.srConfig.DAILY_SEARCHER_FREQ < self.srConfig.MIN_DAILY_SEARCHER_FREQ:
self.srConfig.DAILY_SEARCHER_FREQ = self.srConfig.MIN_DAILY_SEARCHER_FREQ
self.srConfig.MIN_BACKLOG_SEARCHER_FREQ = get_backlog_cycle_time()
if self.srConfig.BACKLOG_SEARCHER_FREQ < self.srConfig.MIN_BACKLOG_SEARCHER_FREQ:
self.srConfig.BACKLOG_SEARCHER_FREQ = self.srConfig.MIN_BACKLOG_SEARCHER_FREQ
if self.srConfig.VERSION_UPDATER_FREQ < self.srConfig.MIN_VERSION_UPDATER_FREQ:
self.srConfig.VERSION_UPDATER_FREQ = self.srConfig.MIN_VERSION_UPDATER_FREQ
if self.srConfig.SHOWUPDATE_HOUR > 23:
self.srConfig.SHOWUPDATE_HOUR = 0
elif self.srConfig.SHOWUPDATE_HOUR < 0:
self.srConfig.SHOWUPDATE_HOUR = 0
if self.srConfig.SUBTITLE_SEARCHER_FREQ < self.srConfig.MIN_SUBTITLE_SEARCHER_FREQ:
self.srConfig.SUBTITLE_SEARCHER_FREQ = self.srConfig.MIN_SUBTITLE_SEARCHER_FREQ
self.srConfig.NEWS_LATEST = self.srConfig.NEWS_LAST_READ
if self.srConfig.SUBTITLES_LANGUAGES[0] == '':
self.srConfig.SUBTITLES_LANGUAGES = []
# initialize metadata_providers
for cur_metadata_tuple in [(self.srConfig.METADATA_KODI, kodi),
(self.srConfig.METADATA_KODI_12PLUS, kodi_12plus),
(self.srConfig.METADATA_MEDIABROWSER, mediabrowser),
(self.srConfig.METADATA_PS3, ps3),
(self.srConfig.METADATA_WDTV, wdtv),
(self.srConfig.METADATA_TIVO, tivo),
(self.srConfig.METADATA_MEDE8ER, mede8er)]:
(cur_metadata_config, cur_metadata_class) = cur_metadata_tuple
tmp_provider = cur_metadata_class.metadata_class()
tmp_provider.set_config(cur_metadata_config)
self.metadataProviderDict[tmp_provider.name] = tmp_provider
# add show queue job
self.srScheduler.add_job(
self.SHOWQUEUE.run,
srIntervalTrigger(**{'seconds': 5}),
name="SHOWQUEUE",
id="SHOWQUEUE"
)
# add search queue job
self.srScheduler.add_job(
self.SEARCHQUEUE.run,
srIntervalTrigger(**{'seconds': 5}),
name="SEARCHQUEUE",
id="SEARCHQUEUE"
)
# add version checker job
self.srScheduler.add_job(
self.VERSIONUPDATER.run,
srIntervalTrigger(
**{'hours': self.srConfig.VERSION_UPDATER_FREQ, 'min': self.srConfig.MIN_VERSION_UPDATER_FREQ}),
name="VERSIONUPDATER",
id="VERSIONUPDATER"
)
# add network timezones updater job
self.srScheduler.add_job(
update_network_dict,
srIntervalTrigger(**{'days': 1}),
name="TZUPDATER",
id="TZUPDATER"
)
# add namecache updater job
self.srScheduler.add_job(
self.NAMECACHE.run,
srIntervalTrigger(
**{'minutes': self.srConfig.NAMECACHE_FREQ, 'min': self.srConfig.MIN_NAMECACHE_FREQ}),
name="NAMECACHE",
id="NAMECACHE"
)
# add show updater job
self.srScheduler.add_job(
self.SHOWUPDATER.run,
srIntervalTrigger(
**{'hours': 1,
'start_date': datetime.datetime.now().replace(hour=self.srConfig.SHOWUPDATE_HOUR)}),
name="SHOWUPDATER",
id="SHOWUPDATER"
)
# add daily search job
self.srScheduler.add_job(
self.DAILYSEARCHER.run,
srIntervalTrigger(
**{'minutes': self.srConfig.DAILY_SEARCHER_FREQ, 'min': self.srConfig.MIN_DAILY_SEARCHER_FREQ}),
name="DAILYSEARCHER",
id="DAILYSEARCHER"
)
# add backlog search job
self.srScheduler.add_job(
self.BACKLOGSEARCHER.run,
srIntervalTrigger(
**{'minutes': self.srConfig.BACKLOG_SEARCHER_FREQ,
'min': self.srConfig.MIN_BACKLOG_SEARCHER_FREQ}),
name="BACKLOG",
id="BACKLOG"
)
# add auto-postprocessing job
self.srScheduler.add_job(
self.AUTOPOSTPROCESSOR.run,
srIntervalTrigger(**{'minutes': self.srConfig.AUTOPOSTPROCESSOR_FREQ,
'min': self.srConfig.MIN_AUTOPOSTPROCESSOR_FREQ}),
name="POSTPROCESSOR",
id="POSTPROCESSOR"
)
# add find proper job
self.srScheduler.add_job(
self.PROPERSEARCHER.run,
srIntervalTrigger(**{
'minutes': {'15m': 15, '45m': 45, '90m': 90, '4h': 4 * 60, 'daily': 24 * 60}[
self.srConfig.PROPER_SEARCHER_INTERVAL]}),
name="PROPERSEARCHER",
id="PROPERSEARCHER"
)
# add trakt.tv checker job
self.srScheduler.add_job(
self.TRAKTSEARCHER.run,
srIntervalTrigger(**{'hours': 1}),
name="TRAKTSEARCHER",
id="TRAKTSEARCHER"
)
# add subtitles finder job
self.srScheduler.add_job(
self.SUBTITLESEARCHER.run,
srIntervalTrigger(**{'hours': self.srConfig.SUBTITLE_SEARCHER_FREQ}),
name="SUBTITLESEARCHER",
id="SUBTITLESEARCHER"
)
# start scheduler service
self.srScheduler.start()
# Pause/Resume PROPERSEARCHER job
(self.srScheduler.get_job('PROPERSEARCHER').pause,
self.srScheduler.get_job('PROPERSEARCHER').resume
)[self.srConfig.DOWNLOAD_PROPERS]()
# Pause/Resume TRAKTSEARCHER job
(self.srScheduler.get_job('TRAKTSEARCHER').pause,
self.srScheduler.get_job('TRAKTSEARCHER').resume
)[self.srConfig.USE_TRAKT]()
# Pause/Resume SUBTITLESEARCHER job
(self.srScheduler.get_job('SUBTITLESEARCHER').pause,
self.srScheduler.get_job('SUBTITLESEARCHER').resume
)[self.srConfig.USE_SUBTITLES]()
# Pause/Resume POSTPROCESS job
(self.srScheduler.get_job('POSTPROCESSOR').pause,
self.srScheduler.get_job('POSTPROCESSOR').resume
)[self.srConfig.PROCESS_AUTOMATICALLY]()
# start webserver
self.srWebServer.start()
# start ioloop event handler
IOLoop.instance().start()
def shutdown(self, status=None, restart=False):
if self.started:
self.started = False
if restart:
self.srLogger.info('SiCKRAGE IS PERFORMING A RESTART!')
else:
self.srLogger.info('SiCKRAGE IS PERFORMING A SHUTDOWN!')
# shutdown/restart webserver
self.srWebServer.shutdown()
# shutdown scheduler
self.srLogger.info("Shutting down scheduler")
self.srScheduler.shutdown()
# shutdown queues
self.srLogger.info("Shutting down queues")
if self.SHOWQUEUE:
self.SHOWQUEUE.shutdown()
if self.SEARCHQUEUE:
self.SEARCHQUEUE.shutdown()
if sickrage.srCore.ADBA_CONNECTION:
self.srLogger.info("Logging out ANIDB connection")
sickrage.srCore.ADBA_CONNECTION.logout()
# save all settings
self.save_all()
if restart:
self.srLogger.info('SiCKRAGE IS RESTARTING!')
else:
self.srLogger.info('SiCKRAGE IS SHUTDOWN!')
# shutdown logging
self.srLogger.shutdown()
# delete pid file
if sickrage.DAEMONIZE:
sickrage.delpid(sickrage.PID_FILE)
# system exit with status
if not restart:
sys.exit(status)
# stop ioloop event handler
IOLoop.current().stop()
def save_all(self):
# write all shows
self.srLogger.info("Saving all shows to the database")
for SHOW in self.SHOWLIST:
try:
SHOW.saveToDB()
except:
continue
# save config
self.srConfig.save()
def load_shows(self):
"""
Populates the showlist with shows from the database
"""
for sqlShow in main_db.MainDB().select("SELECT * FROM tv_shows"):
try:
curshow = TVShow(int(sqlShow["indexer"]), int(sqlShow["indexer_id"]))
self.srLogger.debug("Loading data for show: [{}]".format(curshow.name))
#self.NAMECACHE.buildNameCache(curshow)
curshow.nextEpisode()
self.SHOWLIST += [curshow]
except Exception as e:
self.srLogger.error(
"There was an error creating the show in {}: {}".format(sqlShow["location"], e.message))
self.srLogger.debug(traceback.format_exc())
def __init__(self):
tornado.options.parse_command_line()
application = tornado.web.Application(
[
(r'/', IndexPageHandler),
# --- PROJECT --- #
# List of project summary.
(r'/api/v0/projects', apih.ListProjectSummaryHandler),
# Details of a project. Args: project id
(r'/api/v0/project/([^/]+)/details', apih.ProjectDetailsHandler),
# List of details of resources. Args: project id
(r'/api/v0/project/([^/]+)/resource/details', apih.ProjectResourceDetailsHandler),
# List of language status. Args: project id
(r'/api/v0/project/([^/]+)/translation/status', apih.ProjectTranslationStatusHandler),
# --- JOB --- #
# List of jobs.
(r'/api/v0/jobs', apih.ListJobSummaryHandler),
# Summary of a job. Args: job id
(r'/api/v0/job/([^/]+)', apih.JobSummaryHandler),
# Execute a job. Args: job id
(r'/api/v0/job/([^/]+)/exec', apih.JobExecutionHandler),
# Details of a job. Args: job id
(r'/api/v0/job/([^/]+)/details', apih.JobDetailsHandler),
# List of resource file path and slug. Args: job id
(r'/api/v0/job/([^/]+)/resource/slugs', apih.JobResourceSlugsHandler),
# List of resource name (in translation platform) and slug. Args: job id
(r'/api/v0/job/([^/]+)/translation/slugs', apih.JobTranslationSlugsHandler),
# Sync status (only for ResourceUploaderJob or TranslationUploaderJob). Args: job id
(r'/api/v0/job/([^/]+)/sync/status', apih.JobSyncStatusHandler),
# Job execution status. Args: job id
(r'/api/v0/job/([^/]+)/exec/status', apih.JobExecStatusHandler),
# maybe /job/(^/]+)/log/context/3 (limit = 3) might be useful
# --- CONFIGURATION --- #
# not using now but keep for a while
# (r'/api/v0/config/([^/]+)/([^/]+)', apih.ConfigurationHandler), # job id, 'key' in config file
# Contents of project configuration file. Args: project id
# (r'/api/v0/config/project/([^/]+)', apih.ProjectConfigurationHandler),
# Contents of job configuration file. Args: job id
# (r'/api/v0/config/job/([^/]+)', apih.JobConfigurationHandler),
# Contents of resource configuration file. Args: resource configuration file name
(r'/api/v0/config/resource/([^/]+)', apih.ResourceConfigurationHandler),
# Contents of translation configuration file. Args: translation configuration file name
(r'/api/v0/config/translation/([^/]+)', apih.TranslationConfigurationHandler),
#--- LOG (JOB EXECUTION LOG) --- #
# Log context. Args: log path
(r'/api/v0/log/([^/]+)/context', apih.LogContextHandler),
# --- LOCAL FILES --- #
# List local repositories (git repository).
(r'/api/v0/local/repositories', apih.ListLocalRepositoriesHandler),
# List branches of specified local git repository. Args: repository name
(r'/api/v0/local/repository/([^/]+)/branches', apih.ListLocalRepositoryBranchesHandler),
# List local files under specified directory. Args: repository name, relative path in the repository
(r'/api/v0/local/repository/([^/]+)/files/([^/]+)', apih.ListLocalRepositoryFilesHandler),
# --- RESOURCE REPOSITORY --- #
# List of repositories. Args: platform name
#(r'/api/v0/resource/([^/]+)/repositories', apih.ListResourceRepositoriessHandler),
#--- TRANSLATION REPOSITORY --- #
# List of projects. Args: platform name
(r'/api/v0/translation/([^/]+)/projects', apih.ListTranslationProjectsHandler),
# Project details. Args: platform name, project id
(r'/api/v0/translation/([^/]+)/project/([^/]+)/details', apih.TranslationProjectDetailsHandler),
# Resource details. Args: platform name, project id, resource id
(r'/api/v0/translation/([^/]+)/project/([^/]+)/resource/([^/]+)/details', apih.TranslationResourceDetailsHandler),
# All strings for a language. Args: platform name, project id, resource id, language code
(r'/api/v0/translation/([^/]+)/project/([^/]+)/resource/([^/]+)/translation/([^/]+)/strings', apih.TranslationTranslationStringsHandler),
# Details for a translation string. Args: platform name, project id, resource id, source string id
# (r'/api/v0/translation/([^/]+)/project/([^/]+)/resource/([^/]+)/translation/([^/]+)/details', apih.TranslationTranslationStringDetailsHandler),
# Details for a source string. Args: platform name, project id, resource id, source string id (key)
(r'/api/v0/translation/([^/]+)/project/([^/]+)/resource/([^/]+)/source/([^/]+)/details', apih.TranslationSourceStringDetailsHandler)
],
template_path = os.path.join(os.path.dirname(__file__), 'templates'),
static_path = os.path.join(os.path.dirname(__file__), 'static')
)
self.http_server = tornado.httpserver.HTTPServer(application)
executors = {
'default': {'type': 'threadpool', 'max_workers': 20},
'processpool': ProcessPoolExecutor(max_workers=5)
}
logger.info("Initializing scheduler (pid: {}, port: '{}')...".format(os.getpid(), settings.HTTP_PORT))
self.scheduler = TornadoScheduler()
self.scheduler.configure(executors = executors)
self._restore_jobs()
self.scheduler.start()
logger.info(self.scheduler.print_jobs())
class AuthLogoutHandler(BaseHandler):
def get(self):
self.clear_cookie("live_digg")
self.redirect(self.get_argument("next", "/"))
class EntryModule(tornado.web.UIModule):
def render(self, entry):
return self.render_string("modules/entry.html", entry=entry)
if __name__ == "__main__":
tornado.options.parse_command_line()
http_server = tornado.httpserver.HTTPServer(Application())
http_server.listen(options.port)
scheduler = TornadoScheduler()
scheduler.add_job(pushmsgs.scraper, 'cron', day_of_week='mon-fri',
hour='*', minute='*/15', id="diggScraper")
scheduler.start()
print('Press Ctrl+{0} to exit'.format('Break' if os.name == 'nt' else 'C'))
try:
threading.Thread(target=redis_listener).start()
tornado.ioloop.IOLoop.current().start()
except (KeyboardInterrupt, SystemExit):
pass
def start(self):
self.started = True
self.io_loop = IOLoop.current()
# thread name
threading.currentThread().setName('CORE')
# init core classes
self.api = API()
self.main_db = MainDB(self.db_type, self.db_prefix, self.db_host,
self.db_port, self.db_username, self.db_password)
self.cache_db = CacheDB(self.db_type, self.db_prefix, self.db_host,
self.db_port, self.db_username,
self.db_password)
self.notifier_providers = NotifierProviders()
self.metadata_providers = MetadataProviders()
self.search_providers = SearchProviders()
self.log = Logger()
self.config = Config()
self.alerts = Notifications()
self.scheduler = TornadoScheduler({'apscheduler.timezone': 'UTC'})
self.wserver = WebServer()
self.show_queue = ShowQueue()
self.search_queue = SearchQueue()
self.postprocessor_queue = PostProcessorQueue()
self.version_updater = VersionUpdater()
self.show_updater = ShowUpdater()
self.tz_updater = TimeZoneUpdater()
self.rsscache_updater = RSSCacheUpdater()
self.daily_searcher = DailySearcher()
self.failed_snatch_searcher = FailedSnatchSearcher()
self.backlog_searcher = BacklogSearcher()
self.proper_searcher = ProperSearcher()
self.trakt_searcher = TraktSearcher()
self.subtitle_searcher = SubtitleSearcher()
self.auto_postprocessor = AutoPostProcessor()
self.upnp_client = UPNPClient()
self.quicksearch_cache = QuicksearchCache()
self.announcements = Announcements()
# setup oidc client
realm = KeycloakRealm(server_url='https://auth.sickrage.ca',
realm_name='sickrage')
self.oidc_client = realm.open_id_connect(
client_id=self.oidc_client_id,
client_secret=self.oidc_client_secret)
# check available space
try:
self.log.info("Performing disk space checks")
total_space, available_space = get_free_space(self.data_dir)
if available_space < 100:
self.log.warning(
'Shutting down as SiCKRAGE needs some space to work. You\'ll get corrupted data otherwise. Only %sMB left',
available_space)
return
except Exception:
self.log.error('Failed getting disk space: %s',
traceback.format_exc())
# check if we need to perform a restore first
if os.path.exists(
os.path.abspath(os.path.join(self.data_dir, 'restore'))):
success = restore_app_data(
os.path.abspath(os.path.join(self.data_dir, 'restore')),
self.data_dir)
self.log.info("Restoring SiCKRAGE backup: %s!" %
("FAILED", "SUCCESSFUL")[success])
if success:
self.main_db = MainDB(self.db_type, self.db_prefix,
self.db_host, self.db_port,
self.db_username, self.db_password)
self.cache_db = CacheDB(self.db_type, self.db_prefix,
self.db_host, self.db_port,
self.db_username, self.db_password)
shutil.rmtree(os.path.abspath(
os.path.join(self.data_dir, 'restore')),
ignore_errors=True)
# migrate old database file names to new ones
if os.path.isfile(
os.path.abspath(os.path.join(self.data_dir, 'sickbeard.db'))):
if os.path.isfile(os.path.join(self.data_dir, 'sickrage.db')):
helpers.move_file(
os.path.join(self.data_dir, 'sickrage.db'),
os.path.join(
self.data_dir, '{}.bak-{}'.format(
'sickrage.db',
datetime.datetime.now().strftime(
'%Y%m%d_%H%M%S'))))
helpers.move_file(
os.path.abspath(os.path.join(self.data_dir, 'sickbeard.db')),
os.path.abspath(os.path.join(self.data_dir, 'sickrage.db')))
# init encryption public and private keys
encryption.initialize()
# load config
self.config.load()
# set language
self.config.change_gui_lang(self.config.gui_lang)
# set socket timeout
socket.setdefaulttimeout(self.config.socket_timeout)
# setup logger settings
self.log.logSize = self.config.log_size
self.log.logNr = self.config.log_nr
self.log.logFile = os.path.join(self.data_dir, 'logs', 'sickrage.log')
self.log.debugLogging = self.config.debug
self.log.consoleLogging = not self.quiet
# start logger
self.log.start()
# perform database startup actions
for db in [self.main_db, self.cache_db]:
# perform integrity check
self.log.info("Performing integrity check on {} database".format(
db.name))
db.integrity_check()
# migrate database
self.log.info("Performing migrations on {} database".format(
db.name))
db.migrate()
# sync database repo
self.log.info("Performing sync on {} database".format(db.name))
db.sync_db_repo()
# cleanup
self.log.info("Performing cleanup on {} database".format(db.name))
db.cleanup()
# user agent
if self.config.random_user_agent:
self.user_agent = UserAgent().random
uses_netloc.append('scgi')
FancyURLopener.version = self.user_agent
# set torrent client web url
torrent_webui_url(True)
# load quicksearch cache
self.quicksearch_cache.load()
if self.config.default_page not in ('schedule', 'history', 'IRC'):
self.config.default_page = 'home'
# cleanup cache folder
for folder in ['mako', 'sessions', 'indexers']:
try:
shutil.rmtree(os.path.join(self.cache_dir, folder),
ignore_errors=True)
except Exception:
continue
if self.config.web_port < 21 or self.config.web_port > 65535:
self.config.web_port = 8081
if not self.config.web_cookie_secret:
self.config.web_cookie_secret = generate_secret()
# attempt to help prevent users from breaking links by using a bad url
if not self.config.anon_redirect.endswith('?'):
self.config.anon_redirect = ''
if not re.match(r'\d+\|[^|]+(?:\|[^|]+)*', self.config.root_dirs):
self.config.root_dirs = ''
self.config.naming_force_folders = check_force_season_folders()
if self.config.nzb_method not in ('blackhole', 'sabnzbd', 'nzbget'):
self.config.nzb_method = 'blackhole'
if self.config.torrent_method not in ('blackhole', 'utorrent',
'transmission', 'deluge',
'deluged', 'download_station',
'rtorrent', 'qbittorrent',
'mlnet', 'putio'):
self.config.torrent_method = 'blackhole'
if self.config.autopostprocessor_freq < self.config.min_autopostprocessor_freq:
self.config.autopostprocessor_freq = self.config.min_autopostprocessor_freq
if self.config.daily_searcher_freq < self.config.min_daily_searcher_freq:
self.config.daily_searcher_freq = self.config.min_daily_searcher_freq
if self.config.backlog_searcher_freq < self.config.min_backlog_searcher_freq:
self.config.backlog_searcher_freq = self.config.min_backlog_searcher_freq
if self.config.version_updater_freq < self.config.min_version_updater_freq:
self.config.version_updater_freq = self.config.min_version_updater_freq
if self.config.subtitle_searcher_freq < self.config.min_subtitle_searcher_freq:
self.config.subtitle_searcher_freq = self.config.min_subtitle_searcher_freq
if self.config.failed_snatch_age < self.config.min_failed_snatch_age:
self.config.failed_snatch_age = self.config.min_failed_snatch_age
if self.config.proper_searcher_interval not in ('15m', '45m', '90m',
'4h', 'daily'):
self.config.proper_searcher_interval = 'daily'
if self.config.showupdate_hour < 0 or self.config.showupdate_hour > 23:
self.config.showupdate_hour = 0
# add version checker job
self.scheduler.add_job(self.version_updater.run,
IntervalTrigger(
hours=self.config.version_updater_freq,
timezone='utc'),
name=self.version_updater.name,
id=self.version_updater.name)
# add network timezones updater job
self.scheduler.add_job(self.tz_updater.run,
IntervalTrigger(days=1, timezone='utc'),
name=self.tz_updater.name,
id=self.tz_updater.name)
# add show updater job
self.scheduler.add_job(self.show_updater.run,
IntervalTrigger(
days=1,
start_date=datetime.datetime.now().replace(
hour=self.config.showupdate_hour),
timezone='utc'),
name=self.show_updater.name,
id=self.show_updater.name)
# add rss cache updater job
self.scheduler.add_job(self.rsscache_updater.run,
IntervalTrigger(minutes=15, timezone='utc'),
name=self.rsscache_updater.name,
id=self.rsscache_updater.name)
# add daily search job
self.scheduler.add_job(
self.daily_searcher.run,
IntervalTrigger(minutes=self.config.daily_searcher_freq,
start_date=datetime.datetime.now() +
datetime.timedelta(minutes=4),
timezone='utc'),
name=self.daily_searcher.name,
id=self.daily_searcher.name)
# add failed snatch search job
self.scheduler.add_job(
self.failed_snatch_searcher.run,
IntervalTrigger(hours=1,
start_date=datetime.datetime.now() +
datetime.timedelta(minutes=4),
timezone='utc'),
name=self.failed_snatch_searcher.name,
id=self.failed_snatch_searcher.name)
# add backlog search job
self.scheduler.add_job(
self.backlog_searcher.run,
IntervalTrigger(minutes=self.config.backlog_searcher_freq,
start_date=datetime.datetime.now() +
datetime.timedelta(minutes=30),
timezone='utc'),
name=self.backlog_searcher.name,
id=self.backlog_searcher.name)
# add auto-postprocessing job
self.scheduler.add_job(self.auto_postprocessor.run,
IntervalTrigger(
minutes=self.config.autopostprocessor_freq,
timezone='utc'),
name=self.auto_postprocessor.name,
id=self.auto_postprocessor.name)
# add find proper job
self.scheduler.add_job(self.proper_searcher.run,
IntervalTrigger(minutes={
'15m': 15,
'45m': 45,
'90m': 90,
'4h': 4 * 60,
'daily': 24 * 60
}[self.config.proper_searcher_interval],
timezone='utc'),
name=self.proper_searcher.name,
id=self.proper_searcher.name)
# add trakt.tv checker job
self.scheduler.add_job(self.trakt_searcher.run,
IntervalTrigger(hours=1, timezone='utc'),
name=self.trakt_searcher.name,
id=self.trakt_searcher.name)
# add subtitles finder job
self.scheduler.add_job(self.subtitle_searcher.run,
IntervalTrigger(
hours=self.config.subtitle_searcher_freq,
timezone='utc'),
name=self.subtitle_searcher.name,
id=self.subtitle_searcher.name)
# add upnp client job
self.scheduler.add_job(
self.upnp_client.run,
IntervalTrigger(seconds=self.upnp_client._nat_portmap_lifetime,
timezone='utc'),
name=self.upnp_client.name,
id=self.upnp_client.name)
# add announcements job
self.scheduler.add_job(self.announcements.run,
IntervalTrigger(minutes=15, timezone='utc'),
name=self.announcements.name,
id=self.announcements.name)
# start queues
self.search_queue.start()
self.show_queue.start()
self.postprocessor_queue.start()
# fire off startup events
self.io_loop.add_callback(self.load_shows)
self.io_loop.add_callback(self.version_updater.run)
self.io_loop.add_callback(self.tz_updater.run)
self.io_loop.add_callback(self.announcements.run)
# start scheduler service
self.scheduler.start()
# start web server
self.wserver.start()
# launch browser window
if all(
[not sickrage.app.no_launch, sickrage.app.config.launch_browser]):
self.io_loop.add_callback(
functools.partial(launch_browser,
('http',
'https')[sickrage.app.config.enable_https],
sickrage.app.config.web_host,
sickrage.app.config.web_port))
def started():
threading.currentThread().setName('CORE')
self.log.info("SiCKRAGE :: STARTED")
self.log.info("SiCKRAGE :: APP VERSION:[{}]".format(
sickrage.version()))
self.log.info("SiCKRAGE :: CONFIG VERSION:[v{}]".format(
self.config.config_version))
self.log.info("SiCKRAGE :: DATABASE VERSION:[v{}]".format(
self.main_db.version))
self.log.info("SiCKRAGE :: DATABASE TYPE:[{}]".format(
self.db_type))
self.log.info("SiCKRAGE :: URL:[{}://{}:{}{}]".format(
('http', 'https')[self.config.enable_https],
self.config.web_host, self.config.web_port,
self.config.web_root))
# start io_loop
self.io_loop.add_callback(started)
self.io_loop.start()
