def __init__(self, options): """ :param options: argparse parsed options object """ global manager assert not manager, 'Only one instance of Manager should be created at a time!' self.options = options self.config_base = None self.config_name = None self.config_path = None self.db_filename = None self.engine = None self.lockfile = None self.database_uri = None self.db_upgraded = False self._has_lock = False self.config = {} self.scheduler = Scheduler(self) self.ipc_server = IPCServer(self, options.ipc_port) manager = self self.initialize() # cannot be imported at module level because of circular references from flexget.utils.simple_persistence import SimplePersistence self.persist = SimplePersistence('manager') log.debug('sys.defaultencoding: %s' % sys.getdefaultencoding()) log.debug('sys.getfilesystemencoding: %s' % sys.getfilesystemencoding()) log.debug('os.path.supports_unicode_filenames: %s' % os.path.supports_unicode_filenames) if db_schema.upgrade_required(): log.info('Database upgrade is required. Attempting now.') # Make sure not to fire the lock-acquired event yet # TODO: Detect if any database upgrading is needed and acquire the lock only in one place with self.acquire_lock(event=False): fire_event('manager.upgrade', self) if manager.db_upgraded: fire_event('manager.db_upgraded', self) fire_event('manager.startup', self)
class Manager(object): """Manager class for FlexGet Fires events: * manager.startup After manager has been initialized. This is when application becomes ready to use, however no database lock is present, so the database must not be modified on this event. * manager.lock-acquired The manager does not always require a lock on startup, if one is requested, this event will run when it has been acquired successfully * manager.config-loaded When a valid configuration has been (re)loaded, this event is fired * manager.upgrade If any plugins have declared a newer schema version than exists in the database, this event will be fired to allow plugins to upgrade their tables * manager.shutdown When the manager is exiting * manager.execute.completed If execution in current process was completed * manager.daemon.started * manager.daemon.completed * manager.db_cleanup """ unit_test = False options = None def __init__(self, options): """ :param options: argparse parsed options object """ global manager assert not manager, 'Only one instance of Manager should be created at a time!' self.options = options self.config_base = None self.config_name = None self.config_path = None self.db_filename = None self.engine = None self.lockfile = None self.database_uri = None self.db_upgraded = False self._has_lock = False self.config = {} self.scheduler = Scheduler(self) self.ipc_server = IPCServer(self, options.ipc_port) manager = self self.initialize() # cannot be imported at module level because of circular references from flexget.utils.simple_persistence import SimplePersistence self.persist = SimplePersistence('manager') log.debug('sys.defaultencoding: %s' % sys.getdefaultencoding()) log.debug('sys.getfilesystemencoding: %s' % sys.getfilesystemencoding()) log.debug('os.path.supports_unicode_filenames: %s' % os.path.supports_unicode_filenames) if db_schema.upgrade_required(): log.info('Database upgrade is required. Attempting now.') # Make sure not to fire the lock-acquired event yet # TODO: Detect if any database upgrading is needed and acquire the lock only in one place with self.acquire_lock(event=False): fire_event('manager.upgrade', self) if manager.db_upgraded: fire_event('manager.db_upgraded', self) fire_event('manager.startup', self) def __del__(self): global manager manager = None def initialize(self): """Separated from __init__ so that unit tests can modify options before loading config.""" self.setup_yaml() self.find_config(create=(self.options.cli_command == 'webui')) self.init_sqlalchemy() fire_event('manager.before_config_load', self) self.load_config() fire_event('manager.before_config_validate', self) errors = self.validate_config() if errors: for error in errors: log.critical("[%s] %s", error.json_pointer, error.message) self.shutdown(finish_queue=False) sys.exit(1) fire_event('manager.config-loaded', self) @property def tasks(self): """A list of tasks in the config""" if not self.config: return [] return self.config.get('tasks', {}).keys() @property def has_lock(self): return self._has_lock def run_cli_command(self): """ Starting point when executing from commandline, dispatch execution to correct destination. * :meth:`.execute_command` * :meth:`.daemon_command` * :meth:`.webui_command` * CLI plugin callback function """ command = self.options.cli_command options = getattr(self.options, command) # First check for built-in commands if command == 'execute': self.execute_command(options) elif command == 'daemon': self.daemon_command(options) elif command == 'webui': self.webui_command(options) else: # Otherwise dispatch the command to the callback function options.cli_command_callback(self, options) def execute_command(self, options): """ Send execute command to daemon through IPC or perform execution on current process. Fires events: * manager.execute.completed :param options: argparse options """ # If a daemon is started, send the execution to the daemon ipc_info = self.check_ipc_info() if ipc_info: client = IPCClient(ipc_info['port'], ipc_info['password']) client.execute(dict(options)) self.shutdown() return # Otherwise we run the execution ourselves with self.acquire_lock(): fire_event('manager.execute.started', self) self.scheduler.start(run_schedules=False) self.scheduler.execute(options) self.scheduler.shutdown(finish_queue=True) try: self.scheduler.wait() except KeyboardInterrupt: log.error('Got ctrl-c exiting after this task completes. Press ctrl-c again to abort this task.') else: fire_event('manager.execute.completed', self) self.shutdown(finish_queue=False) def daemon_command(self, options): """ Fires events: * manager.daemon.started * manager.daemon.completed :param options: argparse options """ if options.action == 'start': if options.daemonize: self.daemonize() with self.acquire_lock(): signal.signal(signal.SIGTERM, self._handle_sigterm) self.ipc_server.start() fire_event('manager.daemon.started', self) self.scheduler.start() try: self.scheduler.wait() except KeyboardInterrupt: log.info('Got ctrl-c, shutting down.') fire_event('manager.daemon.completed', self) self.shutdown(finish_queue=False) elif options.action == 'stop': ipc_info = self.check_ipc_info() if ipc_info: client = IPCClient(ipc_info['port'], ipc_info['password']) client.shutdown() self.shutdown() else: log.error('There does not appear to be a daemon running.') elif options.action == 'status': ipc_info = self.check_ipc_info() if ipc_info: log.info('Daemon running. (PID: %s)' % ipc_info['pid']) else: log.info('No daemon appears to be running for this config.') def webui_command(self, options): """ :param options: argparse options """ try: pkg_resources.require('flexget[webui]') except pkg_resources.DistributionNotFound as e: log.error('Dependency not met. %s' % e) log.error('Webui dependencies not installed. You can use `pip install flexget[webui]` to install them.') self.shutdown() return if options.daemonize: self.daemonize() from flexget.ui import webui with self.acquire_lock(): webui.start(self) def _handle_sigterm(self, signum, frame): log.info('Got SIGTERM. Shutting down.') self.shutdown(finish_queue=False) def setup_yaml(self): """Sets up the yaml loader to return unicode objects for strings by default""" def construct_yaml_str(self, node): # Override the default string handling function # to always return unicode objects return self.construct_scalar(node) yaml.Loader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str) yaml.SafeLoader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str) # Set up the dumper to not tag every string with !!python/unicode def unicode_representer(dumper, uni): node = yaml.ScalarNode(tag=u'tag:yaml.org,2002:str', value=uni) return node yaml.add_representer(unicode, unicode_representer) # Set up the dumper to increase the indent for lists def increase_indent_wrapper(func): def increase_indent(self, flow=False, indentless=False): func(self, flow, False) return increase_indent yaml.Dumper.increase_indent = increase_indent_wrapper(yaml.Dumper.increase_indent) yaml.SafeDumper.increase_indent = increase_indent_wrapper(yaml.SafeDumper.increase_indent) def find_config(self, create=False): """ Find the configuration file. :param bool create: If a config file is not found, and create is True, one will be created in the home folder """ config = None home_path = os.path.join(os.path.expanduser('~'), '.flexget') options_config = os.path.expanduser(self.options.config) possible = [] if os.path.isabs(options_config): # explicit path given, don't try anything config = options_config possible = [config] else: log.debug('Figuring out config load paths') possible.append(os.getcwd()) # for virtualenv / dev sandbox if hasattr(sys, 'real_prefix'): log.debug('Adding virtualenv path') possible.append(sys.prefix) # normal lookup locations possible.append(home_path) if sys.platform.startswith('win'): # On windows look in ~/flexget as well, as explorer does not let you create a folder starting with a dot home_path = os.path.join(os.path.expanduser('~'), 'flexget') possible.append(home_path) else: # The freedesktop.org standard config location xdg_config = os.environ.get('XDG_CONFIG_HOME', os.path.join(os.path.expanduser('~'), '.config')) possible.append(os.path.join(xdg_config, 'flexget')) for path in possible: config = os.path.join(path, options_config) if os.path.exists(config): log.debug('Found config: %s' % config) break else: config = None if not (config and os.path.exists(config)): if not create: log.info('Tried to read from: %s' % ', '.join(possible)) log.critical('Failed to find configuration file %s' % options_config) sys.exit(1) config = os.path.join(home_path, options_config) log.info('Config file %s not found. Creating new config %s' % (options_config, config)) with open(config, 'w') as newconfig: # Write empty tasks to the config newconfig.write(yaml.dump({'tasks': {}})) log.debug('Config file %s selected' % config) self.config_path = config self.config_name = os.path.splitext(os.path.basename(config))[0] self.config_base = os.path.normpath(os.path.dirname(config)) self.lockfile = os.path.join(self.config_base, '.%s-lock' % self.config_name) def load_config(self): """ .. warning:: Calls sys.exit(1) if configuration file could not be loaded. This is something we probably want to change. """ with open(self.config_path, 'rb') as f: config = f.read() try: config = config.decode('utf-8') except UnicodeDecodeError: log.critical('Config file must be UTF-8 encoded.') sys.exit(1) try: self.config = yaml.safe_load(config) or {} except Exception as e: msg = str(e).replace('\n', ' ') msg = ' '.join(msg.split()) log.critical(msg) print('') print('-' * 79) print(' Malformed configuration file (check messages above). Common reasons:') print('-' * 79) print('') print(' o Indentation error') print(' o Missing : from end of the line') print(' o Non ASCII characters (use UTF8)') print(' o If text contains any of :[]{}% characters it must be single-quoted ' \ '(eg. value{1} should be \'value{1}\')\n') # Not very good practice but we get several kind of exceptions here, I'm not even sure all of them # At least: ReaderError, YmlScannerError (or something like that) if hasattr(e, 'problem') and hasattr(e, 'context_mark') and hasattr(e, 'problem_mark'): lines = 0 if e.problem is not None: print(' Reason: %s\n' % e.problem) if e.problem == 'mapping values are not allowed here': print(' ----> MOST LIKELY REASON: Missing : from end of the line!') print('') if e.context_mark is not None: print(' Check configuration near line %s, column %s' % (e.context_mark.line, e.context_mark.column)) lines += 1 if e.problem_mark is not None: print(' Check configuration near line %s, column %s' % (e.problem_mark.line, e.problem_mark.column)) lines += 1 if lines: print('') if lines == 1: print(' Fault is almost always in this or previous line\n') if lines == 2: print(' Fault is almost always in one of these lines or previous ones\n') # When --debug escalate to full stacktrace if self.options.debug: raise sys.exit(1) # config loaded successfully log.debug('config_name: %s' % self.config_name) log.debug('config_base: %s' % self.config_base) def save_config(self): """Dumps current config to yaml config file""" config_file = file(os.path.join(self.config_base, self.config_name) + '.yml', 'w') try: config_file.write(yaml.dump(self.config, default_flow_style=False)) finally: config_file.close() def config_changed(self): """Makes sure that all tasks will have the config_modified flag come out true on the next run. Useful when changing the db and all tasks need to be completely reprocessed.""" from flexget.task import config_changed for task in self.tasks: config_changed(task) def pre_check_config(self, config): """Checks configuration file for common mistakes that are easily detectable""" def get_indentation(line): i, n = 0, len(line) while i < n and line[i] == ' ': i += 1 return i def isodd(n): return bool(n % 2) line_num = 0 duplicates = {} # flags prev_indentation = 0 prev_mapping = False prev_list = True prev_scalar = True list_open = False # multiline list with [ for line in config.splitlines(): if '# warnings off' in line.strip().lower(): log.debug('config pre-check warnings off') break line_num += 1 # remove linefeed line = line.rstrip() # empty line if line.strip() == '': continue # comment line if line.strip().startswith('#'): continue indentation = get_indentation(line) if prev_scalar: if indentation <= prev_indentation: prev_scalar = False else: continue cur_list = line.strip().startswith('-') # skipping lines as long as multiline compact list is open if list_open: if line.strip().endswith(']'): list_open = False # print 'closed list at line %s' % line continue else: list_open = line.strip().endswith(': [') or line.strip().endswith(':[') if list_open: # print 'list open at line %s' % line continue # print '#%i: %s' % (line_num, line) # print 'indentation: %s, prev_ind: %s, prev_mapping: %s, prev_list: %s, cur_list: %s' % \ # (indentation, prev_indentation, prev_mapping, prev_list, cur_list) if ':\t' in line: log.critical('Line %s has TAB character after : character. ' 'DO NOT use tab key when editing config!' % line_num) elif '\t' in line: log.warning('Line %s has tabs, use only spaces!' % line_num) if isodd(indentation): log.warning('Config line %s has odd (uneven) indentation' % line_num) if indentation > prev_indentation and not prev_mapping: # line increases indentation, but previous didn't start mapping log.warning('Config line %s is likely missing \':\' at the end' % (line_num - 1)) if indentation > prev_indentation + 2 and prev_mapping and not prev_list: # mapping value after non list indented more than 2 log.warning('Config line %s is indented too much' % line_num) if indentation <= prev_indentation + (2 * (not cur_list)) and prev_mapping and prev_list: log.warning('Config line %s is not indented enough' % line_num) if prev_mapping and cur_list: # list after opening mapping if indentation < prev_indentation or indentation > prev_indentation + 2 + (2 * prev_list): log.warning('Config line %s containing list element is indented incorrectly' % line_num) elif prev_mapping and indentation <= prev_indentation: # after opening a map, indentation doesn't increase log.warning('Config line %s is indented incorrectly (previous line ends with \':\')' % line_num) # notify if user is trying to set same key multiple times in a task (a common mistake) for level in duplicates.iterkeys(): # when indentation goes down, delete everything indented more than that if indentation < level: duplicates[level] = {} if ':' in line: name = line.split(':', 1)[0].strip() ns = duplicates.setdefault(indentation, {}) if name in ns: log.warning('Trying to set value for `%s` in line %s, but it is already defined in line %s!' % (name, line_num, ns[name])) ns[name] = line_num prev_indentation = indentation # this line is a mapping (ends with :) prev_mapping = line[-1] == ':' prev_scalar = line[-1] in '|>' # this line is a list prev_list = line.strip()[0] == '-' if prev_list: # This line is in a list, so clear the duplicates, # as duplicates are not always wrong in a list. see #697 duplicates[indentation] = {} log.debug('Pre-checked %s configuration lines' % line_num) def validate_config(self): """ Check all root level keywords are valid. :returns: A list of `ValidationError`s """ return config_schema.process_config(self.config) def init_sqlalchemy(self): """Initialize SQLAlchemy""" try: if [int(part) for part in sqlalchemy.__version__.split('.')] < [0, 7, 0]: print('FATAL: SQLAlchemy 0.7.0 or newer required. Please upgrade your SQLAlchemy.', file=sys.stderr) sys.exit(1) except ValueError as e: log.critical('Failed to check SQLAlchemy version, you may need to upgrade it') # SQLAlchemy if self.database_uri is None: self.db_filename = os.path.join(self.config_base, 'db-%s.sqlite' % self.config_name) if self.options.test: db_test_filename = os.path.join(self.config_base, 'test-%s.sqlite' % self.config_name) log.info('Test mode, creating a copy from database ...') if os.path.exists(self.db_filename): shutil.copy(self.db_filename, db_test_filename) self.db_filename = db_test_filename log.info('Test database created') # in case running on windows, needs double \\ filename = self.db_filename.replace('\\', '\\\\') self.database_uri = 'sqlite:///%s' % filename if self.db_filename and not os.path.exists(self.db_filename): log.verbose('Creating new database %s ...' % self.db_filename) # fire up the engine log.debug('Connecting to: %s' % self.database_uri) try: self.engine = sqlalchemy.create_engine(self.database_uri, echo=self.options.debug_sql, poolclass=SingletonThreadPool, connect_args={'check_same_thread': False}) # assert_unicode=True except ImportError: print('FATAL: Unable to use SQLite. Are you running Python 2.5 - 2.7 ?\n' 'Python should normally have SQLite support built in.\n' 'If you\'re running correct version of Python then it is not equipped with SQLite.\n' 'You can try installing `pysqlite`. If you have compiled python yourself, ' 'recompile it with SQLite support.', file=sys.stderr) sys.exit(1) Session.configure(bind=self.engine) # create all tables, doesn't do anything to existing tables try: def before_table_create(event, target, bind, tables=None, **kw): if tables: # We need to acquire a lock if we are creating new tables # TODO: Detect if any database upgrading is needed and acquire the lock only in one place self.acquire_lock(event=False).__enter__() Base.metadata.append_ddl_listener('before-create', before_table_create) Base.metadata.create_all(bind=self.engine) except OperationalError as e: if os.path.exists(self.db_filename): print('%s - make sure you have write permissions to file %s' % (e.message, self.db_filename), file=sys.stderr) else: print('%s - make sure you have write permissions to directory %s' % (e.message, self.config_base), file=sys.stderr) raise Exception(e.message) def _read_lock(self): """ Read the values from the lock file. Returns None if there is no current lock file. """ if self.lockfile and os.path.exists(self.lockfile): result = {} with open(self.lockfile) as f: lines = [l for l in f.readlines() if l] for line in lines: key, value = line.split(b':', 1) result[key.strip().lower()] = value.strip() for key in result: if result[key].isdigit(): result[key] = int(result[key]) if not pid_exists(result['pid']): return None return result return None def check_lock(self): """Returns True if there is a lock on the database.""" lock_info = self._read_lock() if not lock_info: return False # Don't count it if we hold the lock if os.getpid() == lock_info['pid']: return False return True def check_ipc_info(self): """If a daemon has a lock on the database, return info to connect to IPC.""" lock_info = self._read_lock() if lock_info and 'port' in lock_info: return lock_info return None @contextmanager def acquire_lock(self, event=True): """ :param bool event: If True, the 'manager.lock-acquired' event will be fired after a lock is obtained """ acquired = False try: # Don't do anything if we already have a lock. This means only the outermost call will release the lock file if not self._has_lock: # Exit if there is an existing lock. if self.check_lock(): with open(self.lockfile) as f: pid = f.read() print('Another process (%s) is running, will exit.' % pid.split('\n')[0], file=sys.stderr) print('If you\'re sure there is no other instance running, delete %s' % self.lockfile, file=sys.stderr) sys.exit(1) self._has_lock = True self.write_lock() acquired = True if event: fire_event('manager.lock-acquired', self) yield finally: if acquired: self.release_lock() self._has_lock = False def write_lock(self, ipc_info=None): assert self._has_lock with open(self.lockfile, 'w') as f: f.write(b'PID: %s\n' % os.getpid()) if ipc_info: for key in sorted(ipc_info): f.write(b'%s: %s\n' % (key, ipc_info[key])) def release_lock(self): if os.path.exists(self.lockfile): os.remove(self.lockfile) log.debug('Removed %s' % self.lockfile) else: log.debug('Lockfile %s not found' % self.lockfile) def daemonize(self): """Daemonizes the current process. Returns the new pid""" if sys.platform.startswith('win'): log.error('Cannot daemonize on windows') return if threading.activeCount() != 1: log.critical('There are %r active threads. ' 'Daemonizing now may cause strange failures.' % threading.enumerate()) log.info('Daemonizing...') try: pid = os.fork() if pid > 0: # Don't run the exit handlers on the parent atexit._exithandlers = [] # exit first parent sys.exit(0) except OSError as e: sys.stderr.write('fork #1 failed: %d (%s)\n' % (e.errno, e.strerror)) sys.exit(1) # decouple from parent environment os.chdir('/') os.setsid() os.umask(0) # do second fork try: pid = os.fork() if pid > 0: # Don't run the exit handlers on the parent atexit._exithandlers = [] # exit from second parent sys.exit(0) except OSError as e: sys.stderr.write('fork #2 failed: %d (%s)\n' % (e.errno, e.strerror)) sys.exit(1) log.info('Daemonize complete. New PID: %s' % os.getpid()) # redirect standard file descriptors sys.stdout.flush() sys.stderr.flush() si = file('/dev/null', 'r') so = file('/dev/null', 'a+') se = file('/dev/null', 'a+', 0) os.dup2(si.fileno(), sys.stdin.fileno()) os.dup2(so.fileno(), sys.stdout.fileno()) os.dup2(se.fileno(), sys.stderr.fileno()) def db_cleanup(self, force=False): """ Perform database cleanup if cleanup interval has been met. Fires events: * manager.db_cleanup If interval was met. Gives session to do the cleanup as a parameter. :param bool force: Run the cleanup no matter whether the interval has been met. """ expired = self.persist.get('last_cleanup', datetime(1900, 1, 1)) < datetime.now() - DB_CLEANUP_INTERVAL if force or expired: log.info('Running database cleanup.') session = Session() try: fire_event('manager.db_cleanup', session) session.commit() finally: session.close() # Just in case some plugin was overzealous in its cleaning, mark the config changed self.config_changed() self.persist['last_cleanup'] = datetime.now() else: log.debug('Not running db cleanup, last run %s' % self.persist.get('last_cleanup')) def shutdown(self, finish_queue=True): """ Application is being exited :param bool finish_queue: Should scheduler finish the task queue """ # Wait for scheduler to finish self.scheduler.shutdown(finish_queue=finish_queue) try: self.scheduler.wait() except KeyboardInterrupt: log.debug('Not waiting for scheduler shutdown due to ctrl-c') # show real stack trace in debug mode if manager.options.debug: raise print('**** Keyboard Interrupt ****') fire_event('manager.shutdown', self) if not self.unit_test: # don't scroll "nosetests" summary results when logging is enabled log.debug('Shutting down') self.engine.dispose() # remove temporary database used in test mode if self.options.test: if not 'test' in self.db_filename: raise Exception('trying to delete non test database?') if self._has_lock: os.remove(self.db_filename) log.info('Removed test database') if not self.unit_test: # don't scroll "nosetests" summary results when logging is enabled log.debug('Shutdown completed')
class Manager(object): """Manager class for FlexGet Fires events: * manager.startup After manager has been initialized. This is when application becomes ready to use, however no database lock is present, so the database must not be modified on this event. * manager.lock-acquired The manager does not always require a lock on startup, if one is requested, this event will run when it has been acquired successfully * manager.config-loaded When a valid configuration has been (re)loaded, this event is fired * manager.upgrade If any plugins have declared a newer schema version than exists in the database, this event will be fired to allow plugins to upgrade their tables * manager.shutdown When the manager is exiting * manager.execute.completed If execution in current process was completed * manager.daemon.started * manager.daemon.completed * manager.db_cleanup """ unit_test = False options = None def __init__(self, options): """ :param options: argparse parsed options object """ global manager assert not manager, 'Only one instance of Manager should be created at a time!' self.options = options self.config_base = None self.config_name = None self.config_path = None self.db_filename = None self.engine = None self.lockfile = None self.database_uri = None self.db_upgraded = False self._has_lock = False self.config = {} self.scheduler = Scheduler(self) self.ipc_server = IPCServer(self, options.ipc_port) manager = self self.initialize() # cannot be imported at module level because of circular references from flexget.utils.simple_persistence import SimplePersistence self.persist = SimplePersistence('manager') log.debug('sys.defaultencoding: %s' % sys.getdefaultencoding()) log.debug('sys.getfilesystemencoding: %s' % sys.getfilesystemencoding()) log.debug('os.path.supports_unicode_filenames: %s' % os.path.supports_unicode_filenames) if db_schema.upgrade_required(): log.info('Database upgrade is required. Attempting now.') # Make sure not to fire the lock-acquired event yet # TODO: Detect if any database upgrading is needed and acquire the lock only in one place with self.acquire_lock(event=False): fire_event('manager.upgrade', self) if manager.db_upgraded: fire_event('manager.db_upgraded', self) fire_event('manager.startup', self) def __del__(self): global manager manager = None def initialize(self): """Separated from __init__ so that unit tests can modify options before loading config.""" self.setup_yaml() self.find_config(create=(self.options.cli_command == 'webui')) self.init_sqlalchemy() fire_event('manager.before_config_load', self) self.load_config() fire_event('manager.before_config_validate', self) errors = self.validate_config() if errors: for error in errors: log.critical("[%s] %s", error.json_pointer, error.message) self.shutdown(finish_queue=False) sys.exit(1) fire_event('manager.config-loaded', self) @property def tasks(self): """A list of tasks in the config""" if not self.config: return [] return self.config.get('tasks', {}).keys() @property def has_lock(self): return self._has_lock def run_cli_command(self): """ Starting point when executing from commandline, dispatch execution to correct destination. * :meth:`.execute_command` * :meth:`.daemon_command` * :meth:`.webui_command` * CLI plugin callback function """ command = self.options.cli_command options = getattr(self.options, command) # First check for built-in commands if command == 'execute': self.execute_command(options) elif command == 'daemon': self.daemon_command(options) elif command == 'webui': self.webui_command(options) else: # Otherwise dispatch the command to the callback function options.cli_command_callback(self, options) def execute_command(self, options): """ Send execute command to daemon through IPC or perform execution on current process. Fires events: * manager.execute.completed :param options: argparse options """ # If a daemon is started, send the execution to the daemon ipc_info = self.check_ipc_info() if ipc_info: client = IPCClient(ipc_info['port'], ipc_info['password']) client.execute(dict(options)) self.shutdown() return # Otherwise we run the execution ourselves with self.acquire_lock(): fire_event('manager.execute.started', self) self.scheduler.start(run_schedules=False) self.scheduler.execute(options) self.scheduler.shutdown(finish_queue=True) try: self.scheduler.wait() except KeyboardInterrupt: log.error( 'Got ctrl-c exiting after this task completes. Press ctrl-c again to abort this task.' ) else: fire_event('manager.execute.completed', self) self.shutdown(finish_queue=False) def daemon_command(self, options): """ Fires events: * manager.daemon.started * manager.daemon.completed :param options: argparse options """ if options.action == 'start': if options.daemonize: self.daemonize() with self.acquire_lock(): signal.signal(signal.SIGTERM, self._handle_sigterm) self.ipc_server.start() fire_event('manager.daemon.started', self) self.scheduler.start() try: self.scheduler.wait() except KeyboardInterrupt: log.info('Got ctrl-c, shutting down.') fire_event('manager.daemon.completed', self) self.shutdown(finish_queue=False) elif options.action == 'stop': ipc_info = self.check_ipc_info() if ipc_info: client = IPCClient(ipc_info['port'], ipc_info['password']) client.shutdown() self.shutdown() else: log.error('There does not appear to be a daemon running.') elif options.action == 'status': ipc_info = self.check_ipc_info() if ipc_info: log.info('Daemon running. (PID: %s)' % ipc_info['pid']) else: log.info('No daemon appears to be running for this config.') def webui_command(self, options): """ :param options: argparse options """ try: pkg_resources.require('flexget[webui]') except pkg_resources.DistributionNotFound as e: log.error('Dependency not met. %s' % e) log.error( 'Webui dependencies not installed. You can use `pip install flexget[webui]` to install them.' ) self.shutdown() return if options.daemonize: self.daemonize() from flexget.ui import webui with self.acquire_lock(): webui.start(self) def _handle_sigterm(self, signum, frame): log.info('Got SIGTERM. Shutting down.') self.shutdown(finish_queue=False) def setup_yaml(self): """Sets up the yaml loader to return unicode objects for strings by default""" def construct_yaml_str(self, node): # Override the default string handling function # to always return unicode objects return self.construct_scalar(node) yaml.Loader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str) yaml.SafeLoader.add_constructor(u'tag:yaml.org,2002:str', construct_yaml_str) # Set up the dumper to not tag every string with !!python/unicode def unicode_representer(dumper, uni): node = yaml.ScalarNode(tag=u'tag:yaml.org,2002:str', value=uni) return node yaml.add_representer(unicode, unicode_representer) # Set up the dumper to increase the indent for lists def increase_indent_wrapper(func): def increase_indent(self, flow=False, indentless=False): func(self, flow, False) return increase_indent yaml.Dumper.increase_indent = increase_indent_wrapper( yaml.Dumper.increase_indent) yaml.SafeDumper.increase_indent = increase_indent_wrapper( yaml.SafeDumper.increase_indent) def find_config(self, create=False): """ Find the configuration file. :param bool create: If a config file is not found, and create is True, one will be created in the home folder """ startup_path = os.path.dirname(os.path.abspath(sys.path[0])) home_path = os.path.join(os.path.expanduser('~'), '.flexget') current_path = os.getcwd() exec_path = sys.path[0] config_path = os.path.dirname(self.options.config) possible = [] if config_path != '': # explicit path given, don't try anything too fancy possible.append(self.options.config) else: log.debug('Figuring out config load paths') # for virtualenv / dev sandbox from flexget import __version__ as version if version == '{git}': log.debug('Running git, adding virtualenv / sandbox paths') possible.append(os.path.join(exec_path, '..')) possible.append(current_path) possible.append(exec_path) # normal lookup locations possible.append(startup_path) possible.append(home_path) if sys.platform.startswith('win'): # On windows look in ~/flexget as well, as explorer does not let you create a folder starting with a dot home_path = os.path.join(os.path.expanduser('~'), 'flexget') possible.append(home_path) else: # The freedesktop.org standard config location xdg_config = os.environ.get( 'XDG_CONFIG_HOME', os.path.join(os.path.expanduser('~'), '.config')) possible.append(os.path.join(xdg_config, 'flexget')) for path in possible: config = os.path.join(path, self.options.config) if os.path.exists(config): log.debug('Found config: %s' % config) break else: if not create: log.info('Tried to read from: %s' % ', '.join(possible)) log.critical('Failed to find configuration file %s' % self.options.config) sys.exit(1) config = os.path.join(home_path, self.options.config) log.info('Config file %s not found. Creating new config %s' % (self.options.config, config)) with open(config, 'w') as newconfig: # Write empty tasks to the config newconfig.write(yaml.dump({'tasks': {}})) self.config_path = config self.config_name = os.path.splitext(os.path.basename(config))[0] self.config_base = os.path.normpath(os.path.dirname(config)) self.lockfile = os.path.join(self.config_base, '.%s-lock' % self.config_name) def load_config(self): """ .. warning:: Calls sys.exit(1) if configuration file could not be loaded. This is something we probably want to change. """ with open(self.config_path, 'rb') as f: config = f.read() try: config = config.decode('utf-8') except UnicodeDecodeError: log.critical('Config file must be UTF-8 encoded.') sys.exit(1) try: self.config = yaml.safe_load(config) or {} except Exception as e: msg = str(e).replace('\n', ' ') msg = ' '.join(msg.split()) log.critical(msg) print('') print('-' * 79) print( ' Malformed configuration file (check messages above). Common reasons:' ) print('-' * 79) print('') print(' o Indentation error') print(' o Missing : from end of the line') print(' o Non ASCII characters (use UTF8)') print(' o If text contains any of :[]{}% characters it must be single-quoted ' \ '(eg. value{1} should be \'value{1}\')\n') # Not very good practice but we get several kind of exceptions here, I'm not even sure all of them # At least: ReaderError, YmlScannerError (or something like that) if hasattr(e, 'problem') and hasattr( e, 'context_mark') and hasattr(e, 'problem_mark'): lines = 0 if e.problem is not None: print(' Reason: %s\n' % e.problem) if e.problem == 'mapping values are not allowed here': print( ' ----> MOST LIKELY REASON: Missing : from end of the line!' ) print('') if e.context_mark is not None: print(' Check configuration near line %s, column %s' % (e.context_mark.line, e.context_mark.column)) lines += 1 if e.problem_mark is not None: print(' Check configuration near line %s, column %s' % (e.problem_mark.line, e.problem_mark.column)) lines += 1 if lines: print('') if lines == 1: print(' Fault is almost always in this or previous line\n') if lines == 2: print( ' Fault is almost always in one of these lines or previous ones\n' ) # When --debug escalate to full stacktrace if self.options.debug: raise sys.exit(1) # config loaded successfully log.debug('config_name: %s' % self.config_name) log.debug('config_base: %s' % self.config_base) def save_config(self): """Dumps current config to yaml config file""" config_file = file( os.path.join(self.config_base, self.config_name) + '.yml', 'w') try: config_file.write(yaml.dump(self.config, default_flow_style=False)) finally: config_file.close() def config_changed(self): """Makes sure that all tasks will have the config_modified flag come out true on the next run. Useful when changing the db and all tasks need to be completely reprocessed.""" from flexget.task import config_changed for task in self.tasks: config_changed(task) def pre_check_config(self, config): """Checks configuration file for common mistakes that are easily detectable""" def get_indentation(line): i, n = 0, len(line) while i < n and line[i] == ' ': i += 1 return i def isodd(n): return bool(n % 2) line_num = 0 duplicates = {} # flags prev_indentation = 0 prev_mapping = False prev_list = True prev_scalar = True list_open = False # multiline list with [ for line in config.splitlines(): if '# warnings off' in line.strip().lower(): log.debug('config pre-check warnings off') break line_num += 1 # remove linefeed line = line.rstrip() # empty line if line.strip() == '': continue # comment line if line.strip().startswith('#'): continue indentation = get_indentation(line) if prev_scalar: if indentation <= prev_indentation: prev_scalar = False else: continue cur_list = line.strip().startswith('-') # skipping lines as long as multiline compact list is open if list_open: if line.strip().endswith(']'): list_open = False # print 'closed list at line %s' % line continue else: list_open = line.strip().endswith( ': [') or line.strip().endswith(':[') if list_open: # print 'list open at line %s' % line continue # print '#%i: %s' % (line_num, line) # print 'indentation: %s, prev_ind: %s, prev_mapping: %s, prev_list: %s, cur_list: %s' % \ # (indentation, prev_indentation, prev_mapping, prev_list, cur_list) if ':\t' in line: log.critical('Line %s has TAB character after : character. ' 'DO NOT use tab key when editing config!' % line_num) elif '\t' in line: log.warning('Line %s has tabs, use only spaces!' % line_num) if isodd(indentation): log.warning('Config line %s has odd (uneven) indentation' % line_num) if indentation > prev_indentation and not prev_mapping: # line increases indentation, but previous didn't start mapping log.warning( 'Config line %s is likely missing \':\' at the end' % (line_num - 1)) if indentation > prev_indentation + 2 and prev_mapping and not prev_list: # mapping value after non list indented more than 2 log.warning('Config line %s is indented too much' % line_num) if indentation <= prev_indentation + ( 2 * (not cur_list)) and prev_mapping and prev_list: log.warning('Config line %s is not indented enough' % line_num) if prev_mapping and cur_list: # list after opening mapping if indentation < prev_indentation or indentation > prev_indentation + 2 + ( 2 * prev_list): log.warning( 'Config line %s containing list element is indented incorrectly' % line_num) elif prev_mapping and indentation <= prev_indentation: # after opening a map, indentation doesn't increase log.warning( 'Config line %s is indented incorrectly (previous line ends with \':\')' % line_num) # notify if user is trying to set same key multiple times in a task (a common mistake) for level in duplicates.iterkeys(): # when indentation goes down, delete everything indented more than that if indentation < level: duplicates[level] = {} if ':' in line: name = line.split(':', 1)[0].strip() ns = duplicates.setdefault(indentation, {}) if name in ns: log.warning( 'Trying to set value for `%s` in line %s, but it is already defined in line %s!' % (name, line_num, ns[name])) ns[name] = line_num prev_indentation = indentation # this line is a mapping (ends with :) prev_mapping = line[-1] == ':' prev_scalar = line[-1] in '|>' # this line is a list prev_list = line.strip()[0] == '-' if prev_list: # This line is in a list, so clear the duplicates, # as duplicates are not always wrong in a list. see #697 duplicates[indentation] = {} log.debug('Pre-checked %s configuration lines' % line_num) def validate_config(self): """ Check all root level keywords are valid. :returns: A list of `ValidationError`s """ return config_schema.process_config(self.config) def init_sqlalchemy(self): """Initialize SQLAlchemy""" try: if [int(part) for part in sqlalchemy.__version__.split('.')] < [0, 7, 0]: print( 'FATAL: SQLAlchemy 0.7.0 or newer required. Please upgrade your SQLAlchemy.', file=sys.stderr) sys.exit(1) except ValueError as e: log.critical( 'Failed to check SQLAlchemy version, you may need to upgrade it' ) # SQLAlchemy if self.database_uri is None: self.db_filename = os.path.join(self.config_base, 'db-%s.sqlite' % self.config_name) if self.options.test: db_test_filename = os.path.join( self.config_base, 'test-%s.sqlite' % self.config_name) log.info('Test mode, creating a copy from database ...') if os.path.exists(self.db_filename): shutil.copy(self.db_filename, db_test_filename) self.db_filename = db_test_filename log.info('Test database created') # in case running on windows, needs double \\ filename = self.db_filename.replace('\\', '\\\\') self.database_uri = 'sqlite:///%s' % filename if self.db_filename and not os.path.exists(self.db_filename): log.verbose('Creating new database %s ...' % self.db_filename) # fire up the engine log.debug('Connecting to: %s' % self.database_uri) try: self.engine = sqlalchemy.create_engine( self.database_uri, echo=self.options.debug_sql, poolclass=SingletonThreadPool, connect_args={'check_same_thread': False}) # assert_unicode=True except ImportError: print( 'FATAL: Unable to use SQLite. Are you running Python 2.5 - 2.7 ?\n' 'Python should normally have SQLite support built in.\n' 'If you\'re running correct version of Python then it is not equipped with SQLite.\n' 'You can try installing `pysqlite`. If you have compiled python yourself, ' 'recompile it with SQLite support.', file=sys.stderr) sys.exit(1) Session.configure(bind=self.engine) # create all tables, doesn't do anything to existing tables try: def before_table_create(event, target, bind, tables=None, **kw): if tables: # We need to acquire a lock if we are creating new tables # TODO: Detect if any database upgrading is needed and acquire the lock only in one place self.acquire_lock(event=False).__enter__() Base.metadata.append_ddl_listener('before-create', before_table_create) Base.metadata.create_all(bind=self.engine) except OperationalError as e: if os.path.exists(self.db_filename): print('%s - make sure you have write permissions to file %s' % (e.message, self.db_filename), file=sys.stderr) else: print( '%s - make sure you have write permissions to directory %s' % (e.message, self.config_base), file=sys.stderr) raise Exception(e.message) def _read_lock(self): """ Read the values from the lock file. Returns None if there is no current lock file. """ if self.lockfile and os.path.exists(self.lockfile): result = {} with open(self.lockfile) as f: lines = [l for l in f.readlines() if l] for line in lines: key, value = line.split(b':', 1) result[key.strip().lower()] = value.strip() for key in result: if result[key].isdigit(): result[key] = int(result[key]) if not pid_exists(result['pid']): return None return result return None def check_lock(self): """Returns True if there is a lock on the database.""" lock_info = self._read_lock() if not lock_info: return False # Don't count it if we hold the lock if os.getpid() == lock_info['pid']: return False return True def check_ipc_info(self): """If a daemon has a lock on the database, return info to connect to IPC.""" lock_info = self._read_lock() if lock_info and 'port' in lock_info: return lock_info return None @contextmanager def acquire_lock(self, event=True): """ :param bool event: If True, the 'manager.lock-acquired' event will be fired after a lock is obtained """ acquired = False try: # Don't do anything if we already have a lock. This means only the outermost call will release the lock file if not self._has_lock: # Exit if there is an existing lock. if self.check_lock(): with open(self.lockfile) as f: pid = f.read() print('Another process (%s) is running, will exit.' % pid.split('\n')[0], file=sys.stderr) print( 'If you\'re sure there is no other instance running, delete %s' % self.lockfile, file=sys.stderr) sys.exit(1) self._has_lock = True self.write_lock() acquired = True if event: fire_event('manager.lock-acquired', self) yield finally: if acquired: self.release_lock() self._has_lock = False def write_lock(self, ipc_info=None): assert self._has_lock with open(self.lockfile, 'w') as f: f.write(b'PID: %s\n' % os.getpid()) if ipc_info: for key in sorted(ipc_info): f.write(b'%s: %s\n' % (key, ipc_info[key])) def release_lock(self): if os.path.exists(self.lockfile): os.remove(self.lockfile) log.debug('Removed %s' % self.lockfile) else: log.debug('Lockfile %s not found' % self.lockfile) def daemonize(self): """Daemonizes the current process. Returns the new pid""" if sys.platform.startswith('win'): log.error('Cannot daemonize on windows') return if threading.activeCount() != 1: log.critical('There are %r active threads. ' 'Daemonizing now may cause strange failures.' % threading.enumerate()) log.info('Daemonizing...') try: pid = os.fork() if pid > 0: # Don't run the exit handlers on the parent atexit._exithandlers = [] # exit first parent sys.exit(0) except OSError as e: sys.stderr.write('fork #1 failed: %d (%s)\n' % (e.errno, e.strerror)) sys.exit(1) # decouple from parent environment os.chdir('/') os.setsid() os.umask(0) # do second fork try: pid = os.fork() if pid > 0: # Don't run the exit handlers on the parent atexit._exithandlers = [] # exit from second parent sys.exit(0) except OSError as e: sys.stderr.write('fork #2 failed: %d (%s)\n' % (e.errno, e.strerror)) sys.exit(1) log.info('Daemonize complete. New PID: %s' % os.getpid()) # redirect standard file descriptors sys.stdout.flush() sys.stderr.flush() si = file('/dev/null', 'r') so = file('/dev/null', 'a+') se = file('/dev/null', 'a+', 0) os.dup2(si.fileno(), sys.stdin.fileno()) os.dup2(so.fileno(), sys.stdout.fileno()) os.dup2(se.fileno(), sys.stderr.fileno()) def db_cleanup(self, force=False): """ Perform database cleanup if cleanup interval has been met. Fires events: * manager.db_cleanup If interval was met. Gives session to do the cleanup as a parameter. :param bool force: Run the cleanup no matter whether the interval has been met. """ expired = self.persist.get('last_cleanup', datetime( 1900, 1, 1)) < datetime.now() - DB_CLEANUP_INTERVAL if force or expired: log.info('Running database cleanup.') session = Session() try: fire_event('manager.db_cleanup', session) session.commit() finally: session.close() # Just in case some plugin was overzealous in its cleaning, mark the config changed self.config_changed() self.persist['last_cleanup'] = datetime.now() else: log.debug('Not running db cleanup, last run %s' % self.persist.get('last_cleanup')) def shutdown(self, finish_queue=True): """ Application is being exited :param bool finish_queue: Should scheduler finish the task queue """ # Wait for scheduler to finish self.scheduler.shutdown(finish_queue=finish_queue) try: self.scheduler.wait() except KeyboardInterrupt: log.debug('Not waiting for scheduler shutdown due to ctrl-c') # show real stack trace in debug mode if manager.options.debug: raise print('**** Keyboard Interrupt ****') fire_event('manager.shutdown', self) if not self.unit_test: # don't scroll "nosetests" summary results when logging is enabled log.debug('Shutting down') self.engine.dispose() # remove temporary database used in test mode if self.options.test: if not 'test' in self.db_filename: raise Exception('trying to delete non test database?') if self._has_lock: os.remove(self.db_filename) log.info('Removed test database') if not self.unit_test: # don't scroll "nosetests" summary results when logging is enabled log.debug('Shutdown completed')