def test_nosession(self, execute_task): persist = SimplePersistence('testplugin') persist['aoeu'] = 'test' assert persist['aoeu'] == 'test' # Make sure it commits and actually persists persist = SimplePersistence('testplugin') assert persist['aoeu'] == 'test'
def welcome_message(manager): # Only run for cli cron executions if manager.options.cli_command != 'execute' or not manager.options.cron: return persistence = SimplePersistence(plugin='welcome') count = persistence.setdefault('count', 5) if not count: return # check for old users, assume old user if db larger than 2 MB if count == 5 and os.stat(manager.db_filename).st_size / 1024 / 1024 >= 2: logger.debug('Looks like old user, skipping welcome message') persistence['count'] = 0 return count -= 1 scheduler = 'scheduler' if sys.platform.startswith('win') else 'crontab' if not count: logger.info( 'FlexGet has been successfully started from {} (--cron). I hope you have {} under control now. This message will not be repeated again.', scheduler, scheduler, ) else: logger.info( '%sFlexGet has been successfully started from %s (--cron). This message will be repeated %i times for your set up verification conveniences.' % ('Congratulations! ' if count == 4 else '', scheduler, count)) persistence['count'] = count
def check_env(manager, options): persistence = SimplePersistence(plugin='cron_env') encoding = sys.getfilesystemencoding() if options.cron: if 'terminal_encoding' in persistence: terminal_encoding = persistence['terminal_encoding'] if terminal_encoding.lower() != encoding.lower(): logger.warning( 'Your cron environment has different filesystem encoding ({}) compared to your terminal environment ({}).', encoding, terminal_encoding, ) if encoding == 'ANSI_X3.4-1968': logger.warning( 'Your current cron environment results filesystem encoding ANSI_X3.4-1968 ' 'which supports only ASCII letters in filenames.') else: log_once( 'Good! Your crontab environment seems to be same as terminal.' ) else: logger.info( 'Please run FlexGet manually once for environment verification purposes.' ) else: logger.debug('Encoding {} stored', encoding) persistence['terminal_encoding'] = encoding
def test_withsession(self): session = Session() persist = SimplePersistence('testplugin', session=session) persist['aoeu'] = 'test' assert persist['aoeu'] == 'test' # Make sure it didn't commit or close our session session.rollback() assert 'aoeu' not in persist
def on_cleanup(session): # Vacuum can take a long time, and is not needed frequently persistence = SimplePersistence('db_vacuum') last_vacuum = persistence.get('last_vacuum') if not last_vacuum or last_vacuum < datetime.now() - VACUUM_INTERVAL: log.info( 'Running VACUUM on database to improve performance and decrease db size.' ) session.execute('VACUUM') persistence['last_vacuum'] = datetime.now()
def __init__(self, options): """ :param options: argparse parsed options object """ global manager assert not manager, 'Only one instance of Manager should be created at a time!' self.options = options self.config_base = None self.config_name = None self.config_path = None self.db_filename = None self.engine = None self.lockfile = None self.database_uri = None self.db_upgraded = False self._has_lock = False self.is_daemon = False self.config = {} self.ipc_server = IPCServer(self, options.ipc_port) self.task_queue = TaskQueue() manager = self log.debug('sys.defaultencoding: %s' % sys.getdefaultencoding()) log.debug('sys.getfilesystemencoding: %s' % sys.getfilesystemencoding()) log.debug('os.path.supports_unicode_filenames: %s' % os.path.supports_unicode_filenames) if codecs.lookup(sys.getfilesystemencoding( )).name == 'ascii' and not os.path.supports_unicode_filenames: log.warning( 'Your locale declares ascii as the filesystem encoding. Any plugins reading filenames from ' 'disk will not work properly for filenames containing non-ascii characters. Make sure your ' 'locale env variables are set up correctly for the environment which is launching FlexGet.' ) self.initialize() # cannot be imported at module level because of circular references from flexget.utils.simple_persistence import SimplePersistence self.persist = SimplePersistence('manager') if db_schema.upgrade_required(): log.info('Database upgrade is required. Attempting now.') # Make sure not to fire the lock-acquired event yet # TODO: Detect if any database upgrading is needed and acquire the lock only in one place with self.acquire_lock(event=False): fire_event('manager.upgrade', self) if manager.db_upgraded: fire_event('manager.db_upgraded', self) fire_event('manager.startup', self)
def on_cleanup(manager): # Vacuum can take a long time, and is not needed frequently persistence = SimplePersistence('db_vacuum') last_vacuum = persistence.get('last_vacuum') if not last_vacuum or last_vacuum < datetime.now() - VACUUM_INTERVAL: logger.info('Running VACUUM on database to improve performance and decrease db size.') with Session() as session: try: session.execute('VACUUM') except OperationalError as e: # Does not work on python 3.6, github issue #1596 logger.error('Could not execute VACUUM command: {}', e) else: persistence['last_vacuum'] = datetime.now()
def __init__(self, options): """ :param options: argparse parsed options object """ global manager assert not manager, 'Only one instance of Manager should be created at a time!' self.options = options self.config_base = None self.config_name = None self.config_path = None self.db_filename = None self.engine = None self.lockfile = None self.database_uri = None self.db_upgraded = False self._has_lock = False self.is_daemon = False self.config = {} self.ipc_server = IPCServer(self, options.ipc_port) self.task_queue = TaskQueue() manager = self self.initialize() # cannot be imported at module level because of circular references from flexget.utils.simple_persistence import SimplePersistence self.persist = SimplePersistence('manager') log.debug('sys.defaultencoding: %s' % sys.getdefaultencoding()) log.debug('sys.getfilesystemencoding: %s' % sys.getfilesystemencoding()) log.debug('os.path.supports_unicode_filenames: %s' % os.path.supports_unicode_filenames) if db_schema.upgrade_required(): log.info('Database upgrade is required. Attempting now.') # Make sure not to fire the lock-acquired event yet # TODO: Detect if any database upgrading is needed and acquire the lock only in one place with self.acquire_lock(event=False): fire_event('manager.upgrade', self) if manager.db_upgraded: fire_event('manager.db_upgraded', self) fire_event('manager.startup', self)
def initialize(self): """ Load plugins, database, and config. Also initializes (but does not start) the task queue and ipc server. This should only be called after obtaining a lock. """ if self.initialized: raise RuntimeError( 'Cannot call initialize on an already initialized manager.') plugin.load_plugins( extra_plugins=[os.path.join(self.config_base, 'plugins')], extra_components=[os.path.join(self.config_base, 'components')], ) # Reparse CLI options now that plugins are loaded if not self.args: self.args = ['--help'] self.options = get_parser().parse_args(self.args) self.task_queue = TaskQueue() self.ipc_server = IPCServer(self, self.options.ipc_port) self.setup_yaml() self.init_sqlalchemy() fire_event('manager.initialize', self) try: self.load_config() except ValueError as e: log.critical('Failed to load config file: %s' % e.args[0]) raise # cannot be imported at module level because of circular references from flexget.utils.simple_persistence import SimplePersistence self.persist = SimplePersistence('manager') if db_schema.upgrade_required(): log.info('Database upgrade is required. Attempting now.') fire_event('manager.upgrade', self) if manager.db_upgraded: fire_event('manager.db_upgraded', self) fire_event('manager.startup', self) self.initialized = True
def __init__(self, options): """ :param options: argparse parsed options object """ global manager assert not manager, 'Only one instance of Manager should be created at a time!' manager = self self.options = options self.config_base = None self.config_name = None self.db_filename = None self.engine = None self.lockfile = None self.database_uri = None self.db_upgraded = False self.config = {} self.tasks = {} self.initialize() # check if stdout is not connected to terminal (cron run) #if not os.isatty(sys.stdout.fileno()): # log.warning('It appears you\'re running from CRON') #else: # log.warning('It appears you\'re running from terminal') # cannot be imported at module level because of circular references from flexget.utils.simple_persistence import SimplePersistence self.persist = SimplePersistence('manager') log.debug('sys.defaultencoding: %s' % sys.getdefaultencoding()) log.debug('sys.getfilesystemencoding: %s' % sys.getfilesystemencoding()) log.debug('os.path.supports_unicode_filenames: %s' % os.path.supports_unicode_filenames) fire_event('manager.upgrade', self) if manager.db_upgraded: fire_event('manager.db_upgraded', self) fire_event('manager.startup', self) self.db_cleanup()
def on_task_input(self, task, config): try: import cloudscraper except ImportError as e: logger.debug('Error importing cloudscraper: {}', e) raise plugin.DependencyError( issued_by='cfscraper', missing='cloudscraper', message='CLOudscraper module required. ImportError: %s' % e, ) scraper = cloudscraper.create_scraper() category = config['category'] persistence = SimplePersistence(plugin='magnetdl') last_magnet = persistence.get(category, None) logger.debug('last_magnet: {}', last_magnet) first_magnet = None stop = False for page in range(0, config['pages']): logger.verbose('Retrieving {} page {}', category, page + 1) url = self._url(category, page) logger.debug('Url: {}', url) try: for entry in self.parse_page(scraper, url): if first_magnet is None: first_magnet = entry['url'] logger.debug('Set first_magnet to {}', first_magnet) persistence[category] = first_magnet if last_magnet == entry['url']: logger.debug('Found page where we have left, stopping') stop = True yield entry except Page404Error: logger.warning('Page {} returned 404, stopping', page) return if stop: return time.sleep(random.randint(1, 5))
from sqlalchemy import Table, Column, Integer, Float, Unicode, Boolean, DateTime, func from sqlalchemy.ext.associationproxy import association_proxy from sqlalchemy.orm import relation from sqlalchemy.schema import ForeignKey from sqlalchemy import func from flexget import db_schema from flexget.utils import requests from flexget.utils.database import with_session, text_date_synonym, json_synonym from flexget.utils.simple_persistence import SimplePersistence log = logging.getLogger('api_tvdb') Base = db_schema.versioned_base('api_tvdb', 6) # This is a FlexGet API key persist = SimplePersistence('api_tvdb') class TVDBRequest(object): API_KEY = '4D297D8CFDE0E105' BASE_URL = 'https://api-beta.thetvdb.com/' BANNER_URL = 'http://thetvdb.com/banners/' def __init__(self, username=None, account_id=None): self.username = username self.account_id = account_id self.auth_key = self.username if self.username else 'default' def get_auth_token(self, refresh=False): tokens = persist.get('auth_tokens') if not tokens:
from flexget.plugin import get_plugin_by_name from flexget.utils import requests from flexget.utils.database import with_session from flexget.utils.simple_persistence import SimplePersistence from flexget.utils.sqlalchemy_utils import table_add_column from flexget.utils.tools import TimedDict Base = db_schema.versioned_base('api_trakt', 4) log = logging.getLogger('api_trakt') # Production Site CLIENT_ID = '57e188bcb9750c79ed452e1674925bc6848bd126e02bb15350211be74c6547af' CLIENT_SECRET = 'db4af7531e8df678b134dbc22445a2c04ebdbdd7213be7f5b6d17dfdfabfcdc2' API_URL = 'https://api-v2launch.trakt.tv/' PIN_URL = 'http://trakt.tv/pin/346' # Stores the last time we checked for updates for shows/movies updated = SimplePersistence('api_trakt') # Oauth account authentication class TraktUserAuth(Base): __tablename__ = 'trakt_user_auth' account = Column(Unicode, primary_key=True) access_token = Column(Unicode) refresh_token = Column(Unicode) created = Column(DateTime) expires = Column(DateTime) def __init__(self, account, access_token, refresh_token, created, expires): self.account = account self.access_token = access_token
def __init__(self, ): self.executed = False self.persistence = SimplePersistence(plugin='welcome')
def __init__(self, ): self.executed = False self.persistence = SimplePersistence(plugin='cron_env')