def test_nosession(self, execute_task):
persist = SimplePersistence('testplugin')
persist['aoeu'] = 'test'
assert persist['aoeu'] == 'test'
# Make sure it commits and actually persists
persist = SimplePersistence('testplugin')
assert persist['aoeu'] == 'test'
def welcome_message(manager):
# Only run for cli cron executions
if manager.options.cli_command != 'execute' or not manager.options.cron:
return
persistence = SimplePersistence(plugin='welcome')
count = persistence.setdefault('count', 5)
if not count:
return
# check for old users, assume old user if db larger than 2 MB
if count == 5 and os.stat(manager.db_filename).st_size / 1024 / 1024 >= 2:
logger.debug('Looks like old user, skipping welcome message')
persistence['count'] = 0
return
count -= 1
scheduler = 'scheduler' if sys.platform.startswith('win') else 'crontab'
if not count:
logger.info(
'FlexGet has been successfully started from {} (--cron). I hope you have {} under control now. This message will not be repeated again.',
scheduler,
scheduler,
)
else:
logger.info(
'%sFlexGet has been successfully started from %s (--cron). This message will be repeated %i times for your set up verification conveniences.'
% ('Congratulations! ' if count == 4 else '', scheduler, count))
persistence['count'] = count
def check_env(manager, options):
persistence = SimplePersistence(plugin='cron_env')
encoding = sys.getfilesystemencoding()
if options.cron:
if 'terminal_encoding' in persistence:
terminal_encoding = persistence['terminal_encoding']
if terminal_encoding.lower() != encoding.lower():
logger.warning(
'Your cron environment has different filesystem encoding ({}) compared to your terminal environment ({}).',
encoding,
terminal_encoding,
)
if encoding == 'ANSI_X3.4-1968':
logger.warning(
'Your current cron environment results filesystem encoding ANSI_X3.4-1968 '
'which supports only ASCII letters in filenames.')
else:
log_once(
'Good! Your crontab environment seems to be same as terminal.'
)
else:
logger.info(
'Please run FlexGet manually once for environment verification purposes.'
)
else:
logger.debug('Encoding {} stored', encoding)
persistence['terminal_encoding'] = encoding
def test_withsession(self):
session = Session()
persist = SimplePersistence('testplugin', session=session)
persist['aoeu'] = 'test'
assert persist['aoeu'] == 'test'
# Make sure it didn't commit or close our session
session.rollback()
assert 'aoeu' not in persist
def on_cleanup(session):
# Vacuum can take a long time, and is not needed frequently
persistence = SimplePersistence('db_vacuum')
last_vacuum = persistence.get('last_vacuum')
if not last_vacuum or last_vacuum < datetime.now() - VACUUM_INTERVAL:
log.info(
'Running VACUUM on database to improve performance and decrease db size.'
)
session.execute('VACUUM')
persistence['last_vacuum'] = datetime.now()
def __init__(self, options):
"""
:param options: argparse parsed options object
"""
global manager
assert not manager, 'Only one instance of Manager should be created at a time!'
self.options = options
self.config_base = None
self.config_name = None
self.config_path = None
self.db_filename = None
self.engine = None
self.lockfile = None
self.database_uri = None
self.db_upgraded = False
self._has_lock = False
self.is_daemon = False
self.config = {}
self.ipc_server = IPCServer(self, options.ipc_port)
self.task_queue = TaskQueue()
manager = self
log.debug('sys.defaultencoding: %s' % sys.getdefaultencoding())
log.debug('sys.getfilesystemencoding: %s' %
sys.getfilesystemencoding())
log.debug('os.path.supports_unicode_filenames: %s' %
os.path.supports_unicode_filenames)
if codecs.lookup(sys.getfilesystemencoding(
)).name == 'ascii' and not os.path.supports_unicode_filenames:
log.warning(
'Your locale declares ascii as the filesystem encoding. Any plugins reading filenames from '
'disk will not work properly for filenames containing non-ascii characters. Make sure your '
'locale env variables are set up correctly for the environment which is launching FlexGet.'
)
self.initialize()
# cannot be imported at module level because of circular references
from flexget.utils.simple_persistence import SimplePersistence
self.persist = SimplePersistence('manager')
if db_schema.upgrade_required():
log.info('Database upgrade is required. Attempting now.')
# Make sure not to fire the lock-acquired event yet
# TODO: Detect if any database upgrading is needed and acquire the lock only in one place
with self.acquire_lock(event=False):
fire_event('manager.upgrade', self)
if manager.db_upgraded:
fire_event('manager.db_upgraded', self)
fire_event('manager.startup', self)
def on_cleanup(manager):
# Vacuum can take a long time, and is not needed frequently
persistence = SimplePersistence('db_vacuum')
last_vacuum = persistence.get('last_vacuum')
if not last_vacuum or last_vacuum < datetime.now() - VACUUM_INTERVAL:
logger.info('Running VACUUM on database to improve performance and decrease db size.')
with Session() as session:
try:
session.execute('VACUUM')
except OperationalError as e:
# Does not work on python 3.6, github issue #1596
logger.error('Could not execute VACUUM command: {}', e)
else:
persistence['last_vacuum'] = datetime.now()
def __init__(self, options):
"""
:param options: argparse parsed options object
"""
global manager
assert not manager, 'Only one instance of Manager should be created at a time!'
self.options = options
self.config_base = None
self.config_name = None
self.config_path = None
self.db_filename = None
self.engine = None
self.lockfile = None
self.database_uri = None
self.db_upgraded = False
self._has_lock = False
self.is_daemon = False
self.config = {}
self.ipc_server = IPCServer(self, options.ipc_port)
self.task_queue = TaskQueue()
manager = self
self.initialize()
# cannot be imported at module level because of circular references
from flexget.utils.simple_persistence import SimplePersistence
self.persist = SimplePersistence('manager')
log.debug('sys.defaultencoding: %s' % sys.getdefaultencoding())
log.debug('sys.getfilesystemencoding: %s' %
sys.getfilesystemencoding())
log.debug('os.path.supports_unicode_filenames: %s' %
os.path.supports_unicode_filenames)
if db_schema.upgrade_required():
log.info('Database upgrade is required. Attempting now.')
# Make sure not to fire the lock-acquired event yet
# TODO: Detect if any database upgrading is needed and acquire the lock only in one place
with self.acquire_lock(event=False):
fire_event('manager.upgrade', self)
if manager.db_upgraded:
fire_event('manager.db_upgraded', self)
fire_event('manager.startup', self)
def initialize(self):
"""
Load plugins, database, and config. Also initializes (but does not start) the task queue and ipc server.
This should only be called after obtaining a lock.
"""
if self.initialized:
raise RuntimeError(
'Cannot call initialize on an already initialized manager.')
plugin.load_plugins(
extra_plugins=[os.path.join(self.config_base, 'plugins')],
extra_components=[os.path.join(self.config_base, 'components')],
)
# Reparse CLI options now that plugins are loaded
if not self.args:
self.args = ['--help']
self.options = get_parser().parse_args(self.args)
self.task_queue = TaskQueue()
self.ipc_server = IPCServer(self, self.options.ipc_port)
self.setup_yaml()
self.init_sqlalchemy()
fire_event('manager.initialize', self)
try:
self.load_config()
except ValueError as e:
log.critical('Failed to load config file: %s' % e.args[0])
raise
# cannot be imported at module level because of circular references
from flexget.utils.simple_persistence import SimplePersistence
self.persist = SimplePersistence('manager')
if db_schema.upgrade_required():
log.info('Database upgrade is required. Attempting now.')
fire_event('manager.upgrade', self)
if manager.db_upgraded:
fire_event('manager.db_upgraded', self)
fire_event('manager.startup', self)
self.initialized = True
def __init__(self, options):
"""
:param options: argparse parsed options object
"""
global manager
assert not manager, 'Only one instance of Manager should be created at a time!'
manager = self
self.options = options
self.config_base = None
self.config_name = None
self.db_filename = None
self.engine = None
self.lockfile = None
self.database_uri = None
self.db_upgraded = False
self.config = {}
self.tasks = {}
self.initialize()
# check if stdout is not connected to terminal (cron run)
#if not os.isatty(sys.stdout.fileno()):
# log.warning('It appears you\'re running from CRON')
#else:
# log.warning('It appears you\'re running from terminal')
# cannot be imported at module level because of circular references
from flexget.utils.simple_persistence import SimplePersistence
self.persist = SimplePersistence('manager')
log.debug('sys.defaultencoding: %s' % sys.getdefaultencoding())
log.debug('sys.getfilesystemencoding: %s' %
sys.getfilesystemencoding())
log.debug('os.path.supports_unicode_filenames: %s' %
os.path.supports_unicode_filenames)
fire_event('manager.upgrade', self)
if manager.db_upgraded:
fire_event('manager.db_upgraded', self)
fire_event('manager.startup', self)
self.db_cleanup()
def on_task_input(self, task, config):
try:
import cloudscraper
except ImportError as e:
logger.debug('Error importing cloudscraper: {}', e)
raise plugin.DependencyError(
issued_by='cfscraper',
missing='cloudscraper',
message='CLOudscraper module required. ImportError: %s' % e,
)
scraper = cloudscraper.create_scraper()
category = config['category']
persistence = SimplePersistence(plugin='magnetdl')
last_magnet = persistence.get(category, None)
logger.debug('last_magnet: {}', last_magnet)
first_magnet = None
stop = False
for page in range(0, config['pages']):
logger.verbose('Retrieving {} page {}', category, page + 1)
url = self._url(category, page)
logger.debug('Url: {}', url)
try:
for entry in self.parse_page(scraper, url):
if first_magnet is None:
first_magnet = entry['url']
logger.debug('Set first_magnet to {}', first_magnet)
persistence[category] = first_magnet
if last_magnet == entry['url']:
logger.debug('Found page where we have left, stopping')
stop = True
yield entry
except Page404Error:
logger.warning('Page {} returned 404, stopping', page)
return
if stop:
return
time.sleep(random.randint(1, 5))
from sqlalchemy import Table, Column, Integer, Float, Unicode, Boolean, DateTime, func
from sqlalchemy.ext.associationproxy import association_proxy
from sqlalchemy.orm import relation
from sqlalchemy.schema import ForeignKey
from sqlalchemy import func
from flexget import db_schema
from flexget.utils import requests
from flexget.utils.database import with_session, text_date_synonym, json_synonym
from flexget.utils.simple_persistence import SimplePersistence
log = logging.getLogger('api_tvdb')
Base = db_schema.versioned_base('api_tvdb', 6)
# This is a FlexGet API key
persist = SimplePersistence('api_tvdb')
class TVDBRequest(object):
API_KEY = '4D297D8CFDE0E105'
BASE_URL = 'https://api-beta.thetvdb.com/'
BANNER_URL = 'http://thetvdb.com/banners/'
def __init__(self, username=None, account_id=None):
self.username = username
self.account_id = account_id
self.auth_key = self.username if self.username else 'default'
def get_auth_token(self, refresh=False):
tokens = persist.get('auth_tokens')
if not tokens:
from flexget.plugin import get_plugin_by_name
from flexget.utils import requests
from flexget.utils.database import with_session
from flexget.utils.simple_persistence import SimplePersistence
from flexget.utils.sqlalchemy_utils import table_add_column
from flexget.utils.tools import TimedDict
Base = db_schema.versioned_base('api_trakt', 4)
log = logging.getLogger('api_trakt')
# Production Site
CLIENT_ID = '57e188bcb9750c79ed452e1674925bc6848bd126e02bb15350211be74c6547af'
CLIENT_SECRET = 'db4af7531e8df678b134dbc22445a2c04ebdbdd7213be7f5b6d17dfdfabfcdc2'
API_URL = 'https://api-v2launch.trakt.tv/'
PIN_URL = 'http://trakt.tv/pin/346'
# Stores the last time we checked for updates for shows/movies
updated = SimplePersistence('api_trakt')
# Oauth account authentication
class TraktUserAuth(Base):
__tablename__ = 'trakt_user_auth'
account = Column(Unicode, primary_key=True)
access_token = Column(Unicode)
refresh_token = Column(Unicode)
created = Column(DateTime)
expires = Column(DateTime)
def __init__(self, account, access_token, refresh_token, created, expires):
self.account = account
self.access_token = access_token
def __init__(self, ):
self.executed = False
self.persistence = SimplePersistence(plugin='welcome')
def __init__(self, ):
self.executed = False
self.persistence = SimplePersistence(plugin='cron_env')
