def on_task_filter(self, task, config):
if not task.accepted:
logger.debug('No accepted entries, not scanning for existing.')
return
logger.verbose('Scanning path(s) for existing files.')
config = self.prepare_config(config)
filenames = {}
for folder in config:
folder = Path(folder).expanduser()
if not folder.exists():
raise plugin.PluginWarning('Path %s does not exist' % folder,
logger)
for p in folder.rglob('*'):
if p.is_file():
key = p.name
# windows file system is not case sensitive
if platform.system() == 'Windows':
key = key.lower()
filenames[key] = p
for entry in task.accepted:
# priority is: filename, location (filename only), title
name = Path(
entry.get('filename', entry.get('location',
entry['title']))).name
if platform.system() == 'Windows':
name = name.lower()
if name in filenames:
logger.debug('Found {} in {}', name, filenames[name])
entry.reject('exists in %s' % filenames[name])
def url_rewrite(self, task, entry):
soup = self._get_soup(task, entry['url'])
link_re = re.compile(r'rarefile\.net.*\.rar$')
# grab links from the main entry:
blog_entry = soup.find('div', class_="entry")
num_links = 0
link_list = None
for paragraph in blog_entry.find_all('p'):
links = paragraph.find_all('a', href=link_re)
if len(links) > num_links:
link_list = links
num_links = len(links)
if 'urls' in entry:
urls = list(entry['urls'])
else:
urls = []
if link_list is not None:
for link in link_list:
urls.append(normalize_unicode(link['href']))
else:
raise UrlRewritingError('No useable links found at %s' % entry['url'])
num_links = len(urls)
logger.verbose('Found {} links at {}.', num_links, entry['url'])
if num_links:
entry['urls'] = urls
entry['url'] = urls[0]
else:
raise UrlRewritingError('No useable links found at %s' % entry['url'])
def process_message(self, nickname, channel):
"""
Pops lines from the line cache and passes them to be parsed
:param str nickname: Nickname of who sent the message
:param str channel: Channel where the message originated from
:return: None
"""
# If we have announcers defined, ignore any messages not from them
if self.announcer_list and nickname not in self.announcer_list:
logger.debug('Ignoring message: from non-announcer {}', nickname)
self.processing_message = False
return
# Clean up the messages
lines = [
MESSAGE_CLEAN.sub('', line)
for line in self.line_cache[channel][nickname]
]
logger.debug('Received line(s): {}', u'\n'.join(lines))
# Generate some entries
if self.linepatterns:
entries = self.entries_from_linepatterns(lines)
elif self.multilinepatterns:
entries, lines = self.entries_from_multilinepatterns(lines)
else:
entries = self.entries_from_lines(lines)
for entry in entries:
# Process the generated entry through the linematched rules
if self.tracker_config is not None and entry:
entry.update(self.process_tracker_config_rules(entry))
elif self.tracker_config is not None:
logger.error('Failed to parse message(s).')
self.processing_message = False
return
entry['title'] = entry.get('irc_torrentname')
entry['url'] = entry.get('irc_torrenturl')
logger.debug('Entry after processing: {}', dict(entry))
if not entry['url'] or not entry['title']:
logger.error('Parsing message failed. Title={}, url={}.',
entry['title'], entry['url'])
continue
logger.verbose('IRC message in {} generated an entry: {}', channel,
entry)
self.queue_entry(entry)
# reset the line cache
if self.multilinepatterns and lines:
self.line_cache[channel][nickname] = lines
logger.debug('Left over lines: {}', '\n'.join(lines))
else:
self.line_cache[channel][nickname] = []
self.processing_message = False
def sftp_connect(conf):
"""
Helper function to connect to an sftp server
"""
sftp = None
tries = CONNECT_TRIES
retry_interval = RETRY_INTERVAL
while not sftp:
try:
sftp = pysftp.Connection(
host=conf.host,
username=conf.username,
private_key=conf.private_key,
password=conf.password,
port=conf.port,
private_key_pass=conf.private_key_pass,
)
sftp.timeout = SOCKET_TIMEOUT
logger.verbose('Connected to {}', conf.host)
except Exception as e:
if not tries:
raise e
else:
logger.debug('Caught exception: {}', e)
logger.warning(
'Failed to connect to {}; waiting {} seconds before retrying.',
conf.host,
retry_interval,
)
time.sleep(retry_interval)
tries -= 1
retry_interval += RETRY_STEP
return sftp
def exposed_handle_cli(self, args):
args = rpyc.utils.classic.obtain(args)
logger.verbose('Running command `{}` for client.', ' '.join(args))
parser = get_parser()
try:
options = parser.parse_args(args, file=self.client_out_stream)
except SystemExit as e:
if e.code:
# TODO: Not sure how to properly propagate the exit code back to client
logger.debug('Parsing cli args caused system exit with status {}.', e.code)
return
context_managers = []
# Don't capture any output when used with --cron
if not options.cron:
# Monkeypatch the console function to be the one from the client
# This means decisions about color formatting, and table sizes can be delayed and
# decided based on the client terminal capabilities.
context_managers.append(
unittest.mock.patch('flexget.terminal._patchable_console', self._conn.root.console)
)
if options.loglevel != 'NONE':
context_managers.append(capture_logs(self.client_log_sink, level=options.loglevel))
with contextlib.ExitStack() as stack:
for cm in context_managers:
stack.enter_context(cm)
self.manager.handle_cli(options)
def _upload_file(self, source: str, to: str) -> None:
if not Path(source).exists():
logger.warning('File no longer exists:', source)
return
destination = self._get_upload_path(source, to)
destination_url: str = urljoin(self.prefix, destination)
if not self.path_exists(to):
try:
self.make_dirs(to)
except Exception as e:
raise SftpError(
f'Failed to create remote directory {to} ({str(e)})')
if not self.is_dir(to):
raise SftpError(f'Not a directory: {to}')
try:
self._put_file(source, destination)
logger.verbose('Successfully uploaded {} to {}', source,
destination_url) # type: ignore
except OSError:
raise SftpError(f'Remote directory does not exist: {to}')
except Exception as e:
raise SftpError(f'Failed to upload {source} ({str(e)})')
def on_task_start(self, task, config):
if isinstance(config, str):
config = {'any': config}
assume = namedtuple('assume', ['target', 'quality'])
self.assumptions = []
for target, quality in list(config.items()):
logger.verbose('New assumption: {} is {}', target, quality)
try:
target = qualities.Requirements(target)
except ValueError:
raise plugin.PluginError(
'%s is not a valid quality. Forgetting assumption.' %
target)
try:
quality = qualities.get(quality)
except ValueError:
raise plugin.PluginError(
'%s is not a valid quality. Forgetting assumption.' %
quality)
self.assumptions.append(assume(target, quality))
self.assumptions.sort(
key=lambda assumption: self.precision(assumption.target),
reverse=True)
for assumption in self.assumptions:
logger.debug('Target {} - Priority {}', assumption.target,
self.precision(assumption.target))
def db_cleanup(manager, session):
# Purge task executions older than 1 year
result = (
session.query(History).filter(History.time < datetime.now() - timedelta(days=365)).delete()
)
if result:
logger.verbose('Removed {} accepted entries from history older than 1 year', result)
def on_task_input(self, task, config=None):
config = self.build_config(config)
url = base_url + config['p_slug'] + config['sort_by']
max_results = config.get('max_results', 1)
rcount = 0
next_page = ''
logger.verbose('Looking for films in Letterboxd list: {}', url)
while next_page is not None and rcount < max_results:
try:
page = requests.get(url).content
except RequestException as e:
raise plugin.PluginError(
'Error retrieving list from Letterboxd: %s' % e)
soup = get_soup(page)
for film in soup.find_all(attrs={config['f_slug']: True}):
if rcount < max_results:
yield self.parse_film(film, config)
if 'max_results' in config:
rcount += 1
next_page = soup.select_one('.paginate-nextprev .next')
if next_page is not None:
next_page = next_page.get('href')
if next_page is not None:
url = base_url + next_page
def _get_series_info(self, task, config, mediaId):
series_info_url = 'https://www.npostart.nl/{0}'
series_info = None
logger.verbose('Retrieving series info for {}', mediaId)
try:
response = requests.get(series_info_url.format(mediaId))
logger.debug('Series info found at: {}', response.url)
page = get_soup(response.content)
series = page.find('section', class_='npo-header-episode-meta')
if series: # sometimes the NPO page does not return valid content
# create a stub to store the common values for all episodes of this series
series_info = {
'npo_url':
response.url, # we were redirected to the true URL
'npo_name':
series.find('h1').text,
'npo_description':
series.find('div', id='metaContent').find('p').text,
'npo_language':
'nl', # hard-code the language as if in NL, for lookup plugins
'npo_version':
page.find('meta', attrs={'name': 'generator'})['content'],
} # include NPO website version
logger.debug('Parsed series info for: {} ({})',
series_info['npo_name'], mediaId)
except RequestException as e:
logger.error('Request error: {}', str(e))
return series_info
def fill_entries_for_url(self, url, params, task):
entries = []
logger.verbose("Fetching '{}', with parameters '{}'", url, params)
try:
r = task.requests.get(url, params=params)
except RequestException as e:
logger.error("Failed fetching '{}', with parameters '{}': {}", url,
params, e)
else:
rss = feedparser.parse(r.content)
logger.debug('Raw RSS: {}', rss)
if rss.entries:
logger.info('No results returned')
for rss_entry in rss.entries:
new_entry = Entry()
for key in list(rss_entry.keys()):
new_entry[key] = rss_entry[key]
new_entry['url'] = new_entry['link']
if rss_entry.enclosures:
size = int(rss_entry.enclosures[0]['length']) # B
new_entry['content_size'] = size / (2**20) # MB
entries.append(new_entry)
return entries
def purge(manager, session: Session) -> None:
"""Purge old messages from database"""
old = datetime.now() - timedelta(days=365)
result = session.query(LogMessage).filter(LogMessage.added < old).delete()
if result:
logger.verbose('Purged {} entries from log_once table.', result)
def _connect(self, connection_tries: int) -> 'pysftp.Connection':
tries: int = connection_tries
retry_interval: int = RETRY_INTERVAL_SEC
logger.debug('Connecting to {}', self.host)
sftp: Optional['pysftp.Connection'] = None
while not sftp:
try:
sftp = pysftp.Connection(
host=self.host,
username=self.username,
private_key=self.private_key,
password=self.password,
port=self.port,
private_key_pass=self.private_key_pass,
)
logger.verbose('Connected to {}', self.host) # type: ignore
except Exception as e:
tries -= 1
if not tries:
raise e
else:
logger.debug('Caught exception: {}', e)
logger.warning(
'Failed to connect to {}; waiting {} seconds before retrying.',
self.host,
retry_interval,
)
time.sleep(retry_interval)
retry_interval += RETRY_STEP_SEC
return sftp
def find_show_id(self, show_name, db_sess):
# Check if we have this show id cached
show_name = show_name.lower()
db_show = db_sess.query(PogcalShow).filter(
PogcalShow.name == show_name).first()
if db_show:
return db_show.id
try:
page = session.get('http://www.pogdesign.co.uk/cat/showselect.php')
except requests.RequestException as e:
logger.error(
'Error looking up show show list from pogdesign calendar: {}',
e)
return
# Try to find the show id from pogdesign show list
show_re = name_to_re(show_name)
soup = get_soup(page.content)
search = re.compile(show_re, flags=re.I)
show = soup.find(text=search)
if show:
id = int(show.find_previous('input')['value'])
db_sess.add(PogcalShow(id=id, name=show_name))
return id
else:
logger.verbose(
'Could not find pogdesign calendar id for show `{}`', show_re)
def exposed_handle_cli(self, args):
args = rpyc.utils.classic.obtain(args)
logger.verbose('Running command `{}` for client.', ' '.join(args))
parser = get_parser()
try:
options = parser.parse_args(args, file=self.client_out_stream)
except SystemExit as e:
if e.code:
# TODO: Not sure how to properly propagate the exit code back to client
logger.debug(
'Parsing cli args caused system exit with status {}.',
e.code)
return
# Saving original terminal size to restore after monkeypatch
original_terminal_info = terminal.terminal_info
# Monkeypatching terminal_size so it'll work using IPC
terminal.terminal_info = self._conn.root.terminal_info
context_managers = []
# Don't capture any output when used with --cron
if not options.cron:
context_managers.append(capture_console(self.client_out_stream))
if options.loglevel != 'NONE':
context_managers.append(
capture_logs(self.client_log_sink, level=options.loglevel))
try:
with contextlib.ExitStack() as stack:
for cm in context_managers:
stack.enter_context(cm)
self.manager.handle_cli(options)
finally:
# Restoring original terminal_size value
terminal.terminal_info = original_terminal_info
def irc_update_config(manager):
global irc_manager, config_hash
# Exit if we're not running daemon mode
if not manager.is_daemon:
return
config = manager.config.get('irc')
# No config, no connections
if not config:
logger.debug('No irc connections defined in the config')
stop_irc(manager)
return
if irc_bot is None:
logger.error(
'ImportError: irc_bot module not found or version is too old. Shutting down daemon.'
)
stop_irc(manager)
manager.shutdown(finish_queue=False)
return
config_hash.setdefault('names', {})
new_config_hash = get_config_hash(config)
if config_hash.get('config') == new_config_hash:
logger.verbose(
'IRC config has not been changed. Not reloading any connections.')
return
config_hash['manager'] = new_config_hash
if irc_manager is not None and irc_manager.is_alive():
irc_manager.update_config(config)
else:
irc_manager = IRCConnectionManager(config)
def on_task_exit(self, task, config):
config = self.prepare_config(config)
if not config['enabled'] or task.options.learn:
return
if not self.client:
self.client = self.create_rpc_client(config)
tracker_re = re.compile(config['tracker'], re.IGNORECASE) if 'tracker' in config else None
preserve_tracker_re = (
re.compile(config['preserve_tracker'], re.IGNORECASE)
if 'preserve_tracker' in config
else None
)
session = self.client.get_session()
remove_ids = []
for torrent in self.client.get_torrents():
logger.verbose(
'Torrent "{}": status: "{}" - ratio: {} - date added: {}',
torrent.name,
torrent.status,
torrent.ratio,
torrent.date_added,
)
downloaded, dummy = self.torrent_info(torrent, config)
if not downloaded:
continue
if config.get('transmission_seed_limits'):
seed_ratio_ok, idle_limit_ok = self.check_seed_limits(torrent, session)
if not seed_ratio_ok or not idle_limit_ok:
continue
if 'min_ratio' in config:
if torrent.ratio < config['min_ratio']:
continue
if 'finished_for' in config:
# done date might be invalid if this torrent was added to transmission when already completed
started_seeding = datetime.fromtimestamp(max(torrent.addedDate, torrent.doneDate))
if started_seeding + parse_timedelta(config['finished_for']) > datetime.now():
continue
tracker_hosts = (
urlparse(tracker['announce']).hostname for tracker in torrent.trackers
)
if 'tracker' in config:
if not any(tracker_re.search(tracker) for tracker in tracker_hosts):
continue
if 'preserve_tracker' in config:
if any(preserve_tracker_re.search(tracker) for tracker in tracker_hosts):
continue
if config.get('directories'):
if not any(
re.search(d, torrent.downloadDir, re.IGNORECASE) for d in config['directories']
):
continue
if task.options.test:
logger.info('Would remove finished torrent `{}` from transmission', torrent.name)
continue
logger.info('Removing finished torrent `{}` from transmission', torrent.name)
remove_ids.append(torrent.id)
if remove_ids:
self.client.remove_torrent(remove_ids, config.get('delete_files'))
def get_tag_ids(self, entry):
tags_ids = []
if not self._tags:
self._tags = {
t["label"].lower(): t["id"]
for t in self.service.get_tags()
}
for tag in self.config.get("tags", []):
if isinstance(tag, int):
# Handle tags by id
if tag not in self._tags.values():
logger.error(
'Unable to add tag with id {} to entry {} as the tag does not exist in radarr',
entry,
tag,
)
continue
tags_ids.append(tag)
else:
# Handle tags by name
tag = entry.render(tag).lower()
found = self._tags.get(tag)
if not found:
logger.verbose('Adding missing tag {} to Radarr', tag)
found = self.service.add_tag(tag)["id"]
self._tags[tag] = found
tags_ids.append(found)
return tags_ids
def stop(self):
if not self.running:
return
logger.verbose('Stopping tray icon')
self.icon.stop()
self.running = False
def discard(self, entry):
show = self._find_entry(entry, filters=False)
if not show:
logger.debug('Did not find matching show in Sonarr for {}, skipping', entry)
return
self.remove_show(show)
logger.verbose('removed show {} from Sonarr', show['title'])
def on_task_input(self, task, config):
config = self.prepare_config(config)
# Create movie entries by parsing imdb list page(s) html using beautifulsoup
logger.verbose('Retrieving imdb list: {}', config['list'])
headers = {'Accept-Language': config.get('force_language')}
params = {'view': 'detail', 'page': 1}
if config['list'] in ['watchlist', 'ratings', 'checkins']:
url = 'http://www.imdb.com/user/%s/%s' % (config['user_id'],
config['list'])
if config['list'] == 'watchlist':
params = {'view': 'detail'}
else:
url = 'http://www.imdb.com/list/%s' % config['list']
if 'all' not in config['type']:
title_types = [
TITLE_TYPE_MAP[title_type] for title_type in config['type']
]
params['title_type'] = ','.join(title_types)
params['sort'] = 'list_order%2Casc'
if config['list'] == 'watchlist':
entries = self.parse_react_widget(task, config, url, params,
headers)
else:
entries = self.parse_html_list(task, config, url, params, headers)
return entries
def _download_file(self, destination: str, delete_origin: bool, source: str) -> None:
destination_path: str = self._get_download_path(source, destination)
destination_dir: str = Path(destination_path).parent.as_posix()
if Path(destination_path).exists():
logger.verbose( # type: ignore
'Skipping {} because destination file {} already exists.', source, destination_path
)
return
Path(destination_dir).mkdir(parents=True, exist_ok=True)
logger.verbose('Downloading file {} to {}', source, destination) # type: ignore
try:
self._sftp.get(source, destination_path)
except Exception as e:
logger.error('Failed to download {} ({})', source, e)
if Path(destination_path).exists():
logger.debug('Removing partially downloaded file {}', destination_path)
Path(destination_path).unlink()
raise e
if delete_origin:
self.remove_file(source)
def on_task_input(self, task, config):
target_task_name = config
subtask_name = '{}>{}'.format(task.name, target_task_name)
subtask_config = task.manager.config['tasks'].get(target_task_name, {})
# TODO: This seen disabling is sorta hacky, is there a better way?
subtask_config.setdefault('seen', False)
input_task = Task(
task.manager,
subtask_name,
config=subtask_config,
# TODO: Do we want to pass other options through?
# TODO: Manual plugin semantics and allow_manual are confusing. Make it less confusing somehow?
options={
'allow_manual': True,
'tasks': [subtask_name]
},
output=task.output,
session_id=task.session_id,
priority=task.priority,
)
logger.verbose('Running task `{}` as subtask.', target_task_name)
input_task.execute()
logger.verbose('Finished running subtask `{}`.', target_task_name)
# Create fresh entries to reset state and strip association to old task
return [Entry(e) for e in input_task.accepted]
def remove_movie(config, movie_id, test_mode=None):
logger.verbose('Deleting movie from Couchpotato')
delete_movie_url = CouchPotatoBase.build_url(config.get('base_url'),
'delete',
config.get('port'),
config.get('api_key'))
delete_movie_url += '&id=%s' % movie_id
CouchPotatoBase.get_json(delete_movie_url)
def on_task_metainfo(self, task, config):
if not self.phase_jobs['metainfo']:
# return if no jobs for this phase
return
modified = sum(
self.process(entry, self.phase_jobs['metainfo'])
for entry in task.entries)
logger.verbose('Modified {} entries.', modified)
def add(self, entry):
if not self._find_entry(entry, filters=False):
show = self.add_show(entry)
if show:
self._shows = None
logger.verbose('Successfully added show {} to Sonarr', show['title'])
else:
logger.debug('entry {} already exists in Sonarr list', entry)
def on_task_filter(self, task, config):
if not self.phase_jobs['filter']:
# return if no jobs for this phase
return
modified = sum(
self.process(entry, self.phase_jobs['filter'])
for entry in task.entries + task.rejected)
logger.verbose('Modified {} entries.', modified)
def delete_entry(self, client, entry):
try:
client.delete(entry['torrent_info_hash'])
logger.verbose('Deleted {} ({}) in rtorrent ', entry['title'],
entry['torrent_info_hash'])
except xmlrpc_client.Error as e:
entry.fail('Failed to delete: %s' % str(e))
return
def add(self, entry):
if not self._find_entry(entry):
self._movies = None
movie = CouchPotatoBase.add_movie(self.config, entry)
logger.verbose('Successfully added movie {} to CouchPotato',
movie['info']['original_title'])
else:
logger.debug('entry {} already exists in couchpotato list', entry)
def init_sqlalchemy(self):
"""Initialize SQLAlchemy"""
try:
if [int(part) for part in sqlalchemy.__version__.split('.')] < [0, 7, 0]:
print(
'FATAL: SQLAlchemy 0.7.0 or newer required. Please upgrade your SQLAlchemy.',
file=sys.stderr,
)
sys.exit(1)
except ValueError as e:
logger.critical('Failed to check SQLAlchemy version, you may need to upgrade it')
# SQLAlchemy
if self.database_uri is None:
# in case running on windows, needs double \\
filename = self.db_filename.replace('\\', '\\\\')
self.database_uri = 'sqlite:///%s' % filename
if self.db_filename and not os.path.exists(self.db_filename):
logger.verbose('Creating new database {} - DO NOT INTERRUPT ...', self.db_filename)
# fire up the engine
logger.debug('Connecting to: {}', self.database_uri)
try:
self.engine = sqlalchemy.create_engine(
self.database_uri,
echo=self.options.debug_sql,
connect_args={'check_same_thread': False, 'timeout': 10},
)
except ImportError as e:
print(
'FATAL: Unable to use SQLite. Are you running Python 2.7, 3.3 or newer ?\n'
'Python should normally have SQLite support built in.\n'
'If you\'re running correct version of Python then it is not equipped with SQLite.\n'
'You can try installing `pysqlite`. If you have compiled python yourself, '
'recompile it with SQLite support.\n'
'Error: %s' % e,
file=sys.stderr,
)
sys.exit(1)
Session.configure(bind=self.engine)
# create all tables, doesn't do anything to existing tables
try:
Base.metadata.create_all(bind=self.engine)
except OperationalError as e:
if os.path.exists(self.db_filename):
print(
'%s - make sure you have write permissions to file %s'
% (e.message, self.db_filename),
file=sys.stderr,
)
else:
print(
'%s - make sure you have write permissions to directory %s'
% (e.message, self.config_base),
file=sys.stderr,
)
raise
