def on_task_filter(self, task, config): if not task.accepted: logger.debug('No accepted entries, not scanning for existing.') return logger.verbose('Scanning path(s) for existing files.') config = self.prepare_config(config) filenames = {} for folder in config: folder = Path(folder).expanduser() if not folder.exists(): raise plugin.PluginWarning('Path %s does not exist' % folder, logger) for p in folder.rglob('*'): if p.is_file(): key = p.name # windows file system is not case sensitive if platform.system() == 'Windows': key = key.lower() filenames[key] = p for entry in task.accepted: # priority is: filename, location (filename only), title name = Path( entry.get('filename', entry.get('location', entry['title']))).name if platform.system() == 'Windows': name = name.lower() if name in filenames: logger.debug('Found {} in {}', name, filenames[name]) entry.reject('exists in %s' % filenames[name])
def url_rewrite(self, task, entry): soup = self._get_soup(task, entry['url']) link_re = re.compile(r'rarefile\.net.*\.rar$') # grab links from the main entry: blog_entry = soup.find('div', class_="entry") num_links = 0 link_list = None for paragraph in blog_entry.find_all('p'): links = paragraph.find_all('a', href=link_re) if len(links) > num_links: link_list = links num_links = len(links) if 'urls' in entry: urls = list(entry['urls']) else: urls = [] if link_list is not None: for link in link_list: urls.append(normalize_unicode(link['href'])) else: raise UrlRewritingError('No useable links found at %s' % entry['url']) num_links = len(urls) logger.verbose('Found {} links at {}.', num_links, entry['url']) if num_links: entry['urls'] = urls entry['url'] = urls[0] else: raise UrlRewritingError('No useable links found at %s' % entry['url'])
def process_message(self, nickname, channel): """ Pops lines from the line cache and passes them to be parsed :param str nickname: Nickname of who sent the message :param str channel: Channel where the message originated from :return: None """ # If we have announcers defined, ignore any messages not from them if self.announcer_list and nickname not in self.announcer_list: logger.debug('Ignoring message: from non-announcer {}', nickname) self.processing_message = False return # Clean up the messages lines = [ MESSAGE_CLEAN.sub('', line) for line in self.line_cache[channel][nickname] ] logger.debug('Received line(s): {}', u'\n'.join(lines)) # Generate some entries if self.linepatterns: entries = self.entries_from_linepatterns(lines) elif self.multilinepatterns: entries, lines = self.entries_from_multilinepatterns(lines) else: entries = self.entries_from_lines(lines) for entry in entries: # Process the generated entry through the linematched rules if self.tracker_config is not None and entry: entry.update(self.process_tracker_config_rules(entry)) elif self.tracker_config is not None: logger.error('Failed to parse message(s).') self.processing_message = False return entry['title'] = entry.get('irc_torrentname') entry['url'] = entry.get('irc_torrenturl') logger.debug('Entry after processing: {}', dict(entry)) if not entry['url'] or not entry['title']: logger.error('Parsing message failed. Title={}, url={}.', entry['title'], entry['url']) continue logger.verbose('IRC message in {} generated an entry: {}', channel, entry) self.queue_entry(entry) # reset the line cache if self.multilinepatterns and lines: self.line_cache[channel][nickname] = lines logger.debug('Left over lines: {}', '\n'.join(lines)) else: self.line_cache[channel][nickname] = [] self.processing_message = False
def sftp_connect(conf): """ Helper function to connect to an sftp server """ sftp = None tries = CONNECT_TRIES retry_interval = RETRY_INTERVAL while not sftp: try: sftp = pysftp.Connection( host=conf.host, username=conf.username, private_key=conf.private_key, password=conf.password, port=conf.port, private_key_pass=conf.private_key_pass, ) sftp.timeout = SOCKET_TIMEOUT logger.verbose('Connected to {}', conf.host) except Exception as e: if not tries: raise e else: logger.debug('Caught exception: {}', e) logger.warning( 'Failed to connect to {}; waiting {} seconds before retrying.', conf.host, retry_interval, ) time.sleep(retry_interval) tries -= 1 retry_interval += RETRY_STEP return sftp
def exposed_handle_cli(self, args): args = rpyc.utils.classic.obtain(args) logger.verbose('Running command `{}` for client.', ' '.join(args)) parser = get_parser() try: options = parser.parse_args(args, file=self.client_out_stream) except SystemExit as e: if e.code: # TODO: Not sure how to properly propagate the exit code back to client logger.debug('Parsing cli args caused system exit with status {}.', e.code) return context_managers = [] # Don't capture any output when used with --cron if not options.cron: # Monkeypatch the console function to be the one from the client # This means decisions about color formatting, and table sizes can be delayed and # decided based on the client terminal capabilities. context_managers.append( unittest.mock.patch('flexget.terminal._patchable_console', self._conn.root.console) ) if options.loglevel != 'NONE': context_managers.append(capture_logs(self.client_log_sink, level=options.loglevel)) with contextlib.ExitStack() as stack: for cm in context_managers: stack.enter_context(cm) self.manager.handle_cli(options)
def _upload_file(self, source: str, to: str) -> None: if not Path(source).exists(): logger.warning('File no longer exists:', source) return destination = self._get_upload_path(source, to) destination_url: str = urljoin(self.prefix, destination) if not self.path_exists(to): try: self.make_dirs(to) except Exception as e: raise SftpError( f'Failed to create remote directory {to} ({str(e)})') if not self.is_dir(to): raise SftpError(f'Not a directory: {to}') try: self._put_file(source, destination) logger.verbose('Successfully uploaded {} to {}', source, destination_url) # type: ignore except OSError: raise SftpError(f'Remote directory does not exist: {to}') except Exception as e: raise SftpError(f'Failed to upload {source} ({str(e)})')
def on_task_start(self, task, config): if isinstance(config, str): config = {'any': config} assume = namedtuple('assume', ['target', 'quality']) self.assumptions = [] for target, quality in list(config.items()): logger.verbose('New assumption: {} is {}', target, quality) try: target = qualities.Requirements(target) except ValueError: raise plugin.PluginError( '%s is not a valid quality. Forgetting assumption.' % target) try: quality = qualities.get(quality) except ValueError: raise plugin.PluginError( '%s is not a valid quality. Forgetting assumption.' % quality) self.assumptions.append(assume(target, quality)) self.assumptions.sort( key=lambda assumption: self.precision(assumption.target), reverse=True) for assumption in self.assumptions: logger.debug('Target {} - Priority {}', assumption.target, self.precision(assumption.target))
def db_cleanup(manager, session): # Purge task executions older than 1 year result = ( session.query(History).filter(History.time < datetime.now() - timedelta(days=365)).delete() ) if result: logger.verbose('Removed {} accepted entries from history older than 1 year', result)
def on_task_input(self, task, config=None): config = self.build_config(config) url = base_url + config['p_slug'] + config['sort_by'] max_results = config.get('max_results', 1) rcount = 0 next_page = '' logger.verbose('Looking for films in Letterboxd list: {}', url) while next_page is not None and rcount < max_results: try: page = requests.get(url).content except RequestException as e: raise plugin.PluginError( 'Error retrieving list from Letterboxd: %s' % e) soup = get_soup(page) for film in soup.find_all(attrs={config['f_slug']: True}): if rcount < max_results: yield self.parse_film(film, config) if 'max_results' in config: rcount += 1 next_page = soup.select_one('.paginate-nextprev .next') if next_page is not None: next_page = next_page.get('href') if next_page is not None: url = base_url + next_page
def _get_series_info(self, task, config, mediaId): series_info_url = 'https://www.npostart.nl/{0}' series_info = None logger.verbose('Retrieving series info for {}', mediaId) try: response = requests.get(series_info_url.format(mediaId)) logger.debug('Series info found at: {}', response.url) page = get_soup(response.content) series = page.find('section', class_='npo-header-episode-meta') if series: # sometimes the NPO page does not return valid content # create a stub to store the common values for all episodes of this series series_info = { 'npo_url': response.url, # we were redirected to the true URL 'npo_name': series.find('h1').text, 'npo_description': series.find('div', id='metaContent').find('p').text, 'npo_language': 'nl', # hard-code the language as if in NL, for lookup plugins 'npo_version': page.find('meta', attrs={'name': 'generator'})['content'], } # include NPO website version logger.debug('Parsed series info for: {} ({})', series_info['npo_name'], mediaId) except RequestException as e: logger.error('Request error: {}', str(e)) return series_info
def fill_entries_for_url(self, url, params, task): entries = [] logger.verbose("Fetching '{}', with parameters '{}'", url, params) try: r = task.requests.get(url, params=params) except RequestException as e: logger.error("Failed fetching '{}', with parameters '{}': {}", url, params, e) else: rss = feedparser.parse(r.content) logger.debug('Raw RSS: {}', rss) if rss.entries: logger.info('No results returned') for rss_entry in rss.entries: new_entry = Entry() for key in list(rss_entry.keys()): new_entry[key] = rss_entry[key] new_entry['url'] = new_entry['link'] if rss_entry.enclosures: size = int(rss_entry.enclosures[0]['length']) # B new_entry['content_size'] = size / (2**20) # MB entries.append(new_entry) return entries
def purge(manager, session: Session) -> None: """Purge old messages from database""" old = datetime.now() - timedelta(days=365) result = session.query(LogMessage).filter(LogMessage.added < old).delete() if result: logger.verbose('Purged {} entries from log_once table.', result)
def _connect(self, connection_tries: int) -> 'pysftp.Connection': tries: int = connection_tries retry_interval: int = RETRY_INTERVAL_SEC logger.debug('Connecting to {}', self.host) sftp: Optional['pysftp.Connection'] = None while not sftp: try: sftp = pysftp.Connection( host=self.host, username=self.username, private_key=self.private_key, password=self.password, port=self.port, private_key_pass=self.private_key_pass, ) logger.verbose('Connected to {}', self.host) # type: ignore except Exception as e: tries -= 1 if not tries: raise e else: logger.debug('Caught exception: {}', e) logger.warning( 'Failed to connect to {}; waiting {} seconds before retrying.', self.host, retry_interval, ) time.sleep(retry_interval) retry_interval += RETRY_STEP_SEC return sftp
def find_show_id(self, show_name, db_sess): # Check if we have this show id cached show_name = show_name.lower() db_show = db_sess.query(PogcalShow).filter( PogcalShow.name == show_name).first() if db_show: return db_show.id try: page = session.get('http://www.pogdesign.co.uk/cat/showselect.php') except requests.RequestException as e: logger.error( 'Error looking up show show list from pogdesign calendar: {}', e) return # Try to find the show id from pogdesign show list show_re = name_to_re(show_name) soup = get_soup(page.content) search = re.compile(show_re, flags=re.I) show = soup.find(text=search) if show: id = int(show.find_previous('input')['value']) db_sess.add(PogcalShow(id=id, name=show_name)) return id else: logger.verbose( 'Could not find pogdesign calendar id for show `{}`', show_re)
def exposed_handle_cli(self, args): args = rpyc.utils.classic.obtain(args) logger.verbose('Running command `{}` for client.', ' '.join(args)) parser = get_parser() try: options = parser.parse_args(args, file=self.client_out_stream) except SystemExit as e: if e.code: # TODO: Not sure how to properly propagate the exit code back to client logger.debug( 'Parsing cli args caused system exit with status {}.', e.code) return # Saving original terminal size to restore after monkeypatch original_terminal_info = terminal.terminal_info # Monkeypatching terminal_size so it'll work using IPC terminal.terminal_info = self._conn.root.terminal_info context_managers = [] # Don't capture any output when used with --cron if not options.cron: context_managers.append(capture_console(self.client_out_stream)) if options.loglevel != 'NONE': context_managers.append( capture_logs(self.client_log_sink, level=options.loglevel)) try: with contextlib.ExitStack() as stack: for cm in context_managers: stack.enter_context(cm) self.manager.handle_cli(options) finally: # Restoring original terminal_size value terminal.terminal_info = original_terminal_info
def irc_update_config(manager): global irc_manager, config_hash # Exit if we're not running daemon mode if not manager.is_daemon: return config = manager.config.get('irc') # No config, no connections if not config: logger.debug('No irc connections defined in the config') stop_irc(manager) return if irc_bot is None: logger.error( 'ImportError: irc_bot module not found or version is too old. Shutting down daemon.' ) stop_irc(manager) manager.shutdown(finish_queue=False) return config_hash.setdefault('names', {}) new_config_hash = get_config_hash(config) if config_hash.get('config') == new_config_hash: logger.verbose( 'IRC config has not been changed. Not reloading any connections.') return config_hash['manager'] = new_config_hash if irc_manager is not None and irc_manager.is_alive(): irc_manager.update_config(config) else: irc_manager = IRCConnectionManager(config)
def on_task_exit(self, task, config): config = self.prepare_config(config) if not config['enabled'] or task.options.learn: return if not self.client: self.client = self.create_rpc_client(config) tracker_re = re.compile(config['tracker'], re.IGNORECASE) if 'tracker' in config else None preserve_tracker_re = ( re.compile(config['preserve_tracker'], re.IGNORECASE) if 'preserve_tracker' in config else None ) session = self.client.get_session() remove_ids = [] for torrent in self.client.get_torrents(): logger.verbose( 'Torrent "{}": status: "{}" - ratio: {} - date added: {}', torrent.name, torrent.status, torrent.ratio, torrent.date_added, ) downloaded, dummy = self.torrent_info(torrent, config) if not downloaded: continue if config.get('transmission_seed_limits'): seed_ratio_ok, idle_limit_ok = self.check_seed_limits(torrent, session) if not seed_ratio_ok or not idle_limit_ok: continue if 'min_ratio' in config: if torrent.ratio < config['min_ratio']: continue if 'finished_for' in config: # done date might be invalid if this torrent was added to transmission when already completed started_seeding = datetime.fromtimestamp(max(torrent.addedDate, torrent.doneDate)) if started_seeding + parse_timedelta(config['finished_for']) > datetime.now(): continue tracker_hosts = ( urlparse(tracker['announce']).hostname for tracker in torrent.trackers ) if 'tracker' in config: if not any(tracker_re.search(tracker) for tracker in tracker_hosts): continue if 'preserve_tracker' in config: if any(preserve_tracker_re.search(tracker) for tracker in tracker_hosts): continue if config.get('directories'): if not any( re.search(d, torrent.downloadDir, re.IGNORECASE) for d in config['directories'] ): continue if task.options.test: logger.info('Would remove finished torrent `{}` from transmission', torrent.name) continue logger.info('Removing finished torrent `{}` from transmission', torrent.name) remove_ids.append(torrent.id) if remove_ids: self.client.remove_torrent(remove_ids, config.get('delete_files'))
def get_tag_ids(self, entry): tags_ids = [] if not self._tags: self._tags = { t["label"].lower(): t["id"] for t in self.service.get_tags() } for tag in self.config.get("tags", []): if isinstance(tag, int): # Handle tags by id if tag not in self._tags.values(): logger.error( 'Unable to add tag with id {} to entry {} as the tag does not exist in radarr', entry, tag, ) continue tags_ids.append(tag) else: # Handle tags by name tag = entry.render(tag).lower() found = self._tags.get(tag) if not found: logger.verbose('Adding missing tag {} to Radarr', tag) found = self.service.add_tag(tag)["id"] self._tags[tag] = found tags_ids.append(found) return tags_ids
def stop(self): if not self.running: return logger.verbose('Stopping tray icon') self.icon.stop() self.running = False
def discard(self, entry): show = self._find_entry(entry, filters=False) if not show: logger.debug('Did not find matching show in Sonarr for {}, skipping', entry) return self.remove_show(show) logger.verbose('removed show {} from Sonarr', show['title'])
def on_task_input(self, task, config): config = self.prepare_config(config) # Create movie entries by parsing imdb list page(s) html using beautifulsoup logger.verbose('Retrieving imdb list: {}', config['list']) headers = {'Accept-Language': config.get('force_language')} params = {'view': 'detail', 'page': 1} if config['list'] in ['watchlist', 'ratings', 'checkins']: url = 'http://www.imdb.com/user/%s/%s' % (config['user_id'], config['list']) if config['list'] == 'watchlist': params = {'view': 'detail'} else: url = 'http://www.imdb.com/list/%s' % config['list'] if 'all' not in config['type']: title_types = [ TITLE_TYPE_MAP[title_type] for title_type in config['type'] ] params['title_type'] = ','.join(title_types) params['sort'] = 'list_order%2Casc' if config['list'] == 'watchlist': entries = self.parse_react_widget(task, config, url, params, headers) else: entries = self.parse_html_list(task, config, url, params, headers) return entries
def _download_file(self, destination: str, delete_origin: bool, source: str) -> None: destination_path: str = self._get_download_path(source, destination) destination_dir: str = Path(destination_path).parent.as_posix() if Path(destination_path).exists(): logger.verbose( # type: ignore 'Skipping {} because destination file {} already exists.', source, destination_path ) return Path(destination_dir).mkdir(parents=True, exist_ok=True) logger.verbose('Downloading file {} to {}', source, destination) # type: ignore try: self._sftp.get(source, destination_path) except Exception as e: logger.error('Failed to download {} ({})', source, e) if Path(destination_path).exists(): logger.debug('Removing partially downloaded file {}', destination_path) Path(destination_path).unlink() raise e if delete_origin: self.remove_file(source)
def on_task_input(self, task, config): target_task_name = config subtask_name = '{}>{}'.format(task.name, target_task_name) subtask_config = task.manager.config['tasks'].get(target_task_name, {}) # TODO: This seen disabling is sorta hacky, is there a better way? subtask_config.setdefault('seen', False) input_task = Task( task.manager, subtask_name, config=subtask_config, # TODO: Do we want to pass other options through? # TODO: Manual plugin semantics and allow_manual are confusing. Make it less confusing somehow? options={ 'allow_manual': True, 'tasks': [subtask_name] }, output=task.output, session_id=task.session_id, priority=task.priority, ) logger.verbose('Running task `{}` as subtask.', target_task_name) input_task.execute() logger.verbose('Finished running subtask `{}`.', target_task_name) # Create fresh entries to reset state and strip association to old task return [Entry(e) for e in input_task.accepted]
def remove_movie(config, movie_id, test_mode=None): logger.verbose('Deleting movie from Couchpotato') delete_movie_url = CouchPotatoBase.build_url(config.get('base_url'), 'delete', config.get('port'), config.get('api_key')) delete_movie_url += '&id=%s' % movie_id CouchPotatoBase.get_json(delete_movie_url)
def on_task_metainfo(self, task, config): if not self.phase_jobs['metainfo']: # return if no jobs for this phase return modified = sum( self.process(entry, self.phase_jobs['metainfo']) for entry in task.entries) logger.verbose('Modified {} entries.', modified)
def add(self, entry): if not self._find_entry(entry, filters=False): show = self.add_show(entry) if show: self._shows = None logger.verbose('Successfully added show {} to Sonarr', show['title']) else: logger.debug('entry {} already exists in Sonarr list', entry)
def on_task_filter(self, task, config): if not self.phase_jobs['filter']: # return if no jobs for this phase return modified = sum( self.process(entry, self.phase_jobs['filter']) for entry in task.entries + task.rejected) logger.verbose('Modified {} entries.', modified)
def delete_entry(self, client, entry): try: client.delete(entry['torrent_info_hash']) logger.verbose('Deleted {} ({}) in rtorrent ', entry['title'], entry['torrent_info_hash']) except xmlrpc_client.Error as e: entry.fail('Failed to delete: %s' % str(e)) return
def add(self, entry): if not self._find_entry(entry): self._movies = None movie = CouchPotatoBase.add_movie(self.config, entry) logger.verbose('Successfully added movie {} to CouchPotato', movie['info']['original_title']) else: logger.debug('entry {} already exists in couchpotato list', entry)
def init_sqlalchemy(self): """Initialize SQLAlchemy""" try: if [int(part) for part in sqlalchemy.__version__.split('.')] < [0, 7, 0]: print( 'FATAL: SQLAlchemy 0.7.0 or newer required. Please upgrade your SQLAlchemy.', file=sys.stderr, ) sys.exit(1) except ValueError as e: logger.critical('Failed to check SQLAlchemy version, you may need to upgrade it') # SQLAlchemy if self.database_uri is None: # in case running on windows, needs double \\ filename = self.db_filename.replace('\\', '\\\\') self.database_uri = 'sqlite:///%s' % filename if self.db_filename and not os.path.exists(self.db_filename): logger.verbose('Creating new database {} - DO NOT INTERRUPT ...', self.db_filename) # fire up the engine logger.debug('Connecting to: {}', self.database_uri) try: self.engine = sqlalchemy.create_engine( self.database_uri, echo=self.options.debug_sql, connect_args={'check_same_thread': False, 'timeout': 10}, ) except ImportError as e: print( 'FATAL: Unable to use SQLite. Are you running Python 2.7, 3.3 or newer ?\n' 'Python should normally have SQLite support built in.\n' 'If you\'re running correct version of Python then it is not equipped with SQLite.\n' 'You can try installing `pysqlite`. If you have compiled python yourself, ' 'recompile it with SQLite support.\n' 'Error: %s' % e, file=sys.stderr, ) sys.exit(1) Session.configure(bind=self.engine) # create all tables, doesn't do anything to existing tables try: Base.metadata.create_all(bind=self.engine) except OperationalError as e: if os.path.exists(self.db_filename): print( '%s - make sure you have write permissions to file %s' % (e.message, self.db_filename), file=sys.stderr, ) else: print( '%s - make sure you have write permissions to directory %s' % (e.message, self.config_base), file=sys.stderr, ) raise