def __init__(self): """Set up the schema""" self.schema = { 'type': 'object', 'properties': { 'username': {'type': 'string'}, 'password': {'type': 'string'}, 'user_agent': {'type': 'string'}, 'search': {'type': 'string'}, 'artist': {'type': 'string'}, 'album': {'type': 'string'}, 'year': {'type': ['string', 'integer']}, 'tags': one_or_more({'type': 'string'}), 'tag_type': {'type': 'string', 'enum': self._opts('tag_type').keys()}, 'encoding': {'type': 'string', 'enum': self._opts('encoding')}, 'format': {'type': 'string', 'enum': self._opts('format')}, 'media': {'type': 'string', 'enum': self._opts('media')}, 'release_type': {'type': 'string', 'enum': self._opts('release_type').keys()}, 'log': {'oneOf': [{'type': 'string', 'enum': self._opts('log').keys()}, {'type': 'boolean'}]}, 'leech_type': {'type': 'string', 'enum': self._opts('leech_type').keys()}, 'hascue': {'type': 'boolean'}, 'scene': {'type': 'boolean'}, 'vanityhouse': {'type': 'boolean'}, }, 'required': ['username', 'password'], 'additionalProperties': False }
def __init__(self): self.schema = { "type": "object", "properties": { "category": {"type": "string"}, "terms": one_or_more({"type": "string"}), "max_results": {"type": "number", "default": 100}, }, "additionalProperties": False, }
def __init__(self): self.schema = { 'type': 'object', 'properties': { 'category': {'type': 'string'}, 'terms': one_or_more({'type': 'string'}), 'max_results': {'type': 'number', 'default': 100} }, 'additionalProperties': False }
def schema(self): """The schema of the plugin Extends the super's schema """ schema = super(InputGazelleMusic, self).schema schema['properties'].update({ 'artist': {'type': 'string'}, 'album': {'type': 'string'}, 'year': {'type': ['string', 'integer']}, 'tags': one_or_more({'type': 'string'}), 'tag_type': {'type': 'string', 'enum': list(self._opts('tag_type').keys())}, 'encoding': {'type': 'string', 'enum': self._opts('encoding')}, 'format': {'type': 'string', 'enum': self._opts('format')}, 'media': {'type': 'string', 'enum': self._opts('media')}, 'release_type': {'type': 'string', 'enum': list(self._opts('release_type').keys())}, 'log': {'oneOf': [{'type': 'string', 'enum': list(self._opts('log').keys())}, {'type': 'boolean'}]}, 'leech_type': {'type': 'string', 'enum': list(self._opts('leech_type').keys())}, 'hascue': {'type': 'boolean'}, 'scene': {'type': 'boolean'}, 'vanityhouse': {'type': 'boolean'} }) return schema
class SearchAlphaRatio(object): """ AlphaRatio search plugin. """ schema = { 'type': 'object', 'properties': { 'username': {'type': 'string'}, 'password': {'type': 'string'}, 'category': one_or_more({'type': 'string', 'enum': list(CATEGORIES.keys())}, unique_items=True), 'order_by': {'type': 'string', 'enum': ['seeders', 'leechers', 'time', 'size', 'year', 'snatched'], 'default': 'time'}, 'order_desc': {'type': 'boolean', 'default': True}, 'scene': {'type': 'boolean'}, 'leechstatus': {'type': 'string', 'enum': list(LEECHSTATUS.keys()), 'default': 'normal'}, }, 'required': ['username', 'password'], 'additionalProperties': False } base_url = 'https://alpharatio.cc/' errors = False def get(self, url, params, username, password, force=False): """ Wrapper to allow refreshing the cookie if it is invalid for some reason :param unicode url: :param dict params: :param str username: :param str password: :param bool force: flag used to refresh the cookie forcefully ie. forgo DB lookup :return: """ cookies = self.get_login_cookie(username, password, force=force) response = requests.get(url, params=params, cookies=cookies) if self.base_url + 'login.php' in response.url: if self.errors: raise plugin.PluginError('AlphaRatio login cookie is invalid. Login page received?') self.errors = True # try again response = self.get(url, params, username, password, force=True) else: self.errors = False return response def get_login_cookie(self, username, password, force=False): """ Retrieves login cookie :param str username: :param str password: :param bool force: if True, then retrieve a fresh cookie instead of looking in the DB :return: """ if not force: with Session() as session: saved_cookie = session.query(AlphaRatioCookie).filter(AlphaRatioCookie.username == username).first() if saved_cookie and saved_cookie.expires and saved_cookie.expires >= datetime.datetime.now(): log.debug('Found valid login cookie') return saved_cookie.cookie url = self.base_url + 'login.php' try: log.debug('Attempting to retrieve AlphaRatio cookie') response = requests.post(url, data={'username': username, 'password': password, 'login': '******', 'keeplogged': '1'}, timeout=30) except RequestException as e: raise plugin.PluginError('AlphaRatio login failed: %s', e) if 'Your username or password was incorrect.' in response.text: raise plugin.PluginError('AlphaRatio login failed: Your username or password was incorrect.') with Session() as session: expires = None for c in requests.cookies: if c.name == 'session': expires = c.expires if expires: expires = datetime.datetime.fromtimestamp(expires) log.debug('Saving or updating AlphaRatio cookie in db') cookie = AlphaRatioCookie(username=username, cookie=dict(requests.cookies), expires=expires) session.merge(cookie) return cookie.cookie @plugin.internet(log) def search(self, task, entry, config): """ Search for entries on AlphaRatio """ params = {} if 'category' in config: categories = config['category'] if isinstance(config['category'], list) else [config['category']] for category in categories: params[CATEGORIES[category]] = 1 if 'scene' in config: params['scene'] = int(config['scene']) ordering = 'desc' if config['order_desc'] else 'asc' entries = set() params.update({'order_by': config['order_by'], 'search_submit': 1, 'action': 'basic', 'order_way': ordering, 'freeleech': LEECHSTATUS[config['leechstatus']]}) for search_string in entry.get('search_strings', [entry['title']]): params['searchstr'] = search_string log.debug('Using search params: %s', params) try: page = self.get(self.base_url + 'torrents.php', params, config['username'], config['password']) log.debug('requesting: %s', page.url) except RequestException as e: log.error('AlphaRatio request failed: %s', e) continue soup = get_soup(page.content) for result in soup.findAll('tr', attrs={'class': 'torrent'}): group_info = result.find('td', attrs={'class': 'big_info'}).find('div', attrs={'class': 'group_info'}) title = group_info.find('a', href=re.compile('torrents.php\?id=\d+')).text url = self.base_url + \ group_info.find('a', href=re.compile('torrents.php\?action=download(?!usetoken)'))['href'] torrent_info = result.findAll('td') log.debug('AlphaRatio size: %s', torrent_info[5].text) size = re.search('(\d+(?:[.,]\d+)*)\s?([KMGTP]B)', torrent_info[4].text) torrent_tags = ', '.join([tag.text for tag in group_info.findAll('div', attrs={'class': 'tags'})]) e = Entry() e['title'] = title e['url'] = url e['torrent_tags'] = torrent_tags e['content_size'] = parse_filesize(size.group(0)) e['torrent_snatches'] = int(torrent_info[5].text) e['torrent_seeds'] = int(torrent_info[6].text) e['torrent_leeches'] = int(torrent_info[7].text) entries.add(e) return entries
'type': 'string', 'pattern': '^([#&][^\x07\x2C\s]{0,200})', 'error_pattern': 'channel name must start with # or & and contain no commas and whitespace', } schema = { 'oneOf': [ { 'type': 'object', 'additionalProperties': { 'type': 'object', 'properties': { 'tracker_file': {'type': 'string'}, 'server': {'type': 'string'}, 'port': {'type': 'integer'}, 'nickname': {'type': 'string'}, 'channels': one_or_more(channel_pattern), 'nickserv_password': {'type': 'string'}, 'invite_nickname': {'type': 'string'}, 'invite_message': {'type': 'string'}, 'task': one_or_more({'type': 'string'}), 'task_re': { 'type': 'array', 'items': { 'type': 'object', 'properties': { 'task': {'type': 'string'}, 'patterns': { 'type': 'array', 'items': { 'type': 'object', 'properties': {
channel_pattern = { 'type': 'string', 'pattern': '^([#&][^\x07\x2C\s]{0,200})', 'error_pattern': 'channel name must start with # or & and contain no commas and whitespace' } schema = { 'oneOf': [ { 'type': 'object', 'additionalProperties': { 'type': 'object', 'properties': { 'tracker_file': {'type': 'string'}, 'server': {'type': 'string'}, 'port': {'type': 'integer'}, 'nickname': {'type': 'string'}, 'channels': one_or_more(channel_pattern), 'nickserv_password': {'type': 'string'}, 'invite_nickname': {'type': 'string'}, 'invite_message': {'type': 'string'}, 'task': one_or_more({ 'type': 'string' }), 'task_re': { 'type': 'object', 'additionalProperties': one_or_more({ 'type': 'object', 'properties': { 'regexp': {'type': 'string'}, 'field': {'type': 'string'} }, 'required': ['regexp', 'field'],
class PluginPyLoad(object): """ Parse task content or url for hoster links and adds them to pyLoad. Example:: pyload: api: http://localhost:8000/api queue: yes username: my_username password: my_password folder: desired_folder package: desired_package_name (jinja2 supported) package_password: desired_package_password hoster: - YoutubeCom parse_url: no multiple_hoster: yes enabled: yes Default values for the config elements:: pyload: api: http://localhost:8000/api queue: no hoster: ALL parse_url: no multiple_hoster: yes enabled: yes """ __author__ = 'http://pyload.org' __version__ = '0.5' DEFAULT_API = 'http://localhost:8000/api' DEFAULT_QUEUE = False DEFAULT_FOLDER = '' DEFAULT_HOSTER = [] DEFAULT_PARSE_URL = False DEFAULT_MULTIPLE_HOSTER = True DEFAULT_PREFERRED_HOSTER_ONLY = False DEFAULT_HANDLE_NO_URL_AS_FAILURE = False schema = { 'oneOf': [ {'type': 'boolean'}, {'type': 'object', 'properties': { 'api': {'type': 'string'}, 'username': {'type': 'string'}, 'password': {'type': 'string'}, 'folder': {'type': 'string'}, 'package': {'type': 'string'}, 'package_password': {'type': 'string'}, 'queue': {'type': 'boolean'}, 'parse_url': {'type': 'boolean'}, 'multiple_hoster': {'type': 'boolean'}, 'hoster': one_or_more({'type': 'string'}), 'preferred_hoster_only': {'type': 'boolean'}, 'handle_no_url_as_failure': {'type': 'boolean'}, 'enabled': {'type': 'boolean'}, }, 'additionalProperties': False } ] } def on_task_output(self, task, config): if not config.get('enabled', True): return if not task.accepted: return self.add_entries(task, config) def add_entries(self, task, config): """Adds accepted entries""" apiurl = config.get('api', self.DEFAULT_API) api = PyloadApi(task.requests, apiurl) try: session = api.get_session(config) except IOError: raise plugin.PluginError('pyLoad not reachable', log) except plugin.PluginError: raise except Exception as e: raise plugin.PluginError('Unknown error: %s' % str(e), log) hoster = config.get('hoster', self.DEFAULT_HOSTER) for entry in task.accepted: # bunch of urls now going to check content = entry.get('description', '') + ' ' + quote(entry['url']) content = json.dumps(content.encode("utf8")) url = json.dumps(entry['url']) if config.get('parse_url', self.DEFAULT_PARSE_URL) else "''" log.debug("Parsing url %s" % url) result = api.query("parseURLs", {"html": content, "url": url, "session": session}) # parsed { plugins: [urls] } parsed = result.json() urls = [] # check for preferred hoster for name in hoster: if name in parsed: urls.extend(parsed[name]) if not config.get('multiple_hoster', self.DEFAULT_MULTIPLE_HOSTER): break # no preferred hoster and not preferred hoster only - add all recognized plugins if not urls and not config.get('preferred_hoster_only', self.DEFAULT_PREFERRED_HOSTER_ONLY): for name, purls in parsed.items(): if name != "BasePlugin": urls.extend(purls) if task.options.test: log.info('Would add `%s` to pyload' % urls) continue # no urls found if not urls: if config.get('handle_no_url_as_failure', self.DEFAULT_HANDLE_NO_URL_AS_FAILURE): entry.fail("No suited urls in entry %s" % entry['title']) else: log.info("No suited urls in entry %s" % entry['title']) continue log.debug("Add %d urls to pyLoad" % len(urls)) try: dest = 1 if config.get('queue', self.DEFAULT_QUEUE) else 0 # Destination.Queue = 1 # Use the title of the entry, if no naming schema for the package is defined. name = config.get('package', entry['title']) # If name has jinja template, render it try: name = entry.render(name) except RenderError as e: name = entry['title'] log.error('Error rendering jinja event: %s' % e) post = {'name': "'%s'" % name.encode("ascii", "ignore"), 'links': str(urls), 'dest': dest, 'session': session} pid = api.query("addPackage", post).text log.debug('added package pid: %s' % pid) # Set Folder folder = config.get('folder', self.DEFAULT_FOLDER) folder = entry.get('path', folder) if folder: # If folder has jinja template, render it try: folder = entry.render(folder) except RenderError as e: folder = self.DEFAULT_FOLDER log.error('Error rendering jinja event: %s' % e) # set folder with api data = json.dumps({'folder': folder}) api.query("setPackageData", {'pid': pid, 'data': data, 'session': session}) # Set Package Password package_password = config.get('package_password') if package_password: data = json.dumps({'password': package_password}) api.query("setPackageData", {'pid': pid, 'data': data, 'session': session}) except Exception as e: entry.fail(str(e))
class FromIMDB(object): """ This plugin enables generating entries based on an entity, an entity being a person, character or company. It's based on IMDBpy which is required (pip install imdbpy). The basic config required just an IMDB ID of the required entity. For example: from_imdb: ch0001354 Schema description: Other than ID, all other properties are meant to filter the full list that the entity generates. id: string that relates to a supported entity type. For example: 'nm0000375'. Required. job_types: a string or list with job types from job_types. Default is 'actor'. content_types: A string or list with content types from content_types. Default is 'movie'. max_entries: The maximum number of entries that can return. This value's purpose is basically flood protection against unruly configurations that will return too many results. Default is 200. Advanced config example: dynamic_movie_queue: from_imdb: id: co0051941 job_types: - actor - director content_types: tv series accept_all: yes movie_queue: add """ job_types = [ 'actor', 'actress', 'director', 'producer', 'writer', 'self', 'editor', 'miscellaneous', 'editorial department', 'cinematographer', 'visual effects', 'thanks', 'music department', 'in development', 'archive footage', 'soundtrack' ] content_types = [ 'movie', 'tv series', 'tv mini series', 'video game', 'video movie', 'tv movie', 'episode' ] content_type_conversion = { 'movie': 'movie', 'tv series': 'tv', 'tv mini series': 'tv', 'tv movie': 'tv', 'episode': 'tv', 'video movie': 'video', 'video game': 'video game' } character_content_type_conversion = { 'movie': 'feature', 'tv series': 'tv', 'tv mini series': 'tv', 'tv movie': 'tv', 'episode': 'tv', 'video movie': 'video', 'video game': 'video-game', } jobs_without_content_type = [ 'actor', 'actress', 'self', 'in development', 'archive footage' ] imdb_pattern = one_or_more( { 'type': 'string', 'pattern': r'(nm|co|ch)\d{7}', 'error_pattern': 'Get the id from the url of the person/company you want to use,' ' e.g. http://imdb.com/text/<id here>/blah' }, unique_items=True) schema = { 'oneOf': [ imdb_pattern, { 'type': 'object', 'properties': { 'id': imdb_pattern, 'job_types': one_or_more({ 'type': 'string', 'enum': job_types }, unique_items=True), 'content_types': one_or_more({ 'type': 'string', 'enum': content_types }, unique_items=True), 'max_entries': { 'type': 'integer' }, 'match_type': { 'type': 'string', 'enum': ['strict', 'loose'] } }, 'required': ['id'], 'additionalProperties': False } ], } def prepare_config(self, config): """ Converts config to dict form and sets defaults if needed """ config = config if isinstance(config, basestring): config = {'id': [config]} elif isinstance(config, list): config = {'id': config} if isinstance(config, dict) and not isinstance(config['id'], list): config['id'] = [config['id']] config.setdefault('content_types', [self.content_types[0]]) config.setdefault('job_types', [self.job_types[0]]) config.setdefault('max_entries', 200) config.setdefault('match_type', 'strict') if isinstance(config.get('content_types'), str_types): log.debug('Converted content type from string to list.') config['content_types'] = [config['content_types']] if isinstance(config['job_types'], str_types): log.debug('Converted job type from string to list.') config['job_types'] = [config['job_types']] # Special case in case user meant to add actress instead of actor (different job types in IMDB) if 'actor' in config['job_types'] and 'actress' not in config[ 'job_types']: config['job_types'].append('actress') return config def get_items(self, config): items = [] for id in config['id']: try: entity_type, entity_object = self.get_entity_type_and_object( id) except Exception as e: log.error( 'Could not resolve entity via ID: {}. ' 'Either error in config or unsupported entity. Error:{}'. format(id, e)) continue items += self.get_items_by_entity(entity_type, entity_object, config.get('content_types'), config.get('job_types'), config.get('match_type')) return set(items) def get_entity_type_and_object(self, imdb_id): """ Return a tuple of entity type and entity object :param imdb_id: string which contains IMDB id :return: entity type, entity object (person, company, etc.) """ if imdb_id.startswith('nm'): person = self.ia.get_person(imdb_id[2:]) log.info('Starting to retrieve items for person: %s' % person) return 'Person', person elif imdb_id.startswith('co'): company = self.ia.get_company(imdb_id[2:]) log.info('Starting to retrieve items for company: %s' % company) return 'Company', company elif imdb_id.startswith('ch'): character = self.ia.get_character(imdb_id[2:]) log.info('Starting to retrieve items for Character: %s' % character) return 'Character', character def get_items_by_entity(self, entity_type, entity_object, content_types, job_types, match_type): """ Gets entity object and return movie list using relevant method """ if entity_type == 'Company': return self.items_by_company(entity_object) if entity_type == 'Character': return self.items_by_character(entity_object, content_types, match_type) elif entity_type == 'Person': return self.items_by_person(entity_object, job_types, content_types, match_type) def flatten_list(self, _list): """ Gets a list of lists and returns a flat list """ for el in _list: if isinstance(el, collections.Iterable) and not isinstance( el, basestring): for sub in self.flatten_list(el): yield sub else: yield el def flat_list(self, non_flat_list, remove_none=False): flat_list = self.flatten_list(non_flat_list) if remove_none: flat_list = [_f for _f in flat_list if _f] return flat_list def filtered_items(self, unfiltered_items, content_types, match_type): items = [] unfiltered_items = set(unfiltered_items) for item in sorted(unfiltered_items): if match_type == 'strict': log.debug( 'Match type is strict, verifying item type to requested content types' ) self.ia.update(item) if item['kind'] in content_types: log.verbose( 'Adding item "{}" to list. Item kind is "{}"'.format( item, item['kind'])) items.append(item) else: log.verbose('Rejecting item "{}". Item kind is "{}'.format( item, item['kind'])) else: log.debug('Match type is loose, all items are being added') items.append(item) return items def items_by_person(self, person, job_types, content_types, match_type): """ Return item list for a person object """ unfiltered_items = self.flat_list([ self.items_by_job_type(person, job_type, content_types) for job_type in job_types ], remove_none=True) return self.filtered_items(unfiltered_items, content_types, match_type) def items_by_content_type(self, person, job_type, content_type): return [ _f for _f in (person.get( job_type + ' ' + self.content_type_conversion[content_type], [])) if _f ] def items_by_job_type(self, person, job_type, content_types): items = person.get( job_type, []) if job_type in self.jobs_without_content_type else [ person.get(job_type + ' ' + 'documentary', []) and person.get(job_type + ' ' + 'short', []) and self.items_by_content_type(person, job_type, content_type) if content_type == 'movie' else self.items_by_content_type( person, job_type, content_type) for content_type in content_types ] return [_f for _f in items if _f] def items_by_character(self, character, content_types, match_type): """ Return items list for a character object :param character: character object :param content_types: content types as defined in config :return: """ unfiltered_items = self.flat_list([ character.get(self.character_content_type_conversion[content_type]) for content_type in content_types ], remove_none=True) return self.filtered_items(unfiltered_items, content_types, match_type) def items_by_company(self, company): """ Return items list for a company object :param company: company object :return: company items list """ return company.get('production companies') @cached('from_imdb', persist='2 hours') def on_task_input(self, task, config): try: from imdb import IMDb self.ia = IMDb() except ImportError: log.error( 'IMDBPY is required for this plugin. Please install using "pip install imdbpy"' ) return entries = [] config = self.prepare_config(config) items = self.get_items(config) if not items: log.error( 'Could not get IMDB item list, check your configuration.') return for item in items: entry = Entry(title=item['title'], imdb_id='tt' + self.ia.get_imdbID(item), url='', imdb_url=self.ia.get_imdbURL(item)) if entry.isvalid(): if entry not in entries: entries.append(entry) if entry and task.options.test: log.info("Test mode. Entry includes:") for key, value in list(entry.items()): log.info(' {}: {}'.format( key.capitalize(), value)) else: log.error('Invalid entry created? %s' % entry) if len(entries) <= config.get('max_entries'): return entries else: log.warning( 'Number of entries (%s) exceeds maximum allowed value %s. ' 'Edit your filters or raise the maximum value by entering a higher "max_entries"' % (len(entries), config.get('max_entries'))) return
class OutputPushover(object): """ Example:: pushover: userkey: <USER_KEY> (can also be a list of userkeys) apikey: <API_KEY> [device: <DEVICE_STRING>] (default: (none)) [title: <MESSAGE_TITLE>] (default: "Download started" -- accepts Jinja2) [message: <MESSAGE_BODY>] (default uses series/tvdb name and imdb if available -- accepts Jinja2) [priority: <PRIORITY>] (default = 0 -- normal = 0, high = 1, silent = -1, emergency = 2) [url: <URL>] (default: "{{imdb_url}}" -- accepts Jinja2) [urltitle: <URL_TITLE>] (default: (none) -- accepts Jinja2) [sound: <SOUND>] (default: pushover default) [retry]: <RETRY>] Configuration parameters are also supported from entries (eg. through set). """ defaults = { 'message': "{% if series_name is defined %}" "{{tvdb_series_name|d(series_name)}} " "{{series_id}} {{tvdb_ep_name|d('')}}" "{% elif imdb_name is defined %}" "{{imdb_name}} {{imdb_year}}" "{% else %}" "{{title}}" "{% endif %}", 'url': '{% if imdb_url is defined %}{{imdb_url}}{% endif %}', 'title': '{{task}}' } schema = { 'type': 'object', 'properties': { 'userkey': one_or_more({'type': 'string'}), 'apikey': {'type': 'string'}, 'device': {'type': 'string'}, 'title': {'type': 'string'}, 'message': {'type': 'string'}, 'priority': {'oneOf': [ {'type': 'number', 'minimum': -2, 'maximum': 2}, {'type': 'string'}]}, 'url': {'type': 'string'}, 'url_title': {'type': 'string'}, 'sound': {'type': 'string'}, 'retry': {'type': 'integer', 'minimum': 30}, 'expire': {'type': 'integer', 'maximum': 86400}, 'callback': {'type': 'string', 'format': 'url'} }, 'required': ['userkey', 'apikey'], 'additionalProperties': False } last_request = datetime.datetime.strptime('2000-01-01', '%Y-%m-%d') @staticmethod def pushover_request(task, data): time_dif = (datetime.datetime.now() - OutputPushover.last_request).seconds # Require at least 5 seconds of waiting between API calls while time_dif < 5: time_dif = (datetime.datetime.now() - OutputPushover.last_request).seconds try: response = task.requests.post(PUSHOVER_URL, data=data, raise_status=False) OutputPushover.last_request = datetime.datetime.now() except RequestException: raise return response def prepare_config(self, config): """ Returns prepared config with Flexget default values :param config: User config :return: Config with defaults """ config = config if not isinstance(config['userkey'], list): config['userkey'] = [config['userkey']] config.setdefault('message', self.defaults['message']) config.setdefault('title', self.defaults['title']) config.setdefault('url', self.defaults['url']) return config # Run last to make sure other outputs are successful before sending notification @plugin.priority(0) def on_task_output(self, task, config): config = self.prepare_config(config) data = {"token": config["apikey"]} # Loop through the provided entries for entry in task.accepted: for key, value in list(config.items()): if key in ['apikey', 'userkey']: continue # Tried to render data in field try: data[key] = entry.render(value) except RenderError as e: log.warning('Problem rendering {0}: {1}'.format(key, e)) data[key] = None except ValueError: data[key] = None # If field is empty or rendering fails, try to render field default if exists if not data[key]: try: data[key] = entry.render(self.defaults.get(key)) except ValueError: if value: data[key] = value # Special case, verify certain fields exists if priority is 2 if data.get('priority') == 2 and not all([data.get('expire'), data.get('retry')]): log.warning('Priority set to 2 but fields "expire" and "retry" are not both present.' ' Lowering priority to 1') data['priority'] = 1 for userkey in config['userkey']: # Build the request data["user"] = userkey # Check for test mode if task.options.test: log.info("Test mode. Pushover notification would be:") for key, value in list(data.items()): log.verbose('{0:>5}{1}: {2}'.format('', key.capitalize(), value)) # Test mode. Skip remainder. continue for retry in range(NUMBER_OF_RETRIES): try: response = self.pushover_request(task, data) except RequestException as e: log.warning('Could not get response from Pushover: {0}.' ' Try {1} out of {2}'.format(e, retry + 1, NUMBER_OF_RETRIES)) continue request_status = response.status_code # error codes and messages from Pushover API if request_status == 200: log.debug("Pushover notification sent") break elif request_status == 500: log.debug("Pushover notification failed, Pushover API having issues. Try {0} out of {1}".format( retry + 1, NUMBER_OF_RETRIES)) continue elif request_status >= 400: errors = json.loads(response.content)['errors'] log.error("Pushover API error: {0}".format(errors[0])) break else: log.error("Unknown error when sending Pushover notification") break else: log.error( 'Could not get response from Pushover after {0} retries, aborting.'.format(NUMBER_OF_RETRIES))
def schema(self): """The schema of the plugin Extends the super's schema """ schema = super().schema schema['properties'].update({ 'artist': { 'type': 'string' }, 'album': { 'type': 'string' }, 'year': { 'type': ['string', 'integer'] }, 'tags': one_or_more({'type': 'string'}), 'tag_type': { 'type': 'string', 'enum': list(self._opts('tag_type').keys()) }, 'encoding': { 'type': 'string', 'enum': self._opts('encoding') }, 'format': { 'type': 'string', 'enum': self._opts('format') }, 'media': { 'type': 'string', 'enum': self._opts('media') }, 'release_type': { 'type': 'string', 'enum': list(self._opts('release_type').keys()), }, 'log': { 'oneOf': [ { 'type': 'string', 'enum': list(self._opts('log').keys()) }, { 'type': 'boolean' }, ] }, 'leech_type': { 'type': 'string', 'enum': list(self._opts('leech_type').keys()) }, 'hascue': { 'type': 'boolean' }, 'scene': { 'type': 'boolean' }, 'vanityhouse': { 'type': 'boolean' }, }) return schema
class UrlRewriteTorrentleech: """ Torrentleech urlrewriter and search plugin. torrentleech: rss_key: xxxxxxxxx (required) username: xxxxxxxx (required) password: xxxxxxxx (required) category: HD Category is any combination of: all, Cam, TS, TS/TC, DVDRip, DVDRip/DVDScreener, WEBRip': 37, HDRip': 43, BDRip, DVDR, DVD-R, HD, Bluray, 4KUpscaled, Real4K, Movie Boxsets, Boxsets': 15, Documentaries, Episodes, TV Boxsets, Episodes HD """ schema = { 'type': 'object', 'properties': { 'rss_key': {'type': 'string'}, 'username': {'type': 'string'}, 'password': {'type': 'string'}, 'category': one_or_more( {'oneOf': [{'type': 'integer'}, {'type': 'string', 'enum': list(CATEGORIES)}]} ), }, 'required': ['rss_key', 'username', 'password'], 'additionalProperties': False, } # urlrewriter API def url_rewritable(self, task, entry): url = entry['url'] if url.endswith('.torrent'): return False if url.startswith('https://www.torrentleech.org/'): return True return False # urlrewriter API def url_rewrite(self, task, entry): if 'url' not in entry: logger.error("Didn't actually get a URL...") else: logger.debug('Got the URL: {}', entry['url']) if entry['url'].startswith('https://www.torrentleech.org/torrents/browse/list/query/'): # use search results = self.search(task, entry) if not results: raise UrlRewritingError("No search results found") # TODO: Search doesn't enforce close match to title, be more picky entry['url'] = results[0]['url'] @plugin.internet(logger) def search(self, task, entry, config=None): """ Search for name from torrentleech. """ request_headers = {'User-Agent': 'curl/7.54.0'} rss_key = config['rss_key'] # build the form request: data = {'username': config['username'], 'password': config['password']} # POST the login form: try: login = task.requests.post( 'https://www.torrentleech.org/user/account/login/', data=data, headers=request_headers, allow_redirects=True, ) except RequestException as e: raise PluginError('Could not connect to torrentleech: %s' % str(e)) if login.url.endswith('/user/account/login/'): raise PluginError('Could not login to torrentleech, faulty credentials?') if not isinstance(config, dict): config = {} # sort = SORT.get(config.get('sort_by', 'seeds')) # if config.get('sort_reverse'): # sort += 1 categories = config.get('category', 'all') # Make sure categories is a list if not isinstance(categories, list): categories = [categories] # If there are any text categories, turn them into their id number categories = [c if isinstance(c, int) else CATEGORIES[c] for c in categories] filter_url = '/categories/{}'.format(','.join(str(c) for c in categories)) entries = set() for search_string in entry.get('search_strings', [entry['title']]): query = normalize_unicode(search_string).replace(":", "") # urllib.quote will crash if the unicode string has non ascii characters, # so encode in utf-8 beforehand url = ( 'https://www.torrentleech.org/torrents/browse/list/query/' + quote(query.encode('utf-8')) + filter_url ) logger.debug('Using {} as torrentleech search url', url) results = task.requests.get(url, headers=request_headers, cookies=login.cookies).json() for torrent in results['torrentList']: entry = Entry() entry['download_headers'] = request_headers entry['title'] = torrent['name'] # construct download URL torrent_url = 'https://www.torrentleech.org/rss/download/{}/{}/{}'.format( torrent['fid'], rss_key, torrent['filename'] ) logger.debug('RSS-ified download link: {}', torrent_url) entry['url'] = torrent_url # seeders/leechers entry['torrent_seeds'] = torrent['seeders'] entry['torrent_leeches'] = torrent['leechers'] entry['torrent_availability'] = torrent_availability( entry['torrent_seeds'], entry['torrent_leeches'] ) entry['content_size'] = parse_filesize(str(torrent['size']) + ' b') entries.add(entry) return sorted(entries, reverse=True, key=lambda x: x.get('torrent_availability'))
class InputRSS(object): """ Parses RSS feed. Hazzlefree configuration for public rss feeds:: rss: <url> Configuration with basic http authentication:: rss: url: <url> username: <name> password: <password> Advanced usages: You may wish to clean up the entry by stripping out all non-ascii characters. This can be done by setting ascii value to yes. Example:: rss: url: <url> ascii: yes In case RSS-feed uses some nonstandard field for urls and automatic detection fails you can configure plugin to use url from any feedparser entry attribute. Example:: rss: url: <url> link: guid If you want to keep information in another rss field attached to the flexget entry, you can use the other_fields option. Example:: rss: url: <url> other_fields: [date] You can disable few possibly annoying warnings by setting silent value to yes on feeds where there are frequently invalid items. Example:: rss: url: <url> silent: yes You can group all the links of an item, to make the download plugin tolerant to broken urls: it will try to download each url until one works. Links are enclosures plus item fields given by the link value, in that order. The value to set is "group_links". Example:: rss: url: <url> group_links: yes """ schema = { 'type': ['string', 'object'], # Simple form, just url or file 'anyOf': [{ 'format': 'url' }, { 'format': 'file' }], # Advanced form, with options 'properties': { 'url': { 'type': 'string', 'anyOf': [{ 'format': 'url' }, { 'format': 'file' }] }, 'username': { 'type': 'string' }, 'password': { 'type': 'string' }, 'title': { 'type': 'string' }, 'link': one_or_more({'type': 'string'}), 'silent': { 'type': 'boolean', 'default': False }, 'ascii': { 'type': 'boolean', 'default': False }, 'filename': { 'type': 'boolean' }, 'group_links': { 'type': 'boolean', 'default': False }, 'all_entries': { 'type': 'boolean', 'default': True }, 'other_fields': { 'type': 'array', 'items': { # Items can be a string, or a dict with a string value 'type': ['string', 'object'], 'additionalProperties': { 'type': 'string' } } } }, 'required': ['url'], 'additionalProperties': False } def build_config(self, config): """Set default values to config""" if isinstance(config, basestring): config = {'url': config} # set the default link value to 'auto' config.setdefault('link', 'auto') # Convert any field names from the config to format feedparser will use for 'link', 'title' and 'other_fields' if config['link'] != 'auto': if not isinstance(config['link'], list): config['link'] = [config['link']] config['link'] = map(fp_field_name, config['link']) config.setdefault('title', 'title') config['title'] = fp_field_name(config['title']) if config.get('other_fields'): other_fields = [] for item in config['other_fields']: if isinstance(item, basestring): key, val = item, item else: key, val = item.items()[0] other_fields.append({fp_field_name(key): val.lower()}) config['other_fields'] = other_fields # set default value for group_links as deactivated config.setdefault('group_links', False) # set default for all_entries config.setdefault('all_entries', True) return config def process_invalid_content(self, task, data, url): """If feedparser reports error, save the received data and log error.""" if data is None: log.critical('Received empty page - no content') return ext = 'xml' if '<html>' in data.lower(): log.critical('Received content is HTML page, not an RSS feed') ext = 'html' if 'login' in data.lower() or 'username' in data.lower(): log.critical('Received content looks a bit like login page') if 'error' in data.lower(): log.critical('Received content looks a bit like error page') received = os.path.join(task.manager.config_base, 'received') if not os.path.isdir(received): os.mkdir(received) filename = task.name sourcename = urlparse.urlparse(url).netloc if sourcename: filename += '-' + sourcename filename = pathscrub(filename, filename=True) filepath = os.path.join(received, '%s.%s' % (filename, ext)) with open(filepath, 'w') as f: f.write(data) log.critical('I have saved the invalid content to %s for you to view', filepath) def add_enclosure_info(self, entry, enclosure, filename=True, multiple=False): """Stores information from an rss enclosure into an Entry.""" entry['url'] = enclosure['href'] # get optional meta-data if 'length' in enclosure: try: entry['size'] = int(enclosure['length']) except: entry['size'] = 0 if 'type' in enclosure: entry['type'] = enclosure['type'] # TODO: better and perhaps join/in download plugin? # Parse filename from enclosure url basename = posixpath.basename(urlparse.urlsplit(entry['url']).path) # If enclosure has size OR there are multiple enclosures use filename from url if (entry.get('size') or multiple and basename) and filename: entry['filename'] = basename log.trace('filename `%s` from enclosure', entry['filename']) @cached('rss') @plugin.internet(log) def on_task_input(self, task, config): config = self.build_config(config) log.debug('Requesting task `%s` url `%s`', task.name, config['url']) # Used to identify which etag/modified to use url_hash = str(hash(config['url'])) # set etag and last modified headers if config has not changed since # last run and if caching wasn't disabled with --no-cache argument. all_entries = (config['all_entries'] or task.config_modified or task.options.nocache or task.options.retry) headers = {} if not all_entries: etag = task.simple_persistence.get('%s_etag' % url_hash, None) if etag: log.debug('Sending etag %s for task %s', etag, task.name) headers['If-None-Match'] = etag modified = task.simple_persistence.get('%s_modified' % url_hash, None) if modified: if not isinstance(modified, basestring): log.debug( 'Invalid date was stored for last modified time.') else: headers['If-Modified-Since'] = modified log.debug('Sending last-modified %s for task %s', headers['If-Modified-Since'], task.name) # Get the feed content if config['url'].startswith(('http', 'https', 'ftp', 'file')): # Get feed using requests library auth = None if 'username' in config and 'password' in config: auth = (config['username'], config['password']) try: # Use the raw response so feedparser can read the headers and status values response = task.requests.get(config['url'], timeout=60, headers=headers, raise_status=False, auth=auth) content = response.content except RequestException as e: raise plugin.PluginError( 'Unable to download the RSS for task %s (%s): %s' % (task.name, config['url'], e)) if config.get('ascii'): # convert content to ascii (cleanup), can also help with parsing problems on malformed feeds content = response.text.encode('ascii', 'ignore') # status checks status = response.status_code if status == 304: log.verbose( '%s hasn\'t changed since last run. Not creating entries.', config['url']) # Let details plugin know that it is ok if this feed doesn't produce any entries task.no_entries_ok = True return [] elif status == 401: raise plugin.PluginError( 'Authentication needed for task %s (%s): %s' % (task.name, config['url'], response.headers['www-authenticate']), log) elif status == 404: raise plugin.PluginError( 'RSS Feed %s (%s) not found' % (task.name, config['url']), log) elif status == 500: raise plugin.PluginError( 'Internal server exception on task %s (%s)' % (task.name, config['url']), log) elif status != 200: raise plugin.PluginError( 'HTTP error %s received from %s' % (status, config['url']), log) # update etag and last modified if not config['all_entries']: etag = response.headers.get('etag') if etag: task.simple_persistence['%s_etag' % url_hash] = etag log.debug('etag %s saved for task %s', etag, task.name) if response.headers.get('last-modified'): modified = response.headers['last-modified'] task.simple_persistence['%s_modified' % url_hash] = modified log.debug('last modified %s saved for task %s', modified, task.name) else: # This is a file, open it with open(config['url'], 'rb') as f: content = f.read() if config.get('ascii'): # Just assuming utf-8 file in this case content = content.decode('utf-8', 'ignore').encode('ascii', 'ignore') if not content: log.error('No data recieved for rss feed.') return try: rss = feedparser.parse(content) except LookupError as e: raise plugin.PluginError('Unable to parse the RSS (from %s): %s' % (config['url'], e)) # check for bozo ex = rss.get('bozo_exception', False) if ex or rss.get('bozo'): if rss.entries: msg = 'Bozo error %s while parsing feed, but entries were produced, ignoring the error.' % type( ex) if config.get('silent', False): log.debug(msg) else: log.verbose(msg) else: if isinstance(ex, feedparser.NonXMLContentType): # see: http://www.feedparser.org/docs/character-encoding.html#advanced.encoding.nonxml log.debug('ignoring feedparser.NonXMLContentType') elif isinstance(ex, feedparser.CharacterEncodingOverride): # see: ticket 88 log.debug('ignoring feedparser.CharacterEncodingOverride') elif isinstance(ex, UnicodeEncodeError): raise plugin.PluginError( 'Feed has UnicodeEncodeError while parsing...') elif isinstance(ex, (xml.sax._exceptions.SAXParseException, xml.sax._exceptions.SAXException)): # save invalid data for review, this is a bit ugly but users seem to really confused when # html pages (login pages) are received self.process_invalid_content(task, content, config['url']) if task.options.debug: log.error('bozo error parsing rss: %s' % ex) raise plugin.PluginError( 'Received invalid RSS content from task %s (%s)' % (task.name, config['url'])) elif isinstance(ex, httplib.BadStatusLine) or isinstance( ex, IOError): raise ex # let the @internet decorator handle else: # all other bozo errors self.process_invalid_content(task, content, config['url']) raise plugin.PluginError( 'Unhandled bozo_exception. Type: %s (task: %s)' % (ex.__class__.__name__, task.name), log) log.debug('encoding %s', rss.encoding) last_entry_id = '' if not all_entries: # Test to make sure entries are in descending order if rss.entries and rss.entries[0].get( 'published_parsed') and rss.entries[-1].get( 'published_parsed'): if rss.entries[0]['published_parsed'] < rss.entries[-1][ 'published_parsed']: # Sort them if they are not rss.entries.sort(key=lambda x: x['published_parsed'], reverse=True) last_entry_id = task.simple_persistence.get('%s_last_entry' % url_hash) # new entries to be created entries = [] # field name for url can be configured by setting link. # default value is auto but for example guid is used in some feeds ignored = 0 for entry in rss.entries: # Check if title field is overridden in config title_field = config.get('title', 'title') # ignore entries without title if not entry.get(title_field): log.debug('skipping entry without title') ignored += 1 continue # Set the title from the source field entry.title = entry[title_field] # Check we haven't already processed this entry in a previous run if last_entry_id == entry.title + entry.get('guid', ''): log.verbose('Not processing entries from last run.') # Let details plugin know that it is ok if this task doesn't produce any entries task.no_entries_ok = True break # remove annoying zero width spaces entry.title = entry.title.replace(u'\u200B', u'') # Dict with fields to grab mapping from rss field name to FlexGet field name fields = { 'guid': 'guid', 'author': 'author', 'description': 'description', 'infohash': 'torrent_info_hash' } # extend the dict of fields to grab with other_fields list in config for field_map in config.get('other_fields', []): fields.update(field_map) # helper # TODO: confusing? refactor into class member ... def add_entry(ea): ea['title'] = entry.title for rss_field, flexget_field in fields.iteritems(): if rss_field in entry: if not isinstance(getattr(entry, rss_field), basestring): # Error if this field is not a string log.error( 'Cannot grab non text field `%s` from rss.', rss_field) # Remove field from list of fields to avoid repeated error config['other_fields'].remove(rss_field) continue if not getattr(entry, rss_field): log.debug( 'Not grabbing blank field %s from rss for %s.', rss_field, ea['title']) continue try: ea[flexget_field] = decode_html(entry[rss_field]) if rss_field in config.get('other_fields', []): # Print a debug message for custom added fields log.debug('Field `%s` set to `%s` for `%s`', rss_field, ea[rss_field], ea['title']) except UnicodeDecodeError: log.warning( 'Failed to decode entry `%s` field `%s`', ea['title'], rss_field) # Also grab pubdate if available if hasattr(entry, 'published_parsed') and entry.published_parsed: ea['rss_pubdate'] = datetime(*entry.published_parsed[:6]) # store basic auth info if 'username' in config and 'password' in config: ea['download_auth'] = (config['username'], config['password']) entries.append(ea) # create from enclosures if present enclosures = entry.get('enclosures', []) if len(enclosures) > 1 and not config.get('group_links'): # There is more than 1 enclosure, create an Entry for each of them log.debug('adding %i entries from enclosures', len(enclosures)) for enclosure in enclosures: if 'href' not in enclosure: log.debug('RSS-entry `%s` enclosure does not have URL', entry.title) continue # There is a valid url for this enclosure, create an Entry for it ee = Entry() self.add_enclosure_info(ee, enclosure, config.get('filename', True), True) add_entry(ee) # If we created entries for enclosures, we should not create an Entry for the main rss item continue # create flexget entry e = Entry() if not isinstance(config.get('link'), list): # If the link field is not a list, search for first valid url if config['link'] == 'auto': # Auto mode, check for a single enclosure url first if len(entry.get( 'enclosures', [])) == 1 and entry['enclosures'][0].get('href'): self.add_enclosure_info(e, entry['enclosures'][0], config.get('filename', True)) else: # If there is no enclosure url, check link, then guid field for urls for field in ['link', 'guid']: if entry.get(field): e['url'] = entry[field] break else: if entry.get(config['link']): e['url'] = entry[config['link']] else: # If link was passed as a list, we create a list of urls for field in config['link']: if entry.get(field): e.setdefault('url', entry[field]) if entry[field] not in e.setdefault('urls', []): e['urls'].append(entry[field]) if config.get('group_links'): # Append a list of urls from enclosures to the urls field if group_links is enabled e.setdefault('urls', [e['url']]).extend([ enc.href for enc in entry.get('enclosures', []) if enc.get('href') not in e['urls'] ]) if not e.get('url'): log.debug('%s does not have link (%s) or enclosure', entry.title, config['link']) ignored += 1 continue add_entry(e) # Save last spot in rss if rss.entries: log.debug('Saving location in rss feed.') try: task.simple_persistence['%s_last_entry' % url_hash] = ( rss.entries[0].title + rss.entries[0].get('guid', '')) except AttributeError: log.debug( 'rss feed location saving skipped: no title information in first entry' ) if ignored: if not config.get('silent'): log.warning( 'Skipped %s RSS-entries without required information (title, link or enclosures)', ignored) return entries
class SearchRarBG(object): """ RarBG search plugin. To perform search against single category: rarbg: category: x264 720p To perform search against multiple categories: rarbg: category: - x264 720p - x264 1080p Movie categories accepted: x264 720p, x264 1080p, XviD, Full BD TV categories accepted: HDTV, SDTV You can use also use category ID manually if you so desire (eg. x264 720p is actually category id '45') """ schema = { 'type': 'object', 'properties': { 'category': one_or_more({ 'oneOf': [ { 'type': 'integer' }, { 'type': 'string', 'enum': list(CATEGORIES) }, ] }), 'sorted_by': { 'type': 'string', 'enum': ['seeders', 'leechers', 'last'], 'default': 'last' }, # min_seeders and min_leechers do not seem to work # 'min_seeders': {'type': 'integer', 'default': 0}, # 'min_leechers': {'type': 'integer', 'default': 0}, 'limit': { 'type': 'integer', 'enum': [25, 50, 100], 'default': 25 }, 'ranked': { 'type': 'boolean', 'default': True } }, "additionalProperties": False } base_url = 'https://torrentapi.org/pubapi.php' def get_token(self): # using rarbg.com to avoid the domain delay as tokens can be requested always r = requests.get('https://rarbg.com/pubapi/pubapi.php', params={ 'get_token': 'get_token', 'format': 'json' }) token = None try: token = r.json().get('token') except ValueError: log.error('Could not retrieve RARBG token.') log.debug('RarBG token: %s' % token) return token @plugin.internet(log) def search(self, task, entry, config): """ Search for entries on RarBG """ categories = config.get('category', 'all') # Ensure categories a list if not isinstance(categories, list): categories = [categories] # Convert named category to its respective category id number categories = [ c if isinstance(c, int) else CATEGORIES[c] for c in categories ] category_url_fragment = urllib.quote(';'.join( str(c) for c in categories)) entries = set() token = self.get_token() if not token: log.error('No token set. Exiting RARBG search.') return entries params = { 'mode': 'search', 'token': token, 'ranked': int(config['ranked']), # 'min_seeders': config['min_seeders'], 'min_leechers': config['min_leechers'], 'sort': config['sorted_by'], 'category': category_url_fragment, 'format': 'json' } for search_string in entry.get('search_strings', [entry['title']]): params.pop('search_string', None) params.pop('search_imdb', None) if entry.get('movie_name'): params['search_imdb'] = entry.get('imdb_id') else: query = normalize_unicode(search_string) query_url_fragment = query.encode('utf8') params['search_string'] = query_url_fragment page = requests.get(self.base_url, params=params) try: r = page.json() except ValueError: log.debug(page.text) break for result in r: entry = Entry() entry['title'] = result.get('f') entry['url'] = result.get('d') entries.add(entry) return entries
class RapidpushNotifier(object): """ Example:: rapidpush: apikey: xxxxxxx (can also be a list of api keys) [category: category, default FlexGet] [group: device group, default no group] [channel: the broadcast notification channel, if provided it will be send to the channel subscribers instead of your devices, default no channel] [priority: 0 - 6 (6 = highest), default 2 (normal)] """ schema = { 'type': 'object', 'properties': { 'api_key': one_or_more({'type': 'string'}), 'category': { 'type': 'string', 'default': 'Flexget' }, 'group': { 'type': 'string' }, 'channel': { 'type': 'string' }, 'priority': { 'type': 'integer', 'minimum': 0, 'maximum': 6 } }, 'additionalProperties': False, 'required': ['api_key'] } def notify(self, title, message, config): """ Send a Rapidpush notification """ wrapper = {} notification = {'title': title, 'message': message} if not isinstance(config['api_key'], list): config['api_key'] = [config['api_key']] if config.get('channel'): wrapper['command'] = 'broadcast' else: wrapper['command'] = 'notify' notification['category'] = config['category'] if config.get('group'): notification['group'] = config['group'] if config.get('priority') is not None: notification['priority'] = config['priority'] wrapper['data'] = notification for key in config['api_key']: wrapper['apikey'] = key try: response = requests.post(RAPIDPUSH_URL, json=wrapper) except RequestException as e: raise PluginWarning(e.args[0]) else: if response.json()['code'] > 400: raise PluginWarning(response.json()['desc'])
class PluginPathBySpace: """Allows setting a field to a folder based on it's space Path will be selected at random if multiple paths match the within Example: path_by_space: select: most_free_percent # or most_free, most_used, most_used_percent, has_free within: 9000 # within in MB or percent. paths: - /drive1/ - /drive2/ - /drive3/ """ schema = { 'type': 'object', 'properties': { 'select': { 'type': 'string', 'enum': list(selector_map.keys()) }, 'to_field': { 'type': 'string', 'default': 'path' }, 'paths': one_or_more({ 'type': 'string', 'format': 'path' }), 'within': { 'oneOf': [ { 'type': 'string', 'format': 'size' }, { 'type': 'string', 'format': 'percent' }, ] }, }, 'required': ['paths', 'select'], 'additionalProperties': False, } @plugin.priority(250) # run before other plugins def on_task_metainfo(self, task, config): selector = selector_map[config['select']] # Convert within to bytes (int) or percent (float) within = config.get('within') if isinstance(within, str) and '%' in within: within = parse_percent(within) else: within = parse_size(within) path = selector(config['paths'], within=within) if path: log.debug('Path %s selected due to (%s)' % (path, config['select'])) for entry in task.all_entries: entry[config['to_field']] = path else: task.abort('Unable to select a path based on %s' % config['select']) return
class PluginExec(object): """ Execute commands Simple example, xecute command for entries that reach output:: exec: echo 'found {{title}} at {{url}}' > file Advanced Example:: exec: on_start: phase: echo "Started" on_input: for_entries: echo 'got {{title}}' on_output: for_accepted: echo 'accepted {{title}} - {{url}} > file You can use all (available) entry fields in the command. """ NAME = 'exec' HANDLED_PHASES = ['start', 'input', 'filter', 'output', 'exit'] schema = { 'oneOf': [ one_or_more({'type': 'string'}), { 'type': 'object', 'properties': { 'on_start': { '$ref': '#/definitions/phaseSettings' }, 'on_input': { '$ref': '#/definitions/phaseSettings' }, 'on_filter': { '$ref': '#/definitions/phaseSettings' }, 'on_output': { '$ref': '#/definitions/phaseSettings' }, 'on_exit': { '$ref': '#/definitions/phaseSettings' }, 'fail_entries': { 'type': 'boolean' }, 'auto_escape': { 'type': 'boolean' }, 'encoding': { 'type': 'string' }, 'allow_background': { 'type': 'boolean' }, }, 'additionalProperties': False, }, ], 'definitions': { 'phaseSettings': { 'type': 'object', 'properties': { 'phase': one_or_more({'type': 'string'}), 'for_entries': one_or_more({'type': 'string'}), 'for_accepted': one_or_more({'type': 'string'}), 'for_rejected': one_or_more({'type': 'string'}), 'for_undecided': one_or_more({'type': 'string'}), 'for_failed': one_or_more({'type': 'string'}), }, 'additionalProperties': False, } }, } def prepare_config(self, config): if isinstance(config, basestring): config = [config] if isinstance(config, list): config = {'on_output': {'for_accepted': config}} if not config.get('encoding'): config['encoding'] = io_encoding for phase_name in config: if phase_name.startswith('on_'): for items_name in config[phase_name]: if isinstance(config[phase_name][items_name], basestring): config[phase_name][items_name] = [ config[phase_name][items_name] ] return config def execute_cmd(self, cmd, allow_background, encoding): log.verbose('Executing: %s', cmd) p = subprocess.Popen( text_to_native_str(cmd, encoding=io_encoding), shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=False, ) if not allow_background: r, w = (p.stdout, p.stdin) response = r.read().decode(io_encoding) r.close() w.close() if response: log.info('Stdout: %s', response.rstrip()) # rstrip to get rid of newlines return p.wait() def execute(self, task, phase_name, config): config = self.prepare_config(config) if phase_name not in config: log.debug('phase %s not configured' % phase_name) return name_map = { 'for_entries': task.entries, 'for_accepted': task.accepted, 'for_rejected': task.rejected, 'for_undecided': task.undecided, 'for_failed': task.failed, } allow_background = config.get('allow_background') for operation, entries in name_map.items(): if operation not in config[phase_name]: continue log.debug('running phase_name: %s operation: %s entries: %s' % (phase_name, operation, len(entries))) for entry in entries: for cmd in config[phase_name][operation]: entrydict = EscapingEntry(entry) if config.get( 'auto_escape') else entry # Do string replacement from entry, but make sure quotes get escaped try: cmd = render_from_entry(cmd, entrydict) except RenderError as e: log.error('Could not set exec command for %s: %s' % (entry['title'], e)) # fail the entry if configured to do so if config.get('fail_entries'): entry.fail( 'Entry `%s` does not have required fields for string replacement.' % entry['title']) continue log.debug('phase_name: %s operation: %s cmd: %s' % (phase_name, operation, cmd)) if task.options.test: log.info('Would execute: %s' % cmd) else: # Make sure the command can be encoded into appropriate encoding, don't actually encode yet, # so logging continues to work. try: cmd.encode(config['encoding']) except UnicodeEncodeError: log.error('Unable to encode cmd `%s` to %s' % (cmd, config['encoding'])) if config.get('fail_entries'): entry.fail( 'cmd `%s` could not be encoded to %s.' % (cmd, config['encoding'])) continue # Run the command, fail entries with non-zero return code if configured to if self.execute_cmd( cmd, allow_background, config['encoding'] ) != 0 and config.get('fail_entries'): entry.fail('exec return code was non-zero') # phase keyword in this if 'phase' in config[phase_name]: for cmd in config[phase_name]['phase']: try: cmd = render_from_task(cmd, task) except RenderError as e: log.error('Error rendering `%s`: %s' % (cmd, e)) else: log.debug('phase cmd: %s' % cmd) if task.options.test: log.info('Would execute: %s' % cmd) else: self.execute_cmd(cmd, allow_background, config['encoding']) def __getattr__(self, item): """Creates methods to handle task phases.""" for phase in self.HANDLED_PHASES: if item == plugin.phase_methods[phase]: # A phase method we handle has been requested break else: # We don't handle this phase raise AttributeError(item) def phase_handler(task, config): self.execute(task, 'on_' + phase, config) # Make sure we run after other plugins so exec can use their output phase_handler.priority = 100 return phase_handler
class OutputRapidPush(object): """ Example:: rapidpush: apikey: xxxxxxx (can also be a list of api keys) [category: category, default FlexGet] [title: title, default New release] [group: device group, default no group] [message: the message, default {{title}}] [channel: the broadcast notification channel, if provided it will be send to the channel subscribers instead of your devices, default no channel] [priority: 0 - 6 (6 = highest), default 2 (normal)] [notify_accepted: boolean true or false, default true] [notify_rejected: boolean true or false, default false] [notify_failed: boolean true or false, default false] [notify_undecided: boolean true or false, default false] Configuration parameters are also supported from entries (eg. through set). """ schema = { 'type': 'object', 'properties': { 'apikey': one_or_more({'type': 'string'}), 'category': {'type': 'string', 'default': 'Flexget'}, 'title': {'type': 'string', 'default': 'New Release'}, 'group': {'type': 'string', 'default': ''}, 'channel': {'type': 'string', 'default': ''}, 'priority': {'type': 'integer', 'default': 2}, 'message': {'type': 'string', 'default': '{{title}}'}, 'notify_accepted': {'type': 'boolean', 'default': True}, 'notify_rejected': {'type': 'boolean', 'default': False}, 'notify_failed': {'type': 'boolean', 'default': False}, 'notify_undecided': {'type': 'boolean', 'default': False} }, 'additionalProperties': False, 'required': ['apikey'] } # Run last to make sure other outputs are successful before sending notification @plugin.priority(0) def on_task_output(self, task, config): # get the parameters if config['notify_accepted']: log.debug("Notify accepted entries") self.process_notifications(task, task.accepted, config) if config['notify_rejected']: log.debug("Notify rejected entries") self.process_notifications(task, task.rejected, config) if config['notify_failed']: log.debug("Notify failed entries") self.process_notifications(task, task.failed, config) if config['notify_undecided']: log.debug("Notify undecided entries") self.process_notifications(task, task.undecided, config) # Process the given events. def process_notifications(self, task, entries, config): for entry in entries: if task.options.test: log.info("Would send RapidPush notification about: %s", entry['title']) continue log.info("Send RapidPush notification about: %s", entry['title']) apikey = entry.get('apikey', config['apikey']) if isinstance(apikey, list): apikey = ','.join(apikey) title = config['title'] try: title = entry.render(title) except RenderError as e: log.error('Error setting RapidPush title: %s' % e) message = config['message'] try: message = entry.render(message) except RenderError as e: log.error('Error setting RapidPush message: %s' % e) # Check if we have to send a normal or a broadcast notification. if not config['channel']: priority = entry.get('priority', config['priority']) category = entry.get('category', config['category']) try: category = entry.render(category) except RenderError as e: log.error('Error setting RapidPush category: %s' % e) group = entry.get('group', config['group']) try: group = entry.render(group) except RenderError as e: log.error('Error setting RapidPush group: %s' % e) # Send the request data_string = json.dumps({ 'title': title, 'message': message, 'priority': priority, 'category': category, 'group': group}) data = {'apikey': apikey, 'command': 'notify', 'data': data_string} else: channel = config['channel'] try: channel = entry.render(channel) except RenderError as e: log.error('Error setting RapidPush channel: %s' % e) # Send the broadcast request data_string = json.dumps({ 'title': title, 'message': message, 'channel': channel}) data = {'apikey': apikey, 'command': 'broadcast', 'data': data_string} response = task.requests.post(url, data=data, raise_status=False) json_data = response.json() if 'code' in json_data: if json_data['code'] == 200: log.debug("RapidPush message sent") else: log.error(json_data['desc'] + " (" + str(json_data['code']) + ")") else: for item in json_data: if json_data[item]['code'] == 200: log.debug(item + ": RapidPush message sent") else: log.error(item + ": " + json_data[item]['desc'] + " (" + str(json_data[item]['code']) + ")")
def __init__(self): """Set up the schema""" self.schema = { 'type': 'object', 'properties': { 'username': { 'type': 'string' }, 'password': { 'type': 'string' }, 'user_agent': { 'type': 'string' }, 'search': { 'type': 'string' }, 'artist': { 'type': 'string' }, 'album': { 'type': 'string' }, 'year': { 'type': ['string', 'integer'] }, 'tags': one_or_more({'type': 'string'}), 'tag_type': { 'type': 'string', 'enum': list(self._opts('tag_type').keys()) }, 'encoding': { 'type': 'string', 'enum': self._opts('encoding') }, 'format': { 'type': 'string', 'enum': self._opts('format') }, 'media': { 'type': 'string', 'enum': self._opts('media') }, 'release_type': { 'type': 'string', 'enum': list(self._opts('release_type').keys()) }, 'log': { 'oneOf': [{ 'type': 'string', 'enum': list(self._opts('log').keys()) }, { 'type': 'boolean' }] }, 'leech_type': { 'type': 'string', 'enum': list(self._opts('leech_type').keys()) }, 'hascue': { 'type': 'boolean' }, 'scene': { 'type': 'boolean' }, 'vanityhouse': { 'type': 'boolean' }, }, 'required': ['username', 'password'], 'additionalProperties': False }
class UrlRewriteIPTorrents: """ IpTorrents urlrewriter and search plugin. iptorrents: rss_key: xxxxxxxxx (required) uid: xxxxxxxx (required) password: xxxxxxxx (required) category: HD Category is any combination of: Movie-all, Movie-3D, Movie-480p, Movie-4K, Movie-BD-R, Movie-BD-Rip, Movie-Cam, Movie-DVD-R, Movie-HD-Bluray, Movie-Kids, Movie-MP4, Movie-Non-English, Movie-Packs, Movie-Web-DL, Movie-x265, Movie-XviD, TV-all, TV-Documentaries, TV-Sports, TV-480p, TV-BD, TV-DVD-R, TV-DVD-Rip, TV-MP4, TV-Mobile, TV-Non-English, TV-Packs, TV-Packs-Non-English, TV-SD-x264, TV-x264, TV-x265, TV-XVID, TV-Web-DL """ schema = { 'type': 'object', 'properties': { 'rss_key': { 'type': 'string' }, 'uid': { 'oneOf': [{ 'type': 'integer' }, { 'type': 'string' }] }, 'password': { 'type': 'string' }, 'category': one_or_more({ 'oneOf': [{ 'type': 'integer' }, { 'type': 'string', 'enum': list(CATEGORIES) }] }), 'free': { 'type': 'boolean', 'default': False }, }, 'required': ['rss_key', 'uid', 'password'], 'additionalProperties': False, } # urlrewriter API def url_rewritable(self, task, entry): url = entry['url'] if url.startswith(BASE_URL + '/download.php/'): return False if url.startswith(BASE_URL + '/'): return True return False # urlrewriter API def url_rewrite(self, task, entry): if 'url' not in entry: logger.error("Didn't actually get a URL...") else: logger.debug('Got the URL: {}', entry['url']) if entry['url'].startswith(SEARCH_URL): # use search results = self.search(task, entry) if not results: raise UrlRewritingError("No search results found") # TODO: Search doesn't enforce close match to title, be more picky entry['url'] = results[0]['url'] @plugin.internet(logger) def search(self, task, entry, config=None): """ Search for name from iptorrents """ categories = config.get('category', 'All') # Make sure categories is a list if not isinstance(categories, list): categories = [categories] # If there are any text categories, turn them into their id number categories = [ c if isinstance(c, int) else CATEGORIES[c] for c in categories ] category_params = {str(c): '' for c in categories if str(c)} entries = set() for search_string in entry.get('search_strings', [entry['title']]): search_params = { key: value for (key, value) in category_params.items() } query = normalize_unicode(search_string) search_params.update({'q': query, 'qf': ''}) logger.debug('searching with params: {}', search_params) if config.get('free'): req = requests.get(FREE_SEARCH_URL, params=search_params, cookies={ 'uid': str(config['uid']), 'pass': config['password'] }) else: req = requests.get(SEARCH_URL, params=search_params, cookies={ 'uid': str(config['uid']), 'pass': config['password'] }) logger.debug('full search URL: {}', req.url) if '/u/' + str(config['uid']) not in req.text: raise plugin.PluginError( "Invalid cookies (user not logged in)...") soup = get_soup(req.content, parser="html.parser") torrents = soup.find('table', {'id': 'torrents'}) results = torrents.findAll('tr') for torrent in results: if torrent.th and 'ac' in torrent.th.get('class'): # Header column continue if torrent.find('td', {'colspan': '99'}): logger.debug('No results found for search {}', search_string) break entry = Entry() link = torrent.find('a', href=re.compile('download'))['href'] entry[ 'url'] = f"{BASE_URL}{link}?torrent_pass={config.get('rss_key')}" entry['title'] = torrent.find('a', href=re.compile('details')).text seeders = torrent.findNext('td', { 'class': 'ac t_seeders' }).text leechers = torrent.findNext('td', { 'class': 'ac t_leechers' }).text entry['torrent_seeds'] = int(seeders) entry['torrent_leeches'] = int(leechers) entry['torrent_availability'] = torrent_availability( entry['torrent_seeds'], entry['torrent_leeches']) size = torrent.findNext( text=re.compile(r'^([\.\d]+) ([GMK]?)B$')) size = re.search(r'^([\.\d]+) ([GMK]?)B$', size) entry['content_size'] = parse_filesize(size.group(0)) logger.debug('Found entry {}', entry) entries.add(entry) return entries
class NexusPHP(object): """ 配置示例 task_name: rss: url: https://www.example.com/rss.xml other_fields: - link nexusphp: cookie: 'my_cookie' discount: - free - 2x seeders: min: 1 max: 30 leechers: min: 1 max: 100 max_complete: 0.8 hr: no """ schema = { 'type': 'object', 'properties': { 'cookie': { 'type': 'string' }, 'discount': one_or_more({ 'type': 'string', 'enum': ['free', '2x', '2xfree', '30%', '50%', '2x50%'] }), 'seeders': { 'type': 'object', 'properties': { 'min': { 'type': 'integer', 'minimum': 0, 'default': 0 }, 'max': { 'type': 'integer', 'minimum': 0, 'default': 100000 } } }, 'leechers': { 'type': 'object', 'properties': { 'min': { 'type': 'integer', 'minimum': 0, 'default': 0 }, 'max': { 'type': 'integer', 'minimum': 0, 'default': 100000 }, 'max_complete': { 'type': 'number', 'minimum': 0, 'maximum': 1, 'default': 1 } } }, 'hr': { 'type': 'boolean' }, 'adapter': { 'type': 'object', 'properties': { 'free': { 'type': 'string', 'default': 'free' }, '2x': { 'type': 'strging', 'default': 'twoup' }, '2xfree': { 'type': 'string', 'default': 'twoupfree' }, '30%': { 'type': 'string', 'default': 'thirtypercent' }, '50%': { 'type': 'string', 'default': 'halfdown' }, '2x50%': { 'type': 'string', 'default': 'twouphalfdown' } } }, 'comment': { 'type': 'boolean' }, 'user-agent': { 'type': 'string' } }, 'required': ['cookie'] } @staticmethod def build_config(config): config = dict(config) config.setdefault('discount', None) config.setdefault('seeders', {'min': 0, 'max': 100000}) config.setdefault('leechers', { 'min': 0, 'max': 100000, 'max_complete': 1 }) config.setdefault('hr', True) config.setdefault('adapter', None) config.setdefault( 'user-agent', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) ' 'Chrome/75.0.3770.142 Safari/537.36') return config @plugin.priority(127) def on_task_modify(self, task, config): if config.get('comment', False): for entry in task.entries: if 'torrent' in entry and 'link' in entry: entry['torrent'].content['comment'] = entry['link'] entry['torrent'].modified = True def on_task_filter(self, task, config): config = self.build_config(config) adapter = HTTPAdapter(max_retries=5) task.requests.mount('http://', adapter) task.requests.mount('https://', adapter) # 先访问一次 预防异常 headers = { 'cookie': config['cookie'], 'user-agent': config['user-agent'] } try: task.requests.get(task.entries[0].get('link'), headers=headers) except: pass def consider_entry(_entry, _link): try: discount, seeders, leechers, hr = NexusPHP._get_info( task, _link, config['cookie'], config['adapter'], config['user-agent']) except plugin.PluginError as e: raise e except Exception as e: log.info('NexusPHP._get_info: ' + str(e)) return seeder_max = config['seeders']['max'] seeder_min = config['seeders']['min'] leecher_max = config['leechers']['max'] leecher_min = config['leechers']['min'] if config['discount']: if discount not in config['discount']: _entry.reject('%s does not match discount' % discount) # 优惠信息不匹配 return if config['hr'] is False and hr: _entry.reject('it is HR') # 拒绝HR if len(seeders) not in range(seeder_min, seeder_max + 1): _entry.reject('%d is out of range of seeder' % len(seeders)) # 做种人数不匹配 return if len(leechers) not in range(leecher_min, leecher_max + 1): _entry.reject('%d is out of range of leecher' % len(leechers)) # 下载人数不匹配 return if len(leechers) != 0: max_complete = max(leechers, key=lambda x: x['completed'])['completed'] else: max_complete = 0 if max_complete > config['leechers']['max_complete']: _entry.reject('%f is more than max_complete' % max_complete) # 最大完成度不匹配 return _entry.accept() futures = [] # 线程任务 with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: for entry in task.entries: link = entry.get('link') if not link: raise plugin.PluginError( "The rss plugin require 'other_fields' which contain 'link'. " "For example: other_fields: - link") futures.append(executor.submit(consider_entry, entry, link)) for f in concurrent.futures.as_completed(futures): exception = f.exception() if isinstance(exception, plugin.PluginError): log.info(exception) @staticmethod # 解析页面,获取优惠、做种者信息、下载者信息 def info_from_page(detail_page, peer_page, discount_fn, hr_fn=None): try: discount = discount_fn(detail_page) except Exception: discount = None # 无优惠 try: if hr_fn: hr = hr_fn(detail_page) else: hr = False for item in [ 'hitandrun', 'hit_run.gif', 'Hit and Run', 'Hit & Run' ]: if item in detail_page.text: hr = True break except Exception: hr = False # 无HR soup = get_soup(peer_page) tables = soup.find_all('table', limit=2) try: seeders = NexusPHP.get_peers(tables[0]) except IndexError: seeders = [] try: leechers = NexusPHP.get_peers(tables[1]) except IndexError: leechers = [] return discount, seeders, leechers, hr @staticmethod def get_peers(table): peers = [] name_index = 0 connectable_index = 1 uploaded_index = 2 downloaded_index = 4 completed_index = 7 for index, tr in enumerate(table.find_all('tr')): try: if index == 0: tds = tr.find_all('td') for i, td in enumerate(tds): text = td.get_text() if text in ['用户', '用戶', '会员/IP']: name_index = i elif text in ['可连接', '可連接', '公网']: connectable_index = i elif text in ['上传', '上傳', '总上传']: uploaded_index = i elif text in ['下载', '下載', '本次下载']: downloaded_index = i elif text == '完成': completed_index = i else: tds = tr.find_all('td') peers.append({ 'name': tds[name_index].get_text(), 'connectable': True if tds[connectable_index].get_text() != '是' else False, 'uploaded': tds[uploaded_index].get_text(), 'downloaded': tds[downloaded_index].get_text(), 'completed': float(tds[completed_index].get_text().strip('%')) / 100 }) except Exception: peers.append({ 'name': '', 'connectable': False, 'uploaded': '', 'downloaded': '', 'completed': 0 }) return peers @staticmethod def _get_info(task, link, cookie, adapter, user_agent): headers = {'cookie': cookie, 'user-agent': user_agent} detail_page = task.requests.get(link, headers=headers) # 详情 detail_page.encoding = 'utf-8' if 'totheglory' in link: peer_url = link else: peer_url = link.replace('details.php', 'viewpeerlist.php', 1) try: peer_page = task.requests.get(peer_url, headers=headers).text # peer详情 except: peer_page = '' if 'login' in detail_page.url: raise plugin.PluginError( "Can't access the site. Your cookie may be wrong!") if adapter: convert = {value: key for key, value in adapter.items()} discount_fn = NexusPHP.generate_discount_fn(convert) return NexusPHP.info_from_page(detail_page, peer_page, discount_fn) sites_discount = { 'chdbits': { 'pro_free.*?</h1>': 'free', 'pro_2up.*?</h1>': '2x', 'pro_free2up.*?</h1>': '2xfree', 'pro_30pctdown.*?</h1>': '30%', 'pro_50pctdown.*?</h1>': '50%', 'pro_50pctdown2up.*?</h1>': '2x50%' }, 'u2.dmhy': { '<td.*?top.*?pro_free.*?优惠历史.*?</td>': 'free', '<td.*?top.*?pro_2up.*?优惠历史.*?</td>': '2x', '<td.*?top.*?pro_free2up.*?优惠历史.*?</td>': '2xfree', '<td.*?top.*?pro_30pctdown.*?优惠历史.*?</td>': '30%', '<td.*?top.*?pro_50pctdown.*?优惠历史.*?</td>': '50%', '<td.*?top.*?pro_50pctdown2up.*?优惠历史.*?</td>': '2x50%', '<td.*?top.*?pro_custom.*?优惠历史.*?</td>': '2x' }, 'yingk': { 'span_frees': 'free', 'span_twoupls': '2x', 'span_twoupfreels': '2xfree', 'span_thirtypercentls': '30%', 'span_halfdowns': '50%', 'span_twouphalfdownls': '2x50%' }, 'totheglory': { '本种子限时不计流量': 'free', '本种子的下载流量计为实际流量的30%': '30%', '本种子的下载流量会减半': '50%', }, 'hdchina': { 'pro_free.*?</h2>': 'free', 'pro_2up.*?</h2>': '2x', 'pro_free2up.*?</h2>': '2xfree', 'pro_30pctdown.*?</h2>': '30%', 'pro_50pctdown.*?</h2>': '50%', 'pro_50pctdown2up.*?</h2>': '2x50%' } } for site, convert in sites_discount.items(): if site in link: discount_fn = NexusPHP.generate_discount_fn(convert) return NexusPHP.info_from_page(detail_page, peer_page, discount_fn) discount_fn = NexusPHP.generate_discount_fn({ 'class=\'free\'.*?免.*?</h1>': 'free', 'class=\'twoup\'.*?2X.*?</h1>': '2x', 'class=\'twoupfree\'.*?2X免.*?</h1>': '2xfree', 'class=\'thirtypercent\'.*?30%.*?</h1>': '30%', 'class=\'halfdown\'.*?50%.*?</h1>': '50%', 'class=\'twouphalfdown\'.*?2X 50%.*?</h1>': '2x50%' }) return NexusPHP.info_from_page(detail_page, peer_page, discount_fn) @staticmethod def generate_discount_fn(convert): def fn(page): html = page.text.replace('\n', '') for key, value in convert.items(): match = re.search(key, html) if match: return value return None return fn
class RTorrentInputPlugin(RTorrentPluginBase): schema = { 'type': 'object', 'properties': { 'uri': { 'type': 'string' }, 'username': { 'type': 'string' }, 'password': { 'type': 'string' }, 'digest_auth': { 'type': 'boolean', 'default': False }, 'view': { 'type': 'string', 'default': 'main' }, 'fields': one_or_more({ 'type': 'string', 'enum': list(RTorrent.default_fields) }), }, 'required': ['uri'], 'additionalProperties': False, } def on_task_input(self, task, config): client = RTorrent( os.path.expanduser(config['uri']), username=config.get('username'), password=config.get('password'), digest_auth=config['digest_auth'], session=task.requests, ) fields = config.get('fields') try: torrents = client.torrents(config['view'], fields=fields) except (OSError, xmlrpc_client.Error) as e: task.abort('Could not get torrents (%s): %s' % (config['view'], e)) return entries = [] for torrent in torrents: entry = Entry( title=torrent['name'], url='%s/%s' % (os.path.expanduser(config['uri']), torrent['hash']), path=torrent['base_path'], torrent_info_hash=torrent['hash'], ) for attr, value in torrent.items(): entry[attr] = value if 'timestamp_finished' in entry: entry['timestamp_finished'] = datetime.fromtimestamp( entry['timestamp_finished']) entries.append(entry) return entries
class EmailNotifier(object): """ Send an e-mail with the list of all succeeded (downloaded) entries. Configuration options =============== =================================================================== Option Description =============== =================================================================== from The email address from which the email will be sent (required) to The email address of the recipient (required) smtp_host The host of the smtp server smtp_port The port of the smtp server smtp_username The username to use to connect to the smtp server smtp_password The password to use to connect to the smtp server smtp_tls Should we use TLS to connect to the smtp server smtp_ssl Should we use SSL to connect to the smtp server =============== =================================================================== Config basic example:: email: from: [email protected] to: [email protected] smtp_host: smtp.host.com Config example with smtp login:: email: from: [email protected] to: [email protected] smtp_host: smtp.host.com smtp_port: 25 smtp_login: true smtp_username: my_smtp_login smtp_password: my_smtp_password smtp_tls: true GMAIL example:: from: [email protected] to: [email protected] smtp_host: smtp.gmail.com smtp_port: 587 smtp_login: true smtp_username: gmailUser smtp_password: gmailPassword smtp_tls: true Default values for the config elements:: email: smtp_host: localhost smtp_port: 25 smtp_login: False smtp_username: smtp_password: smtp_tls: False smtp_ssl: False """ schema = { 'type': 'object', 'properties': { 'to': one_or_more({ 'type': 'string', 'format': 'email' }), 'from': { 'type': 'string', 'default': '*****@*****.**', 'format': 'email' }, 'smtp_host': { 'type': 'string', 'default': 'localhost' }, 'smtp_port': { 'type': 'integer', 'default': 25 }, 'smtp_username': { 'type': 'string' }, 'smtp_password': { 'type': 'string' }, 'smtp_tls': { 'type': 'boolean', 'default': False }, 'smtp_ssl': { 'type': 'boolean', 'default': False }, 'html': { 'type': 'boolean', 'default': False }, }, 'required': ['to'], 'dependencies': { 'smtp_username': ['smtp_password'], 'smtp_password': ['smtp_username'], 'smtp_ssl': ['smtp_tls'] }, 'additionalProperties': False, } def notify(self, title, message, config): """ Send an email notification :param str message: message body :param str title: message subject :param dict config: email plugin config """ if not isinstance(config['to'], list): config['to'] = [config['to']] email = MIMEMultipart('alternative') email['To'] = ','.join(config['to']) email['From'] = config['from'] email['Subject'] = title email['Date'] = formatdate(localtime=True) content_type = 'html' if config['html'] else 'plain' email.attach( MIMEText(message.encode('utf-8'), content_type, _charset='utf-8')) try: log.debug('sending email notification to %s:%s', config['smtp_host'], config['smtp_port']) mail_server = smtplib.SMTP_SSL if config[ 'smtp_ssl'] else smtplib.SMTP mail_server = mail_server(config['smtp_host'], config['smtp_port']) if config['smtp_tls']: mail_server.ehlo() mail_server.starttls() mail_server.ehlo() except (socket.error, OSError) as e: raise PluginWarning(str(e)) try: if config.get('smtp_username'): # Forcing to use `str` type log.debug('logging in to smtp server using username: %s', config['smtp_username']) mail_server.login(text_to_native_str(config['smtp_username']), text_to_native_str(config['smtp_password'])) mail_server.sendmail(email['From'], config['to'], email.as_string()) except IOError as e: raise PluginWarning(str(e)) mail_server.quit()
class PluginTransmission(TransmissionBase): """ Add url from entry url to transmission Example:: transmission: host: localhost port: 9091 netrc: /home/flexget/.tmnetrc username: myusername password: mypassword path: the download location Default values for the config elements:: transmission: host: localhost port: 9091 enabled: yes """ schema = { 'anyOf': [ {'type': 'boolean'}, { 'type': 'object', 'properties': { 'host': {'type': 'string'}, 'port': {'type': 'integer'}, 'netrc': {'type': 'string'}, 'username': {'type': 'string'}, 'password': {'type': 'string'}, 'action': { 'type': 'string', 'enum': ['add', 'remove', 'purge', 'pause', 'resume'], }, 'path': {'type': 'string'}, 'max_up_speed': {'type': 'number'}, 'max_down_speed': {'type': 'number'}, 'max_connections': {'type': 'integer'}, 'ratio': {'type': 'number'}, 'add_paused': {'type': 'boolean'}, 'content_filename': {'type': 'string'}, 'main_file_only': {'type': 'boolean'}, 'main_file_ratio': {'type': 'number'}, 'magnetization_timeout': {'type': 'integer'}, 'enabled': {'type': 'boolean'}, 'include_subs': {'type': 'boolean'}, 'bandwidth_priority': {'type': 'number'}, 'honor_limits': {'type': 'boolean'}, 'include_files': one_or_more({'type': 'string'}), 'skip_files': one_or_more({'type': 'string'}), 'rename_like_files': {'type': 'boolean'}, 'queue_position': {'type': 'integer'}, }, 'additionalProperties': False, }, ] } def prepare_config(self, config): config = TransmissionBase.prepare_config(self, config) config.setdefault('action', 'add') config.setdefault('path', '') config.setdefault('main_file_only', False) config.setdefault('magnetization_timeout', 0) config.setdefault('include_subs', False) config.setdefault('rename_like_files', False) config.setdefault('include_files', []) return config @plugin.priority(120) def on_task_download(self, task, config): """ Call download plugin to generate the temp files we will load into deluge then verify they are valid torrents """ config = self.prepare_config(config) if not config['enabled']: return # If the download plugin is not enabled, we need to call it to get our temp .torrent files if 'download' not in task.config: download = plugin.get('download', self) for entry in task.accepted: if entry.get('transmission_id'): # The torrent is already loaded in deluge, we don't need to get anything continue if config['action'] != 'add' and entry.get('torrent_info_hash'): # If we aren't adding the torrent new, all we need is info hash continue download.get_temp_file(task, entry, handle_magnets=True, fail_html=True) @plugin.priority(135) def on_task_output(self, task, config): config = self.prepare_config(config) # don't add when learning if task.options.learn: return if not config['enabled']: return # Do not run if there is nothing to do if not task.accepted: return if self.client is None: self.client = self.create_rpc_client(config) if self.client: log.debug('Successfully connected to transmission.') else: raise plugin.PluginError("Couldn't connect to transmission.") session_torrents = self.client.get_torrents() for entry in task.accepted: if task.options.test: log.info('Would %s %s in transmission.', config['action'], entry['title']) continue # Compile user options into appropriate dict options = self._make_torrent_options_dict(config, entry) torrent_info = None for t in session_torrents: if t.hashString.lower() == entry.get( 'torrent_info_hash', '' ).lower() or t.id == entry.get('transmission_id'): torrent_info = t log.debug( 'Found %s already loaded in transmission as %s', entry['title'], torrent_info.name, ) break if not torrent_info: if config['action'] != 'add': log.warning( 'Cannot %s %s because it is not loaded in transmission.', config['action'], entry['title'], ) continue downloaded = not entry['url'].startswith('magnet:') # Check that file is downloaded if downloaded and 'file' not in entry: entry.fail('`file` field missing?') continue # Verify the temp file exists if downloaded and not os.path.exists(entry['file']): tmp_path = os.path.join(task.manager.config_base, 'temp') log.debug('entry: %s', entry) log.debug('temp: %s', ', '.join(os.listdir(tmp_path))) entry.fail("Downloaded temp file '%s' doesn't exist!?" % entry['file']) continue try: if downloaded: with open(entry['file'], 'rb') as f: filedump = base64.b64encode(f.read()).decode('utf-8') torrent_info = self.client.add_torrent(filedump, 30, **options['add']) else: # we need to set paused to false so the magnetization begins immediately options['add']['paused'] = False torrent_info = self.client.add_torrent( entry['url'], timeout=30, **options['add'] ) except TransmissionError as e: log.debug('TransmissionError', exc_info=True) log.debug('Failed options dict: %s', options['add']) msg = 'Error adding {} to transmission. TransmissionError: {}'.format( entry['title'], e.message or 'N/A' ) log.error(msg) entry.fail(msg) continue log.info('"%s" torrent added to transmission', entry['title']) # The info returned by the add call is incomplete, refresh it torrent_info = self.client.get_torrent(torrent_info.id) try: total_size = torrent_info.totalSize main_id = None find_main_file = ( options['post'].get('main_file_only') or 'content_filename' in options['post'] ) skip_files = options['post'].get('skip_files') # We need to index the files if any of the following are defined if find_main_file or skip_files: file_list = self.client.get_files(torrent_info.id)[torrent_info.id] if options['post'].get('magnetization_timeout', 0) > 0 and not file_list: log.debug( 'Waiting %d seconds for "%s" to magnetize', options['post']['magnetization_timeout'], entry['title'], ) for _ in range(options['post']['magnetization_timeout']): sleep(1) file_list = self.client.get_files(torrent_info.id)[torrent_info.id] if file_list: total_size = self.client.get_torrent( torrent_info.id, ['id', 'totalSize'] ).totalSize break else: log.warning( '"%s" did not magnetize before the timeout elapsed, ' 'file list unavailable for processing.', entry['title'], ) # Find files based on config dl_list = [] skip_list = [] main_list = [] ext_list = ['*.srt', '*.sub', '*.idx', '*.ssa', '*.ass'] main_ratio = config['main_file_ratio'] if 'main_file_ratio' in options['post']: main_ratio = options['post']['main_file_ratio'] for f in file_list: # No need to set main_id if we're not going to need it if find_main_file and file_list[f]['size'] > total_size * main_ratio: main_id = f if 'include_files' in options['post']: if any( fnmatch(file_list[f]['name'], mask) for mask in options['post']['include_files'] ): dl_list.append(f) elif options['post'].get('include_subs') and any( fnmatch(file_list[f]['name'], mask) for mask in ext_list ): dl_list.append(f) if skip_files: if any(fnmatch(file_list[f]['name'], mask) for mask in skip_files): skip_list.append(f) if main_id is not None: # Look for files matching main ID title but with a different extension if options['post'].get('rename_like_files'): for f in file_list: # if this filename matches main filename we want to rename it as well fs = os.path.splitext(file_list[f]['name']) if fs[0] == os.path.splitext(file_list[main_id]['name'])[0]: main_list.append(f) else: main_list = [main_id] if main_id not in dl_list: dl_list.append(main_id) elif find_main_file: log.warning( 'No files in "%s" are > %d%% of content size, no files renamed.', entry['title'], main_ratio * 100, ) # If we have a main file and want to rename it and associated files if 'content_filename' in options['post'] and main_id is not None: if 'download_dir' not in options['add']: download_dir = self.client.get_session().download_dir else: download_dir = options['add']['download_dir'] # Get new filename without ext file_ext = os.path.splitext(file_list[main_id]['name'])[1] file_path = os.path.dirname( os.path.join(download_dir, file_list[main_id]['name']) ) filename = options['post']['content_filename'] if config['host'] == 'localhost' or config['host'] == '127.0.0.1': counter = 1 while os.path.exists(os.path.join(file_path, filename + file_ext)): # Try appending a (#) suffix till a unique filename is found filename = '%s(%s)' % ( options['post']['content_filename'], counter, ) counter += 1 else: log.debug( 'Cannot ensure content_filename is unique ' 'when adding to a remote transmission daemon.' ) for index in main_list: file_ext = os.path.splitext(file_list[index]['name'])[1] log.debug( 'File %s renamed to %s' % (file_list[index]['name'], filename + file_ext) ) # change to below when set_files will allow setting name, more efficient to have one call # fl[index]['name'] = os.path.basename(pathscrub(filename + file_ext).encode('utf-8')) try: self.client.rename_torrent_path( torrent_info.id, file_list[index]['name'], os.path.basename(str(pathscrub(filename + file_ext))), ) except TransmissionError: log.error('content_filename only supported with transmission 2.8+') if options['post'].get('main_file_only') and main_id is not None: # Set Unwanted Files options['change']['files_unwanted'] = [ x for x in file_list if x not in dl_list ] options['change']['files_wanted'] = dl_list log.debug( 'Downloading %s of %s files in torrent.', len(options['change']['files_wanted']), len(file_list), ) elif ( not options['post'].get('main_file_only') or main_id is None ) and skip_files: # If no main file and we want to skip files if len(skip_list) >= len(file_list): log.debug( 'skip_files filter would cause no files to be downloaded; ' 'including all files in torrent.' ) else: options['change']['files_unwanted'] = skip_list options['change']['files_wanted'] = [ x for x in file_list if x not in skip_list ] log.debug( 'Downloading %s of %s files in torrent.', len(options['change']['files_wanted']), len(file_list), ) # Set any changed file properties if list(options['change'].keys()): self.client.change_torrent(torrent_info.id, 30, **options['change']) if config['action'] == 'add': # if add_paused was defined and set to False start the torrent; # prevents downloading data before we set what files we want start_paused = ( options['post']['paused'] if 'paused' in options['post'] else not self.client.get_session().start_added_torrents ) if start_paused: self.client.stop_torrent(torrent_info.id) else: self.client.start_torrent(torrent_info.id) elif config['action'] in ('remove', 'purge'): self.client.remove_torrent( [torrent_info.id], delete_data=config['action'] == 'purge' ) log.info('%sd %s from transmission', config['action'], torrent_info.name) elif config['action'] == 'pause': self.client.stop_torrent([torrent_info.id]) log.info('paused %s in transmission', torrent_info.name) elif config['action'] == 'resume': self.client.start_torrent([torrent_info.id]) log.info('resumed %s in transmission', torrent_info.name) except TransmissionError as e: log.debug('TransmissionError', exc_info=True) log.debug('Failed options dict: %s', options) msg = 'Error trying to {} {}, TransmissionError: {}'.format( config['action'], entry['title'], e.message or 'N/A' ) log.error(msg) continue def _make_torrent_options_dict(self, config, entry): opt_dic = {} for opt_key in ( 'path', 'add_paused', 'honor_limits', 'bandwidth_priority', 'max_connections', 'max_up_speed', 'max_down_speed', 'ratio', 'main_file_only', 'main_file_ratio', 'magnetization_timeout', 'include_subs', 'content_filename', 'include_files', 'skip_files', 'rename_like_files', 'queue_position', ): # Values do not merge config with task # Task takes priority then config is used if opt_key in entry: opt_dic[opt_key] = entry[opt_key] elif opt_key in config: opt_dic[opt_key] = config[opt_key] options = {'add': {}, 'change': {}, 'post': {}} add = options['add'] if opt_dic.get('path'): try: path = os.path.expanduser(entry.render(opt_dic['path'])) except RenderError as e: log.error('Error setting path for %s: %s' % (entry['title'], e)) else: # Transmission doesn't like it when paths end in a separator path = path.rstrip('\\/') add['download_dir'] = text_to_native_str(pathscrub(path), 'utf-8') # make sure we add it paused, will modify status after adding add['paused'] = True change = options['change'] if 'bandwidth_priority' in opt_dic: change['bandwidthPriority'] = opt_dic['bandwidth_priority'] if 'honor_limits' in opt_dic and not opt_dic['honor_limits']: change['honorsSessionLimits'] = False if 'max_up_speed' in opt_dic: change['uploadLimit'] = opt_dic['max_up_speed'] change['uploadLimited'] = True if 'max_down_speed' in opt_dic: change['downloadLimit'] = opt_dic['max_down_speed'] change['downloadLimited'] = True if 'max_connections' in opt_dic: change['peer_limit'] = opt_dic['max_connections'] if 'ratio' in opt_dic: change['seedRatioLimit'] = opt_dic['ratio'] if opt_dic['ratio'] == -1: # seedRatioMode: # 0 follow the global settings # 1 override the global settings, seeding until a certain ratio # 2 override the global settings, seeding regardless of ratio change['seedRatioMode'] = 2 else: change['seedRatioMode'] = 1 if 'queue_position' in opt_dic: change['queuePosition'] = opt_dic['queue_position'] post = options['post'] # set to modify paused status after if 'add_paused' in opt_dic: post['paused'] = opt_dic['add_paused'] if 'main_file_only' in opt_dic: post['main_file_only'] = opt_dic['main_file_only'] if 'main_file_ratio' in opt_dic: post['main_file_ratio'] = opt_dic['main_file_ratio'] if 'magnetization_timeout' in opt_dic: post['magnetization_timeout'] = opt_dic['magnetization_timeout'] if 'include_subs' in opt_dic: post['include_subs'] = opt_dic['include_subs'] if 'content_filename' in opt_dic: try: post['content_filename'] = entry.render(opt_dic['content_filename']) except RenderError as e: log.error('Unable to render content_filename %s: %s' % (entry['title'], e)) if 'skip_files' in opt_dic: post['skip_files'] = opt_dic['skip_files'] if not isinstance(post['skip_files'], list): post['skip_files'] = [post['skip_files']] if 'include_files' in opt_dic: post['include_files'] = opt_dic['include_files'] if not isinstance(post['include_files'], list): post['include_files'] = [post['include_files']] if 'rename_like_files' in opt_dic: post['rename_like_files'] = opt_dic['rename_like_files'] return options def on_task_learn(self, task, config): """ Make sure all temp files are cleaned up when entries are learned """ # If download plugin is enabled, it will handle cleanup. if 'download' not in task.config: download = plugin.get('download', self) download.cleanup_temp_files(task) on_task_abort = on_task_learn
class Unique(object): """ Take action on entries with duplicate fields, except for the first item Reject the second+ instance of every movie: unique: field: - imdb_id - movie_name action: reject """ schema = { 'type': 'object', 'properties': { 'field': one_or_more({'type': 'string'}), 'action': { 'enum': ['accept', 'reject'] }, }, 'required': ['field'], 'additionalProperties': False, } def prepare_config(self, config): if isinstance(config, bool) or config is None: config = {} config.setdefault('action', 'reject') if not isinstance(config['field'], list): config['field'] = [config['field']] return config def extract_fields(self, entry, field_names): return [entry[field] for field in field_names] def should_ignore(self, item, action): return item.accepted and action == 'accept' or item.rejected and action == 'reject' def on_task_filter(self, task, config): config = self.prepare_config(config) field_names = config['field'] entries = list(task.entries) for i, entry in enumerate(entries): # Ignore already processed entries if self.should_ignore(entry, config['action']): continue try: entry_fields = self.extract_fields(entry, field_names) # Ignore if a field is missing except KeyError: continue # Iterate over next items, try to find a similar item for prospect in entries[i + 1:]: # Ignore processed prospects if self.should_ignore(prospect, config['action']): continue try: prospect_fields = self.extract_fields( prospect, field_names) # Ignore if a field is missing except KeyError: continue if entry_fields and entry_fields == prospect_fields: msg = 'Field {} value {} equals on {} and {}'.format( field_names, entry_fields, entry['title'], prospect['title']) # Mark prospect if config['action'] == 'accept': prospect.accept(msg) else: prospect.reject(msg)
class SftpList: """ Generate entries from SFTP. This plugin requires the pysftp Python module and its dependencies. Configuration: host: Host to connect to. port: Port the remote SSH server is listening on (default 22). username: Username to log in as. password: The password to use. Optional if a private key is provided. private_key: Path to the private key (if any) to log into the SSH server. private_key_pass: Password for the private key (if needed). recursive: Indicates whether the listing should be recursive. get_size: Indicates whetern to calculate the size of the remote file/directory. WARNING: This can be very slow when computing the size of directories! files_only: Indicates wheter to omit diredtories from the results. dirs_only: Indicates whether to omit files from the results. dirs: List of directories to download. socket_timeout_sec: Socket timeout in seconds (default 15 seconds). connection_tries: Number of times to attempt to connect before failing (default 3). Example: sftp_list: host: example.com username: Username private_key: /Users/username/.ssh/id_rsa recursive: False get_size: True files_only: False dirs: - '/path/to/list/' - '/another/path/' """ schema = { 'type': 'object', 'properties': { 'host': { 'type': 'string' }, 'username': { 'type': 'string' }, 'password': { 'type': 'string' }, 'port': { 'type': 'integer', 'default': DEFAULT_SFTP_PORT }, 'files_only': { 'type': 'boolean', 'default': True }, 'dirs_only': { 'type': 'boolean', 'default': False }, 'recursive': { 'type': 'boolean', 'default': False }, 'get_size': { 'type': 'boolean', 'default': True }, 'private_key': { 'type': 'string' }, 'private_key_pass': { 'type': 'string' }, 'dirs': one_or_more({'type': 'string'}), 'socket_timeout_sec': { 'type': 'integer', 'default': DEFAULT_SOCKET_TIMEOUT_SEC }, 'connection_tries': { 'type': 'integer', 'default': DEFAULT_CONNECT_TRIES }, }, 'additionProperties': False, 'required': ['host', 'username'], } @staticmethod def prepare_config(config: dict) -> dict: """ Sets defaults for the provided configuration """ config.setdefault('password', None) config.setdefault('private_key', None) config.setdefault('private_key_pass', None) config.setdefault('dirs', ['.']) return config @classmethod def on_task_input(cls, task: Task, config: dict) -> List[Entry]: """ Input task handler """ config = cls.prepare_config(config) files_only: bool = config['files_only'] dirs_only: bool = config['dirs_only'] recursive: bool = config['recursive'] get_size: bool = config['get_size'] socket_timeout_sec: int = config['socket_timeout_sec'] connection_tries: int = config['connection_tries'] directories: List[str] = [] if files_only and dirs_only: logger.warning( "Both files_only and dirs_only are set. This will result in no entries being discovered." ) if isinstance(config['dirs'], list): directories.extend(config['dirs']) else: directories.append(config['dirs']) sftp_config: SftpConfig = task_config_to_sftp_config(config) sftp: SftpClient = sftp_connect(sftp_config, socket_timeout_sec, connection_tries) entries: List[Entry] = sftp.list_directories(directories, recursive, get_size, files_only, dirs_only) sftp.close() return entries
class SearchRarBG(object): """ RarBG search plugin. To perform search against single category: rarbg: category: x264 720p To perform search against multiple categories: rarbg: category: - x264 720p - x264 1080p Movie categories accepted: x264 720p, x264 1080p, XviD, Full BD TV categories accepted: HDTV, SDTV You can use also use category ID manually if you so desire (eg. x264 720p is actually category id '45') """ schema = { 'type': 'object', 'properties': { 'category': one_or_more({ 'oneOf': [ {'type': 'integer'}, {'type': 'string', 'enum': list(CATEGORIES)}, ]}), 'sorted_by': {'type': 'string', 'enum': ['seeders', 'leechers', 'last'], 'default': 'last'}, # min_seeders and min_leechers seem to be working again 'min_seeders': {'type': 'integer', 'default': 0}, 'min_leechers': {'type': 'integer', 'default': 0}, 'limit': {'type': 'integer', 'enum': [25, 50, 100], 'default': 25}, 'ranked': {'type': 'boolean', 'default': True}, 'use_tvdb': {'type': 'boolean', 'default': False}, }, "additionalProperties": False } base_url = 'https://torrentapi.org/pubapi_v2.php' def get_token(self): # Don't use a session as tokens are not affected by domain limit try: r = get(self.base_url, params={'get_token': 'get_token', 'format': 'json'}).json() token = r.get('token') log.debug('RarBG token: %s' % token) return token except RequestException as e: log.debug('Could not retrieve RarBG token: %s', e.args[0]) @plugin.internet(log) def search(self, task, entry, config): """ Search for entries on RarBG """ categories = config.get('category', 'all') # Ensure categories a list if not isinstance(categories, list): categories = [categories] # Convert named category to its respective category id number categories = [c if isinstance(c, int) else CATEGORIES[c] for c in categories] category_url_fragment = ';'.join(str(c) for c in categories) entries = set() token = self.get_token() if not token: log.error('Could not retrieve token. Abandoning search.') return entries params = {'mode': 'search', 'token': token, 'ranked': int(config['ranked']), 'min_seeders': config['min_seeders'], 'min_leechers': config['min_leechers'], 'sort': config['sorted_by'], 'category': category_url_fragment, 'format': 'json_extended', 'app_id': 'flexget'} for search_string in entry.get('search_strings', [entry['title']]): params.pop('search_string', None) params.pop('search_imdb', None) params.pop('search_tvdb', None) if entry.get('movie_name'): params['search_imdb'] = entry.get('imdb_id') else: query = normalize_scene(search_string) query_url_fragment = query.encode('utf8') params['search_string'] = query_url_fragment if config['use_tvdb']: plugin.get_plugin_by_name('thetvdb_lookup').instance.lazy_series_lookup(entry) params['search_tvdb'] = entry.get('tvdb_id') log.debug('Using tvdb id %s', entry.get('tvdb_id')) try: page = requests.get(self.base_url, params=params) log.debug('requesting: %s', page.url) except RequestException as e: log.error('RarBG request failed: %s' % e.args[0]) continue r = page.json() # error code 20 just means no results were found if r.get('error_code') == 20: searched_string = params.get('search_string') or 'imdb={0}'.format(params.get('search_imdb')) or \ 'tvdb={0}'.format(params.get('tvdb_id')) log.debug('No results found for %s', searched_string) continue elif r.get('error'): log.error('Error code %s: %s', r.get('error_code'), r.get('error')) continue else: for result in r.get('torrent_results'): e = Entry() e['title'] = result.get('title') e['url'] = result.get('download') e['torrent_seeds'] = int(result.get('seeders')) e['torrent_leeches'] = int(result.get('leechers')) e['content_size'] = int(result.get('size')) / 1024 / 1024 episode_info = result.get('episode_info') if episode_info: e['imdb_id'] = episode_info.get('imdb') e['tvdb_id'] = episode_info.get('tvdb') e['tvrage_id'] = episode_info.get('tvrage') entries.add(e) return entries
class OutputPushover(object): """ Example:: pushover: userkey: <USER_KEY> (can also be a list of userkeys) apikey: <API_KEY> [device: <DEVICE_STRING>] (default: (none)) [title: <MESSAGE_TITLE>] (default: "Download started" -- accepts Jinja2) [message: <MESSAGE_BODY>] (default: "{{series_name}} {{series_id}}" -- accepts Jinja2) [priority: <PRIORITY>] (default = 0 -- normal = 0, high = 1, silent = -1) [url: <URL>] (default: "{{imdb_url}}" -- accepts Jinja2) [urltitle: <URL_TITLE>] (default: (none) -- accepts Jinja2) [sound: <SOUND>] (default: pushover default) Configuration parameters are also supported from entries (eg. through set). """ default_message = "{% if series_name is defined %}{{tvdb_series_name|d(series_name)}} " \ "{{series_id}} {{tvdb_ep_name|d('')}}{% elif imdb_name is defined %}{{imdb_name}} "\ "{{imdb_year}}{% else %}{{title}}{% endif %}" schema = { 'type': 'object', 'properties': { 'userkey': one_or_more({'type': 'string'}), 'apikey': { 'type': 'string' }, 'device': { 'type': 'string', 'default': '' }, 'title': { 'type': 'string', 'default': "{{task}}" }, 'message': { 'type': 'string', 'default': default_message }, 'priority': { 'type': 'integer', 'default': 0 }, 'url': { 'type': 'string', 'default': '{% if imdb_url is defined %}{{imdb_url}}{% endif %}' }, 'urltitle': { 'type': 'string', 'default': '' }, 'sound': { 'type': 'string', 'default': '' } }, 'required': ['userkey', 'apikey'], 'additionalProperties': False } # Run last to make sure other outputs are successful before sending notification @plugin.priority(0) def on_task_output(self, task, config): # Support for multiple userkeys userkeys = config["userkey"] if not isinstance(userkeys, list): userkeys = [userkeys] # Set a bunch of local variables from the config apikey = config["apikey"] device = config["device"] priority = config["priority"] sound = config["sound"] # Loop through the provided entries for entry in task.accepted: title = config["title"] message = config["message"] url = config["url"] urltitle = config["urltitle"] # Attempt to render the title field try: title = entry.render(title) except RenderError as e: log.warning("Problem rendering 'title': %s" % e) title = "Download started" # Attempt to render the message field try: message = entry.render(message) except RenderError as e: log.warning("Problem rendering 'message': %s" % e) message = entry["title"] # Attempt to render the url field try: url = entry.render(url) except RenderError as e: log.warning("Problem rendering 'url': %s" % e) url = entry.get("imdb_url", "") # Attempt to render the urltitle field try: urltitle = entry.render(urltitle) except RenderError as e: log.warning("Problem rendering 'urltitle': %s" % e) urltitle = "" for userkey in userkeys: # Build the request data = { "user": userkey, "token": apikey, "title": title, "message": message, "url": url, "url_title": urltitle } if device: data["device"] = device if priority: data["priority"] = priority if sound: data["sound"] = sound # Check for test mode if task.options.test: log.info("Test mode. Pushover notification would be:") if device: log.info(" Device: %s" % device) else: log.info(" Device: [broadcast]") log.info(" Title: %s" % title) log.info(" Message: %s" % message) log.info(" URL: %s" % url) log.info(" URL Title: %s" % urltitle) log.info(" Priority: %d" % priority) log.info(" userkey: %s" % userkey) log.info(" apikey: %s" % apikey) log.info(" sound: %s" % sound) # Test mode. Skip remainder. continue # Make the request try: response = task.requests.post(pushover_url, data=data, raise_status=False) except RequestException as e: log.warning( 'Could not get response from Pushover: {}'.format(e)) return # Check if it succeeded request_status = response.status_code # error codes and messages from Pushover API if request_status == 200: log.debug("Pushover notification sent") elif request_status == 500: log.debug( "Pushover notification failed, Pushover API having issues" ) # TODO: Implement retrying. API requests 5 seconds between retries. elif request_status >= 400: errors = json.loads(response.content)['errors'] log.error("Pushover API error: %s" % errors[0]) else: log.error( "Unknown error when sending Pushover notification")
class SearchMoreThanTV(object): """ MorethanTV search plugin. """ schema = { 'type': 'object', 'properties': { 'username': { 'type': 'string' }, 'password': { 'type': 'string' }, 'category': one_or_more({ 'type': 'string', 'enum': list(CATEGORIES.keys()) }, unique_items=True), 'order_by': { 'type': 'string', 'enum': ['seeders', 'leechers', 'time'], 'default': 'time' }, 'order_way': { 'type': 'string', 'enum': ['desc', 'asc'], 'default': 'desc' }, 'tags': one_or_more({ 'type': 'string', 'enum': TAGS }, unique_items=True), 'all_tags': { 'type': 'boolean', 'default': True } }, 'required': ['username', 'password'], 'additionalProperties': False } base_url = 'https://www.morethan.tv/' errors = False def get(self, url, params, username, password, force=False): """ Wrapper to allow refreshing the cookie if it is invalid for some reason :param str url: :param list params: :param str username: :param str password: :param bool force: flag used to refresh the cookie forcefully ie. forgo DB lookup :return: """ cookies = self.get_login_cookie(username, password, force=force) response = requests.get(url, params=params, cookies=cookies) if self.base_url + 'login.php' in response.url: if self.errors: raise plugin.PluginError( 'MoreThanTV login cookie is invalid. Login page received?') self.errors = True # try again response = self.get(url, params, username, password, force=True) else: self.errors = False return response def get_login_cookie(self, username, password, force=False): """ Retrieves login cookie :param str username: :param str password: :param bool force: if True, then retrieve a fresh cookie instead of looking in the DB :return: """ if not force: with Session() as session: saved_cookie = session.query(MoreThanTVCookie).filter( MoreThanTVCookie.username == username).first() if saved_cookie and saved_cookie.expires and saved_cookie.expires >= datetime.datetime.now( ): log.debug('Found valid login cookie') return saved_cookie.cookie url = self.base_url + 'login.php' try: log.debug('Attempting to retrieve MoreThanTV cookie') response = requests.post(url, data={ 'username': username, 'password': password, 'login': '******', 'keeplogged': '1' }, timeout=30) except RequestException as e: raise plugin.PluginError('MoreThanTV login failed: %s' % e) if 'Your username or password was incorrect.' in response.text: raise plugin.PluginError( 'MoreThanTV login failed: Your username or password was incorrect.' ) with Session() as session: expires = None for c in requests.cookies: if c.name == 'session': expires = c.expires if expires: expires = datetime.datetime.fromtimestamp(expires) log.debug('Saving or updating MoreThanTV cookie in db') cookie = MoreThanTVCookie(username=username, cookie=dict(requests.cookies), expires=expires) session.merge(cookie) return cookie.cookie @plugin.internet(log) def search(self, task, entry, config): """ Search for entries on MoreThanTV """ params = {} if 'category' in config: categories = config['category'] if isinstance( config['category'], list) else [config['category']] for category in categories: params[CATEGORIES[category]] = 1 if 'tags' in config: tags = config['tags'] if isinstance(config['tags'], list) else [config['tags']] tags = ', '.join(tags) params['taglist'] = tags entries = set() params.update({ 'tags_type': int(config['all_tags']), 'order_by': config['order_by'], 'search_submit': 1, 'order_way': config['order_way'], 'action': 'basic', 'group_results': 0 }) for search_string in entry.get('search_strings', [entry['title']]): params['searchstr'] = search_string.replace("'", "") log.debug('Using search params: %s', params) try: page = self.get(self.base_url + 'torrents.php', params, config['username'], config['password']) log.debug('requesting: %s', page.url) except RequestException as e: log.error('MoreThanTV request failed: %s', e) continue soup = get_soup(page.content) for result in soup.findAll('tr', attrs={'class': 'torrent'}): group_info = result.find('td', attrs={ 'class': 'big_info' }).find('div', attrs={'class': 'group_info'}) title = group_info.find( 'a', href=re.compile('torrents.php\?id=\d+')).text url = self.base_url + group_info.find( 'a', href=re.compile('torrents.php\?action=download'))['href'] torrent_info = result.findAll('td', attrs={'class': 'number_column'}) size = re.search('(\d+(?:[.,]\d+)*)\s?([KMG]B)', torrent_info[0].text) torrent_tags = ', '.join([ tag.text for tag in group_info.findAll('div', attrs={'class': 'tags'}) ]) e = Entry() e['title'] = title e['url'] = url e['torrent_snatches'] = int(torrent_info[1].text) e['torrent_seeds'] = int(torrent_info[2].text) e['torrent_leeches'] = int(torrent_info[3].text) e['torrent_internal'] = True if group_info.find( 'span', attrs={'class': 'flag_internal'}) else False e['torrent_fast_server'] = True if group_info.find( 'span', attrs={'class': 'flag_fast'}) else False e['torrent_sticky'] = True if group_info.find( 'span', attrs={'class': 'flag_sticky'}) else False e['torrent_tags'] = torrent_tags e['content_size'] = parse_filesize(size.group(0)) entries.add(e) return entries
class RegexExtract(object): """ Updates an entry with the values of regex matched named groups Usage: regex_extract: field: <string> regex: - <regex> [prefix]: <string> Example: regex_extract: prefix: f1_ field: title regex: - Formula\.?1.(?P<location>*?) """ schema = { 'type': 'object', 'properties': { 'prefix': { 'type': 'string' }, 'field': { 'type': 'string' }, 'regex': one_or_more({ 'type': 'string', 'format': 'regex' }), }, } def on_task_start(self, task, config): regex = config.get('regex') if isinstance(regex, str): regex = [regex] self.regex_list = ReList(regex) # Check the regex try: for _ in self.regex_list: pass except re.error as e: raise plugin.PluginError('Error compiling regex: %s' % str(e)) def on_task_modify(self, task, config): prefix = config.get('prefix') modified = 0 for entry in task.entries: for rx in self.regex_list: entry_field = entry.get('title') log.debug('Matching %s with regex: %s' % (entry_field, rx)) try: match = rx.match(entry_field) except re.error as e: raise plugin.PluginError( 'Error encountered processing regex: %s' % str(e)) if match: log.debug('Successfully matched %s' % entry_field) data = match.groupdict() if prefix: for key in list(data.keys()): data[prefix + key] = data[key] del data[key] log.debug('Values added to entry: %s' % data) entry.update(data) modified += 1 log.info('%d entries matched and modified' % modified)
"type": "string", "pattern": "^([#&][^\x07\x2C\s]{0,200})", "error_pattern": "channel name must start with # or & and contain no commas and whitespace", } schema = { "oneOf": [ { "type": "object", "additionalProperties": { "type": "object", "properties": { "tracker_file": {"type": "string"}, "server": {"type": "string"}, "port": {"type": "integer"}, "nickname": {"type": "string"}, "channels": one_or_more(channel_pattern), "nickserv_password": {"type": "string"}, "invite_nickname": {"type": "string"}, "invite_message": {"type": "string"}, "task": one_or_more({"type": "string"}), "task_re": { "type": "object", "additionalProperties": one_or_more( { "type": "object", "properties": {"regexp": {"type": "string"}, "field": {"type": "string"}}, "required": ["regexp", "field"], "additionalProperties": False, } ), },
class UrlRewriteTorrent411(object): """ torrent411 Urlrewriter and search Plugin. --- RSS (Two Options) -- RSS DOWNLOAD WITH LOGIN rss: url: http://www.t411.in/rss/?cat=210 username: **** password: **** - OR - -- RSS NORMAL URL REWRITE (i.e.: http://www.t411.in/torrents/download/?id=12345678) -- WARNING: NEED CUSTOM COOKIES NOT HANDLE BY THIS PLUGIN rss: url: http://www.t411.in/rss/?cat=210 --- SEARCH WITHIN SITE discover: what: - emit_movie_queue: yes from: - torrent411: username: xxxxxxxx (required) password: xxxxxxxx (required) category: Film sub_category: Multi-Francais --- Category is one of these: Animation, Animation-Serie, Concert, Documentaire, Emission-TV, Film, Serie-TV, Series, Spectacle, Sport, Video-clips --- Sub-Category is any combination of: Anglais, VFF, Muet, Multi-Francais, Multi-Quebecois, VFQ, VFSTFR, VOSTFR, VOASTA BDrip-BRrip-SD, Bluray-4K, Bluray-Full-Remux, DVD-R-5, DVD-R-9, DVDrip, HDrip-1080p, HDrip-720p, HDlight-1080p, HDlight-720p, TVrip-SD, TVripHD-1080p, TVripHD-720p, VCD-SVCD-VHSrip, WEBrip, WEBripHD-1080p, WEBripHD-1080p 2D, 3D-Converti-Amateur, 3D-Converti-Pro, 3D-Natif """ schema = { 'type': 'object', 'properties': { 'username': { 'type': 'string' }, 'password': { 'type': 'string' }, 'category': { 'type': 'string' }, 'sub_category': one_or_more({ 'type': 'string', 'enum': list(SUB_CATEGORIES) }), }, 'deprecated': '"torrent411" plugin has been replaced by the "t411" plugin.', 'required': ['username', 'password'], 'additionalProperties': False } # urlrewriter API def url_rewritable(self, task, entry): url = entry['url'] if re.match( r'^(https?://)?(www\.)?t411\.in/torrents/(?!download/)[-A-Za-z0-9+&@#/%|?=~_|!:,.;]+', url): return True return False # urlrewriter API def url_rewrite(self, task, entry): if 'url' not in entry: log.error("Didn't actually get a URL...") else: url = entry['url'] log.debug("Got the URL: %s" % entry['url']) rawdata = "" try: opener = urllib2.build_opener() opener.addheaders = [('User-agent', 'Mozilla/5.0')] response = opener.open(url) except Exception as e: raise UrlRewritingError("Connection Error for %s : %s" % (url, e)) rawdata = response.read() match = re.search( r"<a href=\"/torrents/download/\?id=(\d*?)\">.*\.torrent</a>", rawdata) if match: torrent_id = match.group(1) log.debug("Got the Torrent ID: %s" % torrent_id) entry[ 'url'] = 'http://www.t411.in/torrents/download/?id=' + torrent_id if 'download_auth' in entry: auth_handler = t411Auth(*entry['download_auth']) entry['download_auth'] = auth_handler else: raise UrlRewritingError("Cannot find torrent ID") @plugin.internet(log) def search(self, task, entry, config=None): """ Search for name from torrent411. """ url_base = 'http://www.t411.in' if not isinstance(config, dict): config = {} category = config.get('category') if category in list(CATEGORIES): category = CATEGORIES[category] sub_categories = config.get('sub_category') if not isinstance(sub_categories, list): sub_categories = [sub_categories] filter_url = '' if isinstance(category, int): filter_url = '&cat=%s' % str(category) if sub_categories[0] is not None: sub_categories = [SUB_CATEGORIES[c] for c in sub_categories] filter_url = filter_url + '&' + '&'.join([ urllib.quote_plus('term[%s][]' % c[0]).encode('utf-8') + '=' + str(c[1]) for c in sub_categories ]) entries = set() for search_string in entry.get('search_strings', [entry['title']]): query = normalize_unicode(search_string) url_search = ('/torrents/search/?search=%40name+' + urllib.quote_plus(query.encode('utf-8')) + filter_url) opener = urllib2.build_opener() opener.addheaders = [('User-agent', 'Mozilla/5.0')] response = opener.open(url_base + url_search) data = response.read() soup = get_soup(data) tb = soup.find("table", class_="results") if not tb: continue for tr in tb.findAll('tr')[1:][:-1]: entry = Entry() nfo_link_res = re.search('torrents/nfo/\?id=(\d+)', str(tr)) if nfo_link_res is not None: tid = nfo_link_res.group(1) title_res = re.search( '<a href=\"//www.t411.in/torrents/([-A-Za-z0-9+&@#/%|?=~_|!:,.;]+)\" title="([^"]*)">', str(tr)) if title_res is not None: entry['title'] = title_res.group(2).decode('utf-8') size = tr('td')[5].contents[0] entry[ 'url'] = 'http://www.t411.in/torrents/download/?id=%s' % tid entry['torrent_seeds'] = tr('td')[7].contents[0] entry['torrent_leeches'] = tr('td')[8].contents[0] entry['search_sort'] = torrent_availability( entry['torrent_seeds'], entry['torrent_leeches']) size = re.search('([\.\d]+) ([GMK]?)B', size) if size: if size.group(2) == 'G': entry['content_size'] = int( float(size.group(1)) * 1000**3 / 1024**2) elif size.group(2) == 'M': entry['content_size'] = int( float(size.group(1)) * 1000**2 / 1024**2) elif size.group(2) == 'K': entry['content_size'] = int( float(size.group(1)) * 1000 / 1024**2) else: entry['content_size'] = int( float(size.group(1)) / 1024**2) auth_handler = t411Auth(config['username'], config['password']) entry['download_auth'] = auth_handler entries.add(entry) return sorted(entries, reverse=True, key=lambda x: x.get('search_sort'))
class BaseFileOps(object): # Defined by subclasses log = None along = { 'type': 'object', 'properties': { 'files': one_or_more({'type': 'string'}), 'subdirs': one_or_more({'type': 'string'}) }, 'additionalProperties': False, 'required': ['files'] } def on_task_output(self, task, config): if config is True: config = {} elif config is False: return for entry in task.accepted: if 'location' not in entry: self.log.verbose( 'Cannot handle %s because it does not have the field location.' % entry['title']) continue src = entry['location'] src_isdir = os.path.isdir(src) try: # check location if not os.path.exists(src): raise plugin.PluginWarning( 'location `%s` does not exists (anymore).' % src) if src_isdir: if not config.get('allow_dir'): raise plugin.PluginWarning( 'location `%s` is a directory.' % src) elif not os.path.isfile(src): raise plugin.PluginWarning('location `%s` is not a file.' % src) # search for namesakes siblings = {} # dict of (path=ext) pairs if not src_isdir and 'along' in config: parent = os.path.dirname(src) filename_no_ext = os.path.splitext( os.path.basename(src))[0] subdirs = [parent] + config['along'].get('subdirs', []) for subdir in subdirs: if subdir == parent: abs_subdirs = [subdir] else: # use glob to get a list of matching dirs abs_subdirs = glob.glob( os.path.join(parent, os.path.normpath(subdir))) # iterate over every dir returned by glob looking for matching ext for abs_subdir in abs_subdirs: if os.path.isdir(abs_subdir): for ext in config['along']['files']: siblings.update( get_siblings(ext, src, filename_no_ext, abs_subdir)) # execute action in subclasses self.handle_entry(task, config, entry, siblings) except OSError as err: entry.fail(str(err)) continue def clean_source(self, task, config, entry): min_size = entry.get('clean_source', config.get('clean_source', -1)) if min_size < 0: return base_path = os.path.split(entry.get('old_location', entry['location']))[0] # everything here happens after a successful execution of the main action: the entry has been moved in a # different location, or it does not exists anymore. so from here we can just log warnings and move on. if not os.path.isdir(base_path): self.log.warning( 'Cannot delete path `%s` because it does not exists (anymore).' % base_path) return dir_size = get_directory_size(base_path) / 1024 / 1024 if dir_size >= min_size: self.log.info( 'Path `%s` left because it exceeds safety value set in clean_source option.' % base_path) return if task.options.test: self.log.info('Would delete `%s` and everything under it.' % base_path) return try: shutil.rmtree(base_path) self.log.info( 'Path `%s` has been deleted because was less than clean_source safe value.' % base_path) except Exception as err: self.log.warning('Unable to delete path `%s`: %s' % (base_path, err)) def handle_entry(self, task, config, entry, siblings): raise NotImplementedError()
class EmbyRefreshLibrary: """ Refresh Emby Library Example: emby_refresh: server: host: http://localhost:8096 username: <username> apikey: <apikey> return_host: wan when: accepted """ auth = None schema = { 'type': 'object', 'properties': { **SCHEMA_SERVER_TAG, 'when': one_or_more({ 'type': 'string', 'enum': [ 'accepted', 'rejected', 'failed', 'no_entries', 'aborted', 'always' ], }), }, 'required': ['server'], 'additionalProperties': False, } def login(self, config): if self.auth and self.auth.logged: return if not isinstance(config, dict): config = {} self.auth = EmbyAuth(**config) self.auth.login(True) def prepare_config(self, config): config.setdefault('when', ['always']) when = config['when'] if when and not isinstance(when, list): config['when'] = [when] return def library_refresh(self): EmbyApiLibrary.library_refresh(self.auth) def on_task_start(self, task, config): self.login(config) @plugin.internet(logger) def on_task_exit(self, task, config): self.login(config) self.prepare_config(config) conditions = [ task.accepted and 'accepted' in config['when'], task.rejected and 'rejected' in config['when'], task.failed and 'failed' in config['when'], not task.all_entries and 'no_entries' in config['when'], 'always' in config['when'], ] if any(conditions): self.library_refresh() def on_task_abort(self, task, config): self.prepare_config(config) if 'aborted' in config['when']: self.library_refresh()
class PushoverNotifier(object): """ Example:: pushover: user_key: <USER_KEY> (can also be a list of userkeys) token: <TOKEN> [device: <DEVICE_STRING>] [title: <MESSAGE_TITLE>] [message: <MESSAGE_BODY>] [priority: <PRIORITY>] [url: <URL>] [url_title: <URL_TITLE>] [sound: <SOUND>] [retry]: <RETRY>] [expire]: <EXPIRE>] [callback]: <CALLBACK>] [html]: <HTML>] """ schema = { 'type': 'object', 'properties': { 'user_key': one_or_more({'type': 'string'}), 'api_key': {'type': 'string', 'default': 'aPwSHwkLcNaavShxktBpgJH4bRWc3m'}, 'device': one_or_more({'type': 'string'}), 'priority': {'oneOf': [ {'type': 'number', 'minimum': -2, 'maximum': 2}, {'type': 'string'}]}, 'url': {'type': 'string'}, 'url_title': {'type': 'string'}, 'sound': {'type': 'string'}, 'retry': {'type': 'integer', 'minimum': 30}, 'expire': {'type': 'integer', 'maximum': 86400}, 'callback': {'type': 'string'}, 'html': {'type': 'boolean'} }, 'required': ['user_key'], 'additionalProperties': False } def notify(self, title, message, config): """ Sends a Pushover notification :param str title: the message's title :param str message: the message to send :param dict config: The pushover config """ notification = {'token': config.get('api_key'), 'message': message, 'title': title, 'device': config.get('device'), 'priority': config.get('priority'), 'url': config.get('url'), 'url_title': config.get('url_title'), 'sound': config.get('sound'), 'retry': config.get('retry'), 'expire': config.get('expire'), 'callback': config.get('callback')} # HTML parsing mode if config.get('html'): notification['html'] = 1 # Support multiple devices if isinstance(notification['device'], list): notification['device'] = ','.join(notification['device']) # Special case, verify certain fields exists if priority is 2 priority = config.get('priority') expire = config.get('expire') retry = config.get('retry') if priority == 2 and not all([expire, retry]): log.warning('Priority set to 2 but fields "expire" and "retry" are not both present.Lowering priority to 1') notification['priority'] = 1 if not isinstance(config['user_key'], list): config['user_key'] = [config['user_key']] for user in config['user_key']: notification['user'] = user try: response = requests.post(PUSHOVER_URL, data=notification) except RequestException as e: if e.response is not None: if e.response.status_code == 429: reset_time = datetime.datetime.fromtimestamp( int(e.response.headers['X-Limit-App-Reset'])).strftime('%Y-%m-%d %H:%M:%S') error_message = 'Monthly pushover message limit reached. Next reset: %s' % reset_time else: error_message = e.response.json()['errors'][0] else: error_message = str(e) raise PluginWarning(error_message) reset_time = datetime.datetime.fromtimestamp( int(response.headers['X-Limit-App-Reset'])).strftime('%Y-%m-%d %H:%M:%S') remaining = response.headers['X-Limit-App-Remaining'] log.debug('Pushover notification sent. Notifications remaining until next reset: %s. ' 'Next reset at: %s', remaining, reset_time)
class Filesystem: """ Uses local path content as an input. Can use recursion if configured. Recursion is False by default. Can be configured to true or get integer that will specify max depth in relation to base folder. All files/dir/symlinks are retrieved by default. Can be changed by using the 'retrieve' property. Example 1:: Single path filesystem: /storage/movies/ Example 2:: List of paths filesystem: - /storage/movies/ - /storage/tv/ Example 3:: Object with list of paths filesystem: path: - /storage/movies/ - /storage/tv/ mask: '*.mkv' Example 4:: filesystem: path: - /storage/movies/ - /storage/tv/ recursive: 4 # 4 levels deep from each base folder retrieve: files # Only files will be retrieved Example 5:: filesystem: path: - /storage/movies/ - /storage/tv/ recursive: yes # No limit to depth, all sub dirs will be accessed retrieve: # Only files and dirs will be retrieved - files - dirs """ retrieval_options = ['files', 'dirs', 'symlinks'] paths = one_or_more({ 'type': 'string', 'format': 'path' }, unique_items=True) schema = { 'oneOf': [ paths, { 'type': 'object', 'properties': { 'path': paths, 'mask': { 'type': 'string' }, 'regexp': { 'type': 'string', 'format': 'regex' }, 'recursive': { 'oneOf': [{ 'type': 'integer', 'minimum': 2 }, { 'type': 'boolean' }] }, 'retrieve': one_or_more({ 'type': 'string', 'enum': retrieval_options }, unique_items=True), }, 'required': ['path'], 'additionalProperties': False, }, ] } def prepare_config(self, config): from fnmatch import translate config = config # Converts config to a dict with a list of paths if not isinstance(config, dict): config = {'path': config} if not isinstance(config['path'], list): config['path'] = [config['path']] config.setdefault('recursive', False) # If mask was specified, turn it in to a regexp if config.get('mask'): config['regexp'] = translate(config['mask']) # If no mask or regexp specified, accept all files config.setdefault('regexp', '.') # Sets the default retrieval option to files config.setdefault('retrieve', self.retrieval_options) return config def create_entry(self, filepath: Path, test_mode): """ Creates a single entry using a filepath and a type (file/dir) """ filepath = filepath.absolute() entry = Entry() entry['location'] = str(filepath) entry['url'] = Path(filepath).absolute().as_uri() entry['filename'] = filepath.name if filepath.is_file(): entry['title'] = filepath.stem else: entry['title'] = filepath.name file_stat = filepath.stat() try: entry['timestamp'] = datetime.fromtimestamp(file_stat.st_mtime) except Exception as e: logger.warning('Error setting timestamp for {}: {}', filepath, e) entry['timestamp'] = None entry['accessed'] = datetime.fromtimestamp(file_stat.st_atime) entry['modified'] = datetime.fromtimestamp(file_stat.st_mtime) entry['created'] = datetime.fromtimestamp(file_stat.st_ctime) if entry.isvalid(): if test_mode: logger.info("Test mode. Entry includes:") logger.info(' Title: {}', entry['title']) logger.info(' URL: {}', entry['url']) logger.info(' Filename: {}', entry['filename']) logger.info(' Location: {}', entry['location']) logger.info(' Timestamp: {}', entry['timestamp']) return entry else: logger.error('Non valid entry created: {} ', entry) return def get_max_depth(self, recursion, base_depth): if recursion is False: return base_depth + 1 elif recursion is True: return float('inf') else: return base_depth + recursion @staticmethod def get_folder_objects(folder: Path, recursion: bool): return folder.rglob('*') if recursion else folder.iterdir() def get_entries_from_path(self, path_list, match, recursion, test_mode, get_files, get_dirs, get_symlinks): entries = [] for folder in path_list: logger.verbose('Scanning folder {}. Recursion is set to {}.', folder, recursion) folder = Path(folder).expanduser() if not folder.exists(): logger.error('{} does not exist (anymore.)', folder) continue logger.debug('Scanning {}', folder) base_depth = len(folder.parts) max_depth = self.get_max_depth(recursion, base_depth) folder_objects = self.get_folder_objects(folder, recursion) for path_object in folder_objects: logger.debug( 'Checking if {} qualifies to be added as an entry.', path_object) try: path_object.exists() except UnicodeError: logger.error( 'File {} not decodable with filesystem encoding: {}', path_object, sys.getfilesystemencoding(), ) continue entry = None object_depth = len(path_object.parts) if object_depth <= max_depth: if match(str(path_object)): if ((path_object.is_dir() and get_dirs) or (path_object.is_symlink() and get_symlinks) or (path_object.is_file() and not path_object.is_symlink() and get_files)): entry = self.create_entry(path_object, test_mode) else: logger.debug( "Path object's {} type doesn't match requested object types.", path_object, ) if entry and entry not in entries: entries.append(entry) return entries def on_task_input(self, task, config): config = self.prepare_config(config) path_list = config['path'] test_mode = task.options.test match = re.compile(config['regexp'], re.IGNORECASE).match recursive = config['recursive'] get_files = 'files' in config['retrieve'] get_dirs = 'dirs' in config['retrieve'] get_symlinks = 'symlinks' in config['retrieve'] logger.verbose('Starting to scan folders.') return self.get_entries_from_path(path_list, match, recursive, test_mode, get_files, get_dirs, get_symlinks)