def __init__(self):
"""Set up the schema"""
self.schema = {
'type': 'object',
'properties': {
'username': {'type': 'string'},
'password': {'type': 'string'},
'user_agent': {'type': 'string'},
'search': {'type': 'string'},
'artist': {'type': 'string'},
'album': {'type': 'string'},
'year': {'type': ['string', 'integer']},
'tags': one_or_more({'type': 'string'}),
'tag_type': {'type': 'string', 'enum': self._opts('tag_type').keys()},
'encoding': {'type': 'string', 'enum': self._opts('encoding')},
'format': {'type': 'string', 'enum': self._opts('format')},
'media': {'type': 'string', 'enum': self._opts('media')},
'release_type': {'type': 'string', 'enum': self._opts('release_type').keys()},
'log': {'oneOf': [{'type': 'string', 'enum': self._opts('log').keys()}, {'type': 'boolean'}]},
'leech_type': {'type': 'string', 'enum': self._opts('leech_type').keys()},
'hascue': {'type': 'boolean'},
'scene': {'type': 'boolean'},
'vanityhouse': {'type': 'boolean'},
},
'required': ['username', 'password'],
'additionalProperties': False
}
def __init__(self):
self.schema = {
"type": "object",
"properties": {
"category": {"type": "string"},
"terms": one_or_more({"type": "string"}),
"max_results": {"type": "number", "default": 100},
},
"additionalProperties": False,
}
def __init__(self):
self.schema = {
'type': 'object',
'properties': {
'category': {'type': 'string'},
'terms': one_or_more({'type': 'string'}),
'max_results': {'type': 'number', 'default': 100}
},
'additionalProperties': False
}
def schema(self):
"""The schema of the plugin
Extends the super's schema
"""
schema = super(InputGazelleMusic, self).schema
schema['properties'].update({
'artist': {'type': 'string'},
'album': {'type': 'string'},
'year': {'type': ['string', 'integer']},
'tags': one_or_more({'type': 'string'}),
'tag_type': {'type': 'string', 'enum': list(self._opts('tag_type').keys())},
'encoding': {'type': 'string', 'enum': self._opts('encoding')},
'format': {'type': 'string', 'enum': self._opts('format')},
'media': {'type': 'string', 'enum': self._opts('media')},
'release_type': {'type': 'string', 'enum': list(self._opts('release_type').keys())},
'log': {'oneOf': [{'type': 'string', 'enum': list(self._opts('log').keys())}, {'type': 'boolean'}]},
'leech_type': {'type': 'string', 'enum': list(self._opts('leech_type').keys())},
'hascue': {'type': 'boolean'},
'scene': {'type': 'boolean'},
'vanityhouse': {'type': 'boolean'}
})
return schema
class SearchAlphaRatio(object):
"""
AlphaRatio search plugin.
"""
schema = {
'type': 'object',
'properties': {
'username': {'type': 'string'},
'password': {'type': 'string'},
'category': one_or_more({'type': 'string', 'enum': list(CATEGORIES.keys())}, unique_items=True),
'order_by': {'type': 'string', 'enum': ['seeders', 'leechers', 'time', 'size', 'year', 'snatched'],
'default': 'time'},
'order_desc': {'type': 'boolean', 'default': True},
'scene': {'type': 'boolean'},
'leechstatus': {'type': 'string', 'enum': list(LEECHSTATUS.keys()), 'default': 'normal'},
},
'required': ['username', 'password'],
'additionalProperties': False
}
base_url = 'https://alpharatio.cc/'
errors = False
def get(self, url, params, username, password, force=False):
"""
Wrapper to allow refreshing the cookie if it is invalid for some reason
:param unicode url:
:param dict params:
:param str username:
:param str password:
:param bool force: flag used to refresh the cookie forcefully ie. forgo DB lookup
:return:
"""
cookies = self.get_login_cookie(username, password, force=force)
response = requests.get(url, params=params, cookies=cookies)
if self.base_url + 'login.php' in response.url:
if self.errors:
raise plugin.PluginError('AlphaRatio login cookie is invalid. Login page received?')
self.errors = True
# try again
response = self.get(url, params, username, password, force=True)
else:
self.errors = False
return response
def get_login_cookie(self, username, password, force=False):
"""
Retrieves login cookie
:param str username:
:param str password:
:param bool force: if True, then retrieve a fresh cookie instead of looking in the DB
:return:
"""
if not force:
with Session() as session:
saved_cookie = session.query(AlphaRatioCookie).filter(AlphaRatioCookie.username == username).first()
if saved_cookie and saved_cookie.expires and saved_cookie.expires >= datetime.datetime.now():
log.debug('Found valid login cookie')
return saved_cookie.cookie
url = self.base_url + 'login.php'
try:
log.debug('Attempting to retrieve AlphaRatio cookie')
response = requests.post(url, data={'username': username, 'password': password, 'login': '******',
'keeplogged': '1'}, timeout=30)
except RequestException as e:
raise plugin.PluginError('AlphaRatio login failed: %s', e)
if 'Your username or password was incorrect.' in response.text:
raise plugin.PluginError('AlphaRatio login failed: Your username or password was incorrect.')
with Session() as session:
expires = None
for c in requests.cookies:
if c.name == 'session':
expires = c.expires
if expires:
expires = datetime.datetime.fromtimestamp(expires)
log.debug('Saving or updating AlphaRatio cookie in db')
cookie = AlphaRatioCookie(username=username, cookie=dict(requests.cookies), expires=expires)
session.merge(cookie)
return cookie.cookie
@plugin.internet(log)
def search(self, task, entry, config):
"""
Search for entries on AlphaRatio
"""
params = {}
if 'category' in config:
categories = config['category'] if isinstance(config['category'], list) else [config['category']]
for category in categories:
params[CATEGORIES[category]] = 1
if 'scene' in config:
params['scene'] = int(config['scene'])
ordering = 'desc' if config['order_desc'] else 'asc'
entries = set()
params.update({'order_by': config['order_by'], 'search_submit': 1, 'action': 'basic', 'order_way': ordering,
'freeleech': LEECHSTATUS[config['leechstatus']]})
for search_string in entry.get('search_strings', [entry['title']]):
params['searchstr'] = search_string
log.debug('Using search params: %s', params)
try:
page = self.get(self.base_url + 'torrents.php', params, config['username'], config['password'])
log.debug('requesting: %s', page.url)
except RequestException as e:
log.error('AlphaRatio request failed: %s', e)
continue
soup = get_soup(page.content)
for result in soup.findAll('tr', attrs={'class': 'torrent'}):
group_info = result.find('td', attrs={'class': 'big_info'}).find('div', attrs={'class': 'group_info'})
title = group_info.find('a', href=re.compile('torrents.php\?id=\d+')).text
url = self.base_url + \
group_info.find('a', href=re.compile('torrents.php\?action=download(?!usetoken)'))['href']
torrent_info = result.findAll('td')
log.debug('AlphaRatio size: %s', torrent_info[5].text)
size = re.search('(\d+(?:[.,]\d+)*)\s?([KMGTP]B)', torrent_info[4].text)
torrent_tags = ', '.join([tag.text for tag in group_info.findAll('div', attrs={'class': 'tags'})])
e = Entry()
e['title'] = title
e['url'] = url
e['torrent_tags'] = torrent_tags
e['content_size'] = parse_filesize(size.group(0))
e['torrent_snatches'] = int(torrent_info[5].text)
e['torrent_seeds'] = int(torrent_info[6].text)
e['torrent_leeches'] = int(torrent_info[7].text)
entries.add(e)
return entries
'type': 'string',
'pattern': '^([#&][^\x07\x2C\s]{0,200})',
'error_pattern': 'channel name must start with # or & and contain no commas and whitespace',
}
schema = {
'oneOf': [
{
'type': 'object',
'additionalProperties': {
'type': 'object',
'properties': {
'tracker_file': {'type': 'string'},
'server': {'type': 'string'},
'port': {'type': 'integer'},
'nickname': {'type': 'string'},
'channels': one_or_more(channel_pattern),
'nickserv_password': {'type': 'string'},
'invite_nickname': {'type': 'string'},
'invite_message': {'type': 'string'},
'task': one_or_more({'type': 'string'}),
'task_re': {
'type': 'array',
'items': {
'type': 'object',
'properties': {
'task': {'type': 'string'},
'patterns': {
'type': 'array',
'items': {
'type': 'object',
'properties': {
channel_pattern = {
'type': 'string', 'pattern': '^([#&][^\x07\x2C\s]{0,200})',
'error_pattern': 'channel name must start with # or & and contain no commas and whitespace'
}
schema = {
'oneOf': [
{
'type': 'object',
'additionalProperties': {
'type': 'object',
'properties': {
'tracker_file': {'type': 'string'},
'server': {'type': 'string'},
'port': {'type': 'integer'},
'nickname': {'type': 'string'},
'channels': one_or_more(channel_pattern),
'nickserv_password': {'type': 'string'},
'invite_nickname': {'type': 'string'},
'invite_message': {'type': 'string'},
'task': one_or_more({
'type': 'string'
}),
'task_re': {
'type': 'object',
'additionalProperties': one_or_more({
'type': 'object',
'properties': {
'regexp': {'type': 'string'},
'field': {'type': 'string'}
},
'required': ['regexp', 'field'],
class PluginPyLoad(object):
"""
Parse task content or url for hoster links and adds them to pyLoad.
Example::
pyload:
api: http://localhost:8000/api
queue: yes
username: my_username
password: my_password
folder: desired_folder
package: desired_package_name (jinja2 supported)
package_password: desired_package_password
hoster:
- YoutubeCom
parse_url: no
multiple_hoster: yes
enabled: yes
Default values for the config elements::
pyload:
api: http://localhost:8000/api
queue: no
hoster: ALL
parse_url: no
multiple_hoster: yes
enabled: yes
"""
__author__ = 'http://pyload.org'
__version__ = '0.5'
DEFAULT_API = 'http://localhost:8000/api'
DEFAULT_QUEUE = False
DEFAULT_FOLDER = ''
DEFAULT_HOSTER = []
DEFAULT_PARSE_URL = False
DEFAULT_MULTIPLE_HOSTER = True
DEFAULT_PREFERRED_HOSTER_ONLY = False
DEFAULT_HANDLE_NO_URL_AS_FAILURE = False
schema = {
'oneOf': [
{'type': 'boolean'},
{'type': 'object',
'properties': {
'api': {'type': 'string'},
'username': {'type': 'string'},
'password': {'type': 'string'},
'folder': {'type': 'string'},
'package': {'type': 'string'},
'package_password': {'type': 'string'},
'queue': {'type': 'boolean'},
'parse_url': {'type': 'boolean'},
'multiple_hoster': {'type': 'boolean'},
'hoster': one_or_more({'type': 'string'}),
'preferred_hoster_only': {'type': 'boolean'},
'handle_no_url_as_failure': {'type': 'boolean'},
'enabled': {'type': 'boolean'},
},
'additionalProperties': False
}
]
}
def on_task_output(self, task, config):
if not config.get('enabled', True):
return
if not task.accepted:
return
self.add_entries(task, config)
def add_entries(self, task, config):
"""Adds accepted entries"""
apiurl = config.get('api', self.DEFAULT_API)
api = PyloadApi(task.requests, apiurl)
try:
session = api.get_session(config)
except IOError:
raise plugin.PluginError('pyLoad not reachable', log)
except plugin.PluginError:
raise
except Exception as e:
raise plugin.PluginError('Unknown error: %s' % str(e), log)
hoster = config.get('hoster', self.DEFAULT_HOSTER)
for entry in task.accepted:
# bunch of urls now going to check
content = entry.get('description', '') + ' ' + quote(entry['url'])
content = json.dumps(content.encode("utf8"))
url = json.dumps(entry['url']) if config.get('parse_url', self.DEFAULT_PARSE_URL) else "''"
log.debug("Parsing url %s" % url)
result = api.query("parseURLs", {"html": content, "url": url, "session": session})
# parsed { plugins: [urls] }
parsed = result.json()
urls = []
# check for preferred hoster
for name in hoster:
if name in parsed:
urls.extend(parsed[name])
if not config.get('multiple_hoster', self.DEFAULT_MULTIPLE_HOSTER):
break
# no preferred hoster and not preferred hoster only - add all recognized plugins
if not urls and not config.get('preferred_hoster_only', self.DEFAULT_PREFERRED_HOSTER_ONLY):
for name, purls in parsed.items():
if name != "BasePlugin":
urls.extend(purls)
if task.options.test:
log.info('Would add `%s` to pyload' % urls)
continue
# no urls found
if not urls:
if config.get('handle_no_url_as_failure', self.DEFAULT_HANDLE_NO_URL_AS_FAILURE):
entry.fail("No suited urls in entry %s" % entry['title'])
else:
log.info("No suited urls in entry %s" % entry['title'])
continue
log.debug("Add %d urls to pyLoad" % len(urls))
try:
dest = 1 if config.get('queue', self.DEFAULT_QUEUE) else 0 # Destination.Queue = 1
# Use the title of the entry, if no naming schema for the package is defined.
name = config.get('package', entry['title'])
# If name has jinja template, render it
try:
name = entry.render(name)
except RenderError as e:
name = entry['title']
log.error('Error rendering jinja event: %s' % e)
post = {'name': "'%s'" % name.encode("ascii", "ignore"),
'links': str(urls),
'dest': dest,
'session': session}
pid = api.query("addPackage", post).text
log.debug('added package pid: %s' % pid)
# Set Folder
folder = config.get('folder', self.DEFAULT_FOLDER)
folder = entry.get('path', folder)
if folder:
# If folder has jinja template, render it
try:
folder = entry.render(folder)
except RenderError as e:
folder = self.DEFAULT_FOLDER
log.error('Error rendering jinja event: %s' % e)
# set folder with api
data = json.dumps({'folder': folder})
api.query("setPackageData", {'pid': pid, 'data': data, 'session': session})
# Set Package Password
package_password = config.get('package_password')
if package_password:
data = json.dumps({'password': package_password})
api.query("setPackageData", {'pid': pid, 'data': data, 'session': session})
except Exception as e:
entry.fail(str(e))
class FromIMDB(object):
"""
This plugin enables generating entries based on an entity, an entity being a person, character or company.
It's based on IMDBpy which is required (pip install imdbpy). The basic config required just an IMDB ID of the
required entity.
For example:
from_imdb: ch0001354
Schema description:
Other than ID, all other properties are meant to filter the full list that the entity generates.
id: string that relates to a supported entity type. For example: 'nm0000375'. Required.
job_types: a string or list with job types from job_types. Default is 'actor'.
content_types: A string or list with content types from content_types. Default is 'movie'.
max_entries: The maximum number of entries that can return. This value's purpose is basically flood protection
against unruly configurations that will return too many results. Default is 200.
Advanced config example:
dynamic_movie_queue:
from_imdb:
id: co0051941
job_types:
- actor
- director
content_types: tv series
accept_all: yes
movie_queue: add
"""
job_types = [
'actor', 'actress', 'director', 'producer', 'writer', 'self', 'editor',
'miscellaneous', 'editorial department', 'cinematographer',
'visual effects', 'thanks', 'music department', 'in development',
'archive footage', 'soundtrack'
]
content_types = [
'movie', 'tv series', 'tv mini series', 'video game', 'video movie',
'tv movie', 'episode'
]
content_type_conversion = {
'movie': 'movie',
'tv series': 'tv',
'tv mini series': 'tv',
'tv movie': 'tv',
'episode': 'tv',
'video movie': 'video',
'video game': 'video game'
}
character_content_type_conversion = {
'movie': 'feature',
'tv series': 'tv',
'tv mini series': 'tv',
'tv movie': 'tv',
'episode': 'tv',
'video movie': 'video',
'video game': 'video-game',
}
jobs_without_content_type = [
'actor', 'actress', 'self', 'in development', 'archive footage'
]
imdb_pattern = one_or_more(
{
'type':
'string',
'pattern':
r'(nm|co|ch)\d{7}',
'error_pattern':
'Get the id from the url of the person/company you want to use,'
' e.g. http://imdb.com/text/<id here>/blah'
},
unique_items=True)
schema = {
'oneOf': [
imdb_pattern, {
'type': 'object',
'properties': {
'id':
imdb_pattern,
'job_types':
one_or_more({
'type': 'string',
'enum': job_types
},
unique_items=True),
'content_types':
one_or_more({
'type': 'string',
'enum': content_types
},
unique_items=True),
'max_entries': {
'type': 'integer'
},
'match_type': {
'type': 'string',
'enum': ['strict', 'loose']
}
},
'required': ['id'],
'additionalProperties': False
}
],
}
def prepare_config(self, config):
"""
Converts config to dict form and sets defaults if needed
"""
config = config
if isinstance(config, basestring):
config = {'id': [config]}
elif isinstance(config, list):
config = {'id': config}
if isinstance(config, dict) and not isinstance(config['id'], list):
config['id'] = [config['id']]
config.setdefault('content_types', [self.content_types[0]])
config.setdefault('job_types', [self.job_types[0]])
config.setdefault('max_entries', 200)
config.setdefault('match_type', 'strict')
if isinstance(config.get('content_types'), str_types):
log.debug('Converted content type from string to list.')
config['content_types'] = [config['content_types']]
if isinstance(config['job_types'], str_types):
log.debug('Converted job type from string to list.')
config['job_types'] = [config['job_types']]
# Special case in case user meant to add actress instead of actor (different job types in IMDB)
if 'actor' in config['job_types'] and 'actress' not in config[
'job_types']:
config['job_types'].append('actress')
return config
def get_items(self, config):
items = []
for id in config['id']:
try:
entity_type, entity_object = self.get_entity_type_and_object(
id)
except Exception as e:
log.error(
'Could not resolve entity via ID: {}. '
'Either error in config or unsupported entity. Error:{}'.
format(id, e))
continue
items += self.get_items_by_entity(entity_type, entity_object,
config.get('content_types'),
config.get('job_types'),
config.get('match_type'))
return set(items)
def get_entity_type_and_object(self, imdb_id):
"""
Return a tuple of entity type and entity object
:param imdb_id: string which contains IMDB id
:return: entity type, entity object (person, company, etc.)
"""
if imdb_id.startswith('nm'):
person = self.ia.get_person(imdb_id[2:])
log.info('Starting to retrieve items for person: %s' % person)
return 'Person', person
elif imdb_id.startswith('co'):
company = self.ia.get_company(imdb_id[2:])
log.info('Starting to retrieve items for company: %s' % company)
return 'Company', company
elif imdb_id.startswith('ch'):
character = self.ia.get_character(imdb_id[2:])
log.info('Starting to retrieve items for Character: %s' %
character)
return 'Character', character
def get_items_by_entity(self, entity_type, entity_object, content_types,
job_types, match_type):
"""
Gets entity object and return movie list using relevant method
"""
if entity_type == 'Company':
return self.items_by_company(entity_object)
if entity_type == 'Character':
return self.items_by_character(entity_object, content_types,
match_type)
elif entity_type == 'Person':
return self.items_by_person(entity_object, job_types,
content_types, match_type)
def flatten_list(self, _list):
"""
Gets a list of lists and returns a flat list
"""
for el in _list:
if isinstance(el, collections.Iterable) and not isinstance(
el, basestring):
for sub in self.flatten_list(el):
yield sub
else:
yield el
def flat_list(self, non_flat_list, remove_none=False):
flat_list = self.flatten_list(non_flat_list)
if remove_none:
flat_list = [_f for _f in flat_list if _f]
return flat_list
def filtered_items(self, unfiltered_items, content_types, match_type):
items = []
unfiltered_items = set(unfiltered_items)
for item in sorted(unfiltered_items):
if match_type == 'strict':
log.debug(
'Match type is strict, verifying item type to requested content types'
)
self.ia.update(item)
if item['kind'] in content_types:
log.verbose(
'Adding item "{}" to list. Item kind is "{}"'.format(
item, item['kind']))
items.append(item)
else:
log.verbose('Rejecting item "{}". Item kind is "{}'.format(
item, item['kind']))
else:
log.debug('Match type is loose, all items are being added')
items.append(item)
return items
def items_by_person(self, person, job_types, content_types, match_type):
"""
Return item list for a person object
"""
unfiltered_items = self.flat_list([
self.items_by_job_type(person, job_type, content_types)
for job_type in job_types
],
remove_none=True)
return self.filtered_items(unfiltered_items, content_types, match_type)
def items_by_content_type(self, person, job_type, content_type):
return [
_f for _f in (person.get(
job_type + ' ' +
self.content_type_conversion[content_type], [])) if _f
]
def items_by_job_type(self, person, job_type, content_types):
items = person.get(
job_type, []) if job_type in self.jobs_without_content_type else [
person.get(job_type + ' ' + 'documentary', [])
and person.get(job_type + ' ' + 'short', [])
and self.items_by_content_type(person, job_type, content_type)
if content_type == 'movie' else self.items_by_content_type(
person, job_type, content_type)
for content_type in content_types
]
return [_f for _f in items if _f]
def items_by_character(self, character, content_types, match_type):
"""
Return items list for a character object
:param character: character object
:param content_types: content types as defined in config
:return:
"""
unfiltered_items = self.flat_list([
character.get(self.character_content_type_conversion[content_type])
for content_type in content_types
],
remove_none=True)
return self.filtered_items(unfiltered_items, content_types, match_type)
def items_by_company(self, company):
"""
Return items list for a company object
:param company: company object
:return: company items list
"""
return company.get('production companies')
@cached('from_imdb', persist='2 hours')
def on_task_input(self, task, config):
try:
from imdb import IMDb
self.ia = IMDb()
except ImportError:
log.error(
'IMDBPY is required for this plugin. Please install using "pip install imdbpy"'
)
return
entries = []
config = self.prepare_config(config)
items = self.get_items(config)
if not items:
log.error(
'Could not get IMDB item list, check your configuration.')
return
for item in items:
entry = Entry(title=item['title'],
imdb_id='tt' + self.ia.get_imdbID(item),
url='',
imdb_url=self.ia.get_imdbURL(item))
if entry.isvalid():
if entry not in entries:
entries.append(entry)
if entry and task.options.test:
log.info("Test mode. Entry includes:")
for key, value in list(entry.items()):
log.info(' {}: {}'.format(
key.capitalize(), value))
else:
log.error('Invalid entry created? %s' % entry)
if len(entries) <= config.get('max_entries'):
return entries
else:
log.warning(
'Number of entries (%s) exceeds maximum allowed value %s. '
'Edit your filters or raise the maximum value by entering a higher "max_entries"'
% (len(entries), config.get('max_entries')))
return
class OutputPushover(object):
"""
Example::
pushover:
userkey: <USER_KEY> (can also be a list of userkeys)
apikey: <API_KEY>
[device: <DEVICE_STRING>] (default: (none))
[title: <MESSAGE_TITLE>] (default: "Download started" -- accepts Jinja2)
[message: <MESSAGE_BODY>] (default uses series/tvdb name and imdb if available -- accepts Jinja2)
[priority: <PRIORITY>] (default = 0 -- normal = 0, high = 1, silent = -1, emergency = 2)
[url: <URL>] (default: "{{imdb_url}}" -- accepts Jinja2)
[urltitle: <URL_TITLE>] (default: (none) -- accepts Jinja2)
[sound: <SOUND>] (default: pushover default)
[retry]: <RETRY>]
Configuration parameters are also supported from entries (eg. through set).
"""
defaults = {
'message': "{% if series_name is defined %}"
"{{tvdb_series_name|d(series_name)}} "
"{{series_id}} {{tvdb_ep_name|d('')}}"
"{% elif imdb_name is defined %}"
"{{imdb_name}} {{imdb_year}}"
"{% else %}"
"{{title}}"
"{% endif %}",
'url': '{% if imdb_url is defined %}{{imdb_url}}{% endif %}',
'title': '{{task}}'
}
schema = {
'type': 'object',
'properties': {
'userkey': one_or_more({'type': 'string'}),
'apikey': {'type': 'string'},
'device': {'type': 'string'},
'title': {'type': 'string'},
'message': {'type': 'string'},
'priority': {'oneOf': [
{'type': 'number', 'minimum': -2, 'maximum': 2},
{'type': 'string'}]},
'url': {'type': 'string'},
'url_title': {'type': 'string'},
'sound': {'type': 'string'},
'retry': {'type': 'integer', 'minimum': 30},
'expire': {'type': 'integer', 'maximum': 86400},
'callback': {'type': 'string', 'format': 'url'}
},
'required': ['userkey', 'apikey'],
'additionalProperties': False
}
last_request = datetime.datetime.strptime('2000-01-01', '%Y-%m-%d')
@staticmethod
def pushover_request(task, data):
time_dif = (datetime.datetime.now() - OutputPushover.last_request).seconds
# Require at least 5 seconds of waiting between API calls
while time_dif < 5:
time_dif = (datetime.datetime.now() - OutputPushover.last_request).seconds
try:
response = task.requests.post(PUSHOVER_URL, data=data, raise_status=False)
OutputPushover.last_request = datetime.datetime.now()
except RequestException:
raise
return response
def prepare_config(self, config):
"""
Returns prepared config with Flexget default values
:param config: User config
:return: Config with defaults
"""
config = config
if not isinstance(config['userkey'], list):
config['userkey'] = [config['userkey']]
config.setdefault('message', self.defaults['message'])
config.setdefault('title', self.defaults['title'])
config.setdefault('url', self.defaults['url'])
return config
# Run last to make sure other outputs are successful before sending notification
@plugin.priority(0)
def on_task_output(self, task, config):
config = self.prepare_config(config)
data = {"token": config["apikey"]}
# Loop through the provided entries
for entry in task.accepted:
for key, value in list(config.items()):
if key in ['apikey', 'userkey']:
continue
# Tried to render data in field
try:
data[key] = entry.render(value)
except RenderError as e:
log.warning('Problem rendering {0}: {1}'.format(key, e))
data[key] = None
except ValueError:
data[key] = None
# If field is empty or rendering fails, try to render field default if exists
if not data[key]:
try:
data[key] = entry.render(self.defaults.get(key))
except ValueError:
if value:
data[key] = value
# Special case, verify certain fields exists if priority is 2
if data.get('priority') == 2 and not all([data.get('expire'), data.get('retry')]):
log.warning('Priority set to 2 but fields "expire" and "retry" are not both present.'
' Lowering priority to 1')
data['priority'] = 1
for userkey in config['userkey']:
# Build the request
data["user"] = userkey
# Check for test mode
if task.options.test:
log.info("Test mode. Pushover notification would be:")
for key, value in list(data.items()):
log.verbose('{0:>5}{1}: {2}'.format('', key.capitalize(), value))
# Test mode. Skip remainder.
continue
for retry in range(NUMBER_OF_RETRIES):
try:
response = self.pushover_request(task, data)
except RequestException as e:
log.warning('Could not get response from Pushover: {0}.'
' Try {1} out of {2}'.format(e, retry + 1, NUMBER_OF_RETRIES))
continue
request_status = response.status_code
# error codes and messages from Pushover API
if request_status == 200:
log.debug("Pushover notification sent")
break
elif request_status == 500:
log.debug("Pushover notification failed, Pushover API having issues. Try {0} out of {1}".format(
retry + 1, NUMBER_OF_RETRIES))
continue
elif request_status >= 400:
errors = json.loads(response.content)['errors']
log.error("Pushover API error: {0}".format(errors[0]))
break
else:
log.error("Unknown error when sending Pushover notification")
break
else:
log.error(
'Could not get response from Pushover after {0} retries, aborting.'.format(NUMBER_OF_RETRIES))
def schema(self):
"""The schema of the plugin
Extends the super's schema
"""
schema = super().schema
schema['properties'].update({
'artist': {
'type': 'string'
},
'album': {
'type': 'string'
},
'year': {
'type': ['string', 'integer']
},
'tags': one_or_more({'type': 'string'}),
'tag_type': {
'type': 'string',
'enum': list(self._opts('tag_type').keys())
},
'encoding': {
'type': 'string',
'enum': self._opts('encoding')
},
'format': {
'type': 'string',
'enum': self._opts('format')
},
'media': {
'type': 'string',
'enum': self._opts('media')
},
'release_type': {
'type': 'string',
'enum': list(self._opts('release_type').keys()),
},
'log': {
'oneOf': [
{
'type': 'string',
'enum': list(self._opts('log').keys())
},
{
'type': 'boolean'
},
]
},
'leech_type': {
'type': 'string',
'enum': list(self._opts('leech_type').keys())
},
'hascue': {
'type': 'boolean'
},
'scene': {
'type': 'boolean'
},
'vanityhouse': {
'type': 'boolean'
},
})
return schema
class UrlRewriteTorrentleech:
"""
Torrentleech urlrewriter and search plugin.
torrentleech:
rss_key: xxxxxxxxx (required)
username: xxxxxxxx (required)
password: xxxxxxxx (required)
category: HD
Category is any combination of: all, Cam, TS, TS/TC, DVDRip,
DVDRip/DVDScreener, WEBRip': 37, HDRip': 43, BDRip, DVDR,
DVD-R, HD, Bluray, 4KUpscaled, Real4K, Movie Boxsets,
Boxsets': 15, Documentaries, Episodes, TV Boxsets, Episodes HD
"""
schema = {
'type': 'object',
'properties': {
'rss_key': {'type': 'string'},
'username': {'type': 'string'},
'password': {'type': 'string'},
'category': one_or_more(
{'oneOf': [{'type': 'integer'}, {'type': 'string', 'enum': list(CATEGORIES)}]}
),
},
'required': ['rss_key', 'username', 'password'],
'additionalProperties': False,
}
# urlrewriter API
def url_rewritable(self, task, entry):
url = entry['url']
if url.endswith('.torrent'):
return False
if url.startswith('https://www.torrentleech.org/'):
return True
return False
# urlrewriter API
def url_rewrite(self, task, entry):
if 'url' not in entry:
logger.error("Didn't actually get a URL...")
else:
logger.debug('Got the URL: {}', entry['url'])
if entry['url'].startswith('https://www.torrentleech.org/torrents/browse/list/query/'):
# use search
results = self.search(task, entry)
if not results:
raise UrlRewritingError("No search results found")
# TODO: Search doesn't enforce close match to title, be more picky
entry['url'] = results[0]['url']
@plugin.internet(logger)
def search(self, task, entry, config=None):
"""
Search for name from torrentleech.
"""
request_headers = {'User-Agent': 'curl/7.54.0'}
rss_key = config['rss_key']
# build the form request:
data = {'username': config['username'], 'password': config['password']}
# POST the login form:
try:
login = task.requests.post(
'https://www.torrentleech.org/user/account/login/',
data=data,
headers=request_headers,
allow_redirects=True,
)
except RequestException as e:
raise PluginError('Could not connect to torrentleech: %s' % str(e))
if login.url.endswith('/user/account/login/'):
raise PluginError('Could not login to torrentleech, faulty credentials?')
if not isinstance(config, dict):
config = {}
# sort = SORT.get(config.get('sort_by', 'seeds'))
# if config.get('sort_reverse'):
# sort += 1
categories = config.get('category', 'all')
# Make sure categories is a list
if not isinstance(categories, list):
categories = [categories]
# If there are any text categories, turn them into their id number
categories = [c if isinstance(c, int) else CATEGORIES[c] for c in categories]
filter_url = '/categories/{}'.format(','.join(str(c) for c in categories))
entries = set()
for search_string in entry.get('search_strings', [entry['title']]):
query = normalize_unicode(search_string).replace(":", "")
# urllib.quote will crash if the unicode string has non ascii characters,
# so encode in utf-8 beforehand
url = (
'https://www.torrentleech.org/torrents/browse/list/query/'
+ quote(query.encode('utf-8'))
+ filter_url
)
logger.debug('Using {} as torrentleech search url', url)
results = task.requests.get(url, headers=request_headers, cookies=login.cookies).json()
for torrent in results['torrentList']:
entry = Entry()
entry['download_headers'] = request_headers
entry['title'] = torrent['name']
# construct download URL
torrent_url = 'https://www.torrentleech.org/rss/download/{}/{}/{}'.format(
torrent['fid'], rss_key, torrent['filename']
)
logger.debug('RSS-ified download link: {}', torrent_url)
entry['url'] = torrent_url
# seeders/leechers
entry['torrent_seeds'] = torrent['seeders']
entry['torrent_leeches'] = torrent['leechers']
entry['torrent_availability'] = torrent_availability(
entry['torrent_seeds'], entry['torrent_leeches']
)
entry['content_size'] = parse_filesize(str(torrent['size']) + ' b')
entries.add(entry)
return sorted(entries, reverse=True, key=lambda x: x.get('torrent_availability'))
class InputRSS(object):
"""
Parses RSS feed.
Hazzlefree configuration for public rss feeds::
rss: <url>
Configuration with basic http authentication::
rss:
url: <url>
username: <name>
password: <password>
Advanced usages:
You may wish to clean up the entry by stripping out all non-ascii characters.
This can be done by setting ascii value to yes.
Example::
rss:
url: <url>
ascii: yes
In case RSS-feed uses some nonstandard field for urls and automatic detection fails
you can configure plugin to use url from any feedparser entry attribute.
Example::
rss:
url: <url>
link: guid
If you want to keep information in another rss field attached to the flexget entry,
you can use the other_fields option.
Example::
rss:
url: <url>
other_fields: [date]
You can disable few possibly annoying warnings by setting silent value to
yes on feeds where there are frequently invalid items.
Example::
rss:
url: <url>
silent: yes
You can group all the links of an item, to make the download plugin tolerant
to broken urls: it will try to download each url until one works.
Links are enclosures plus item fields given by the link value, in that order.
The value to set is "group_links".
Example::
rss:
url: <url>
group_links: yes
"""
schema = {
'type': ['string', 'object'],
# Simple form, just url or file
'anyOf': [{
'format': 'url'
}, {
'format': 'file'
}],
# Advanced form, with options
'properties': {
'url': {
'type': 'string',
'anyOf': [{
'format': 'url'
}, {
'format': 'file'
}]
},
'username': {
'type': 'string'
},
'password': {
'type': 'string'
},
'title': {
'type': 'string'
},
'link': one_or_more({'type': 'string'}),
'silent': {
'type': 'boolean',
'default': False
},
'ascii': {
'type': 'boolean',
'default': False
},
'filename': {
'type': 'boolean'
},
'group_links': {
'type': 'boolean',
'default': False
},
'all_entries': {
'type': 'boolean',
'default': True
},
'other_fields': {
'type': 'array',
'items': {
# Items can be a string, or a dict with a string value
'type': ['string', 'object'],
'additionalProperties': {
'type': 'string'
}
}
}
},
'required': ['url'],
'additionalProperties': False
}
def build_config(self, config):
"""Set default values to config"""
if isinstance(config, basestring):
config = {'url': config}
# set the default link value to 'auto'
config.setdefault('link', 'auto')
# Convert any field names from the config to format feedparser will use for 'link', 'title' and 'other_fields'
if config['link'] != 'auto':
if not isinstance(config['link'], list):
config['link'] = [config['link']]
config['link'] = map(fp_field_name, config['link'])
config.setdefault('title', 'title')
config['title'] = fp_field_name(config['title'])
if config.get('other_fields'):
other_fields = []
for item in config['other_fields']:
if isinstance(item, basestring):
key, val = item, item
else:
key, val = item.items()[0]
other_fields.append({fp_field_name(key): val.lower()})
config['other_fields'] = other_fields
# set default value for group_links as deactivated
config.setdefault('group_links', False)
# set default for all_entries
config.setdefault('all_entries', True)
return config
def process_invalid_content(self, task, data, url):
"""If feedparser reports error, save the received data and log error."""
if data is None:
log.critical('Received empty page - no content')
return
ext = 'xml'
if '<html>' in data.lower():
log.critical('Received content is HTML page, not an RSS feed')
ext = 'html'
if 'login' in data.lower() or 'username' in data.lower():
log.critical('Received content looks a bit like login page')
if 'error' in data.lower():
log.critical('Received content looks a bit like error page')
received = os.path.join(task.manager.config_base, 'received')
if not os.path.isdir(received):
os.mkdir(received)
filename = task.name
sourcename = urlparse.urlparse(url).netloc
if sourcename:
filename += '-' + sourcename
filename = pathscrub(filename, filename=True)
filepath = os.path.join(received, '%s.%s' % (filename, ext))
with open(filepath, 'w') as f:
f.write(data)
log.critical('I have saved the invalid content to %s for you to view',
filepath)
def add_enclosure_info(self,
entry,
enclosure,
filename=True,
multiple=False):
"""Stores information from an rss enclosure into an Entry."""
entry['url'] = enclosure['href']
# get optional meta-data
if 'length' in enclosure:
try:
entry['size'] = int(enclosure['length'])
except:
entry['size'] = 0
if 'type' in enclosure:
entry['type'] = enclosure['type']
# TODO: better and perhaps join/in download plugin?
# Parse filename from enclosure url
basename = posixpath.basename(urlparse.urlsplit(entry['url']).path)
# If enclosure has size OR there are multiple enclosures use filename from url
if (entry.get('size') or multiple and basename) and filename:
entry['filename'] = basename
log.trace('filename `%s` from enclosure', entry['filename'])
@cached('rss')
@plugin.internet(log)
def on_task_input(self, task, config):
config = self.build_config(config)
log.debug('Requesting task `%s` url `%s`', task.name, config['url'])
# Used to identify which etag/modified to use
url_hash = str(hash(config['url']))
# set etag and last modified headers if config has not changed since
# last run and if caching wasn't disabled with --no-cache argument.
all_entries = (config['all_entries'] or task.config_modified
or task.options.nocache or task.options.retry)
headers = {}
if not all_entries:
etag = task.simple_persistence.get('%s_etag' % url_hash, None)
if etag:
log.debug('Sending etag %s for task %s', etag, task.name)
headers['If-None-Match'] = etag
modified = task.simple_persistence.get('%s_modified' % url_hash,
None)
if modified:
if not isinstance(modified, basestring):
log.debug(
'Invalid date was stored for last modified time.')
else:
headers['If-Modified-Since'] = modified
log.debug('Sending last-modified %s for task %s',
headers['If-Modified-Since'], task.name)
# Get the feed content
if config['url'].startswith(('http', 'https', 'ftp', 'file')):
# Get feed using requests library
auth = None
if 'username' in config and 'password' in config:
auth = (config['username'], config['password'])
try:
# Use the raw response so feedparser can read the headers and status values
response = task.requests.get(config['url'],
timeout=60,
headers=headers,
raise_status=False,
auth=auth)
content = response.content
except RequestException as e:
raise plugin.PluginError(
'Unable to download the RSS for task %s (%s): %s' %
(task.name, config['url'], e))
if config.get('ascii'):
# convert content to ascii (cleanup), can also help with parsing problems on malformed feeds
content = response.text.encode('ascii', 'ignore')
# status checks
status = response.status_code
if status == 304:
log.verbose(
'%s hasn\'t changed since last run. Not creating entries.',
config['url'])
# Let details plugin know that it is ok if this feed doesn't produce any entries
task.no_entries_ok = True
return []
elif status == 401:
raise plugin.PluginError(
'Authentication needed for task %s (%s): %s' %
(task.name, config['url'],
response.headers['www-authenticate']), log)
elif status == 404:
raise plugin.PluginError(
'RSS Feed %s (%s) not found' % (task.name, config['url']),
log)
elif status == 500:
raise plugin.PluginError(
'Internal server exception on task %s (%s)' %
(task.name, config['url']), log)
elif status != 200:
raise plugin.PluginError(
'HTTP error %s received from %s' % (status, config['url']),
log)
# update etag and last modified
if not config['all_entries']:
etag = response.headers.get('etag')
if etag:
task.simple_persistence['%s_etag' % url_hash] = etag
log.debug('etag %s saved for task %s', etag, task.name)
if response.headers.get('last-modified'):
modified = response.headers['last-modified']
task.simple_persistence['%s_modified' %
url_hash] = modified
log.debug('last modified %s saved for task %s', modified,
task.name)
else:
# This is a file, open it
with open(config['url'], 'rb') as f:
content = f.read()
if config.get('ascii'):
# Just assuming utf-8 file in this case
content = content.decode('utf-8',
'ignore').encode('ascii', 'ignore')
if not content:
log.error('No data recieved for rss feed.')
return
try:
rss = feedparser.parse(content)
except LookupError as e:
raise plugin.PluginError('Unable to parse the RSS (from %s): %s' %
(config['url'], e))
# check for bozo
ex = rss.get('bozo_exception', False)
if ex or rss.get('bozo'):
if rss.entries:
msg = 'Bozo error %s while parsing feed, but entries were produced, ignoring the error.' % type(
ex)
if config.get('silent', False):
log.debug(msg)
else:
log.verbose(msg)
else:
if isinstance(ex, feedparser.NonXMLContentType):
# see: http://www.feedparser.org/docs/character-encoding.html#advanced.encoding.nonxml
log.debug('ignoring feedparser.NonXMLContentType')
elif isinstance(ex, feedparser.CharacterEncodingOverride):
# see: ticket 88
log.debug('ignoring feedparser.CharacterEncodingOverride')
elif isinstance(ex, UnicodeEncodeError):
raise plugin.PluginError(
'Feed has UnicodeEncodeError while parsing...')
elif isinstance(ex, (xml.sax._exceptions.SAXParseException,
xml.sax._exceptions.SAXException)):
# save invalid data for review, this is a bit ugly but users seem to really confused when
# html pages (login pages) are received
self.process_invalid_content(task, content, config['url'])
if task.options.debug:
log.error('bozo error parsing rss: %s' % ex)
raise plugin.PluginError(
'Received invalid RSS content from task %s (%s)' %
(task.name, config['url']))
elif isinstance(ex, httplib.BadStatusLine) or isinstance(
ex, IOError):
raise ex # let the @internet decorator handle
else:
# all other bozo errors
self.process_invalid_content(task, content, config['url'])
raise plugin.PluginError(
'Unhandled bozo_exception. Type: %s (task: %s)' %
(ex.__class__.__name__, task.name), log)
log.debug('encoding %s', rss.encoding)
last_entry_id = ''
if not all_entries:
# Test to make sure entries are in descending order
if rss.entries and rss.entries[0].get(
'published_parsed') and rss.entries[-1].get(
'published_parsed'):
if rss.entries[0]['published_parsed'] < rss.entries[-1][
'published_parsed']:
# Sort them if they are not
rss.entries.sort(key=lambda x: x['published_parsed'],
reverse=True)
last_entry_id = task.simple_persistence.get('%s_last_entry' %
url_hash)
# new entries to be created
entries = []
# field name for url can be configured by setting link.
# default value is auto but for example guid is used in some feeds
ignored = 0
for entry in rss.entries:
# Check if title field is overridden in config
title_field = config.get('title', 'title')
# ignore entries without title
if not entry.get(title_field):
log.debug('skipping entry without title')
ignored += 1
continue
# Set the title from the source field
entry.title = entry[title_field]
# Check we haven't already processed this entry in a previous run
if last_entry_id == entry.title + entry.get('guid', ''):
log.verbose('Not processing entries from last run.')
# Let details plugin know that it is ok if this task doesn't produce any entries
task.no_entries_ok = True
break
# remove annoying zero width spaces
entry.title = entry.title.replace(u'\u200B', u'')
# Dict with fields to grab mapping from rss field name to FlexGet field name
fields = {
'guid': 'guid',
'author': 'author',
'description': 'description',
'infohash': 'torrent_info_hash'
}
# extend the dict of fields to grab with other_fields list in config
for field_map in config.get('other_fields', []):
fields.update(field_map)
# helper
# TODO: confusing? refactor into class member ...
def add_entry(ea):
ea['title'] = entry.title
for rss_field, flexget_field in fields.iteritems():
if rss_field in entry:
if not isinstance(getattr(entry, rss_field),
basestring):
# Error if this field is not a string
log.error(
'Cannot grab non text field `%s` from rss.',
rss_field)
# Remove field from list of fields to avoid repeated error
config['other_fields'].remove(rss_field)
continue
if not getattr(entry, rss_field):
log.debug(
'Not grabbing blank field %s from rss for %s.',
rss_field, ea['title'])
continue
try:
ea[flexget_field] = decode_html(entry[rss_field])
if rss_field in config.get('other_fields', []):
# Print a debug message for custom added fields
log.debug('Field `%s` set to `%s` for `%s`',
rss_field, ea[rss_field],
ea['title'])
except UnicodeDecodeError:
log.warning(
'Failed to decode entry `%s` field `%s`',
ea['title'], rss_field)
# Also grab pubdate if available
if hasattr(entry,
'published_parsed') and entry.published_parsed:
ea['rss_pubdate'] = datetime(*entry.published_parsed[:6])
# store basic auth info
if 'username' in config and 'password' in config:
ea['download_auth'] = (config['username'],
config['password'])
entries.append(ea)
# create from enclosures if present
enclosures = entry.get('enclosures', [])
if len(enclosures) > 1 and not config.get('group_links'):
# There is more than 1 enclosure, create an Entry for each of them
log.debug('adding %i entries from enclosures', len(enclosures))
for enclosure in enclosures:
if 'href' not in enclosure:
log.debug('RSS-entry `%s` enclosure does not have URL',
entry.title)
continue
# There is a valid url for this enclosure, create an Entry for it
ee = Entry()
self.add_enclosure_info(ee, enclosure,
config.get('filename', True), True)
add_entry(ee)
# If we created entries for enclosures, we should not create an Entry for the main rss item
continue
# create flexget entry
e = Entry()
if not isinstance(config.get('link'), list):
# If the link field is not a list, search for first valid url
if config['link'] == 'auto':
# Auto mode, check for a single enclosure url first
if len(entry.get(
'enclosures',
[])) == 1 and entry['enclosures'][0].get('href'):
self.add_enclosure_info(e, entry['enclosures'][0],
config.get('filename', True))
else:
# If there is no enclosure url, check link, then guid field for urls
for field in ['link', 'guid']:
if entry.get(field):
e['url'] = entry[field]
break
else:
if entry.get(config['link']):
e['url'] = entry[config['link']]
else:
# If link was passed as a list, we create a list of urls
for field in config['link']:
if entry.get(field):
e.setdefault('url', entry[field])
if entry[field] not in e.setdefault('urls', []):
e['urls'].append(entry[field])
if config.get('group_links'):
# Append a list of urls from enclosures to the urls field if group_links is enabled
e.setdefault('urls', [e['url']]).extend([
enc.href for enc in entry.get('enclosures', [])
if enc.get('href') not in e['urls']
])
if not e.get('url'):
log.debug('%s does not have link (%s) or enclosure',
entry.title, config['link'])
ignored += 1
continue
add_entry(e)
# Save last spot in rss
if rss.entries:
log.debug('Saving location in rss feed.')
try:
task.simple_persistence['%s_last_entry' % url_hash] = (
rss.entries[0].title + rss.entries[0].get('guid', ''))
except AttributeError:
log.debug(
'rss feed location saving skipped: no title information in first entry'
)
if ignored:
if not config.get('silent'):
log.warning(
'Skipped %s RSS-entries without required information (title, link or enclosures)',
ignored)
return entries
class SearchRarBG(object):
"""
RarBG search plugin.
To perform search against single category:
rarbg:
category: x264 720p
To perform search against multiple categories:
rarbg:
category:
- x264 720p
- x264 1080p
Movie categories accepted: x264 720p, x264 1080p, XviD, Full BD
TV categories accepted: HDTV, SDTV
You can use also use category ID manually if you so desire (eg. x264 720p is actually category id '45')
"""
schema = {
'type': 'object',
'properties': {
'category':
one_or_more({
'oneOf': [
{
'type': 'integer'
},
{
'type': 'string',
'enum': list(CATEGORIES)
},
]
}),
'sorted_by': {
'type': 'string',
'enum': ['seeders', 'leechers', 'last'],
'default': 'last'
},
# min_seeders and min_leechers do not seem to work
# 'min_seeders': {'type': 'integer', 'default': 0},
# 'min_leechers': {'type': 'integer', 'default': 0},
'limit': {
'type': 'integer',
'enum': [25, 50, 100],
'default': 25
},
'ranked': {
'type': 'boolean',
'default': True
}
},
"additionalProperties": False
}
base_url = 'https://torrentapi.org/pubapi.php'
def get_token(self):
# using rarbg.com to avoid the domain delay as tokens can be requested always
r = requests.get('https://rarbg.com/pubapi/pubapi.php',
params={
'get_token': 'get_token',
'format': 'json'
})
token = None
try:
token = r.json().get('token')
except ValueError:
log.error('Could not retrieve RARBG token.')
log.debug('RarBG token: %s' % token)
return token
@plugin.internet(log)
def search(self, task, entry, config):
"""
Search for entries on RarBG
"""
categories = config.get('category', 'all')
# Ensure categories a list
if not isinstance(categories, list):
categories = [categories]
# Convert named category to its respective category id number
categories = [
c if isinstance(c, int) else CATEGORIES[c] for c in categories
]
category_url_fragment = urllib.quote(';'.join(
str(c) for c in categories))
entries = set()
token = self.get_token()
if not token:
log.error('No token set. Exiting RARBG search.')
return entries
params = {
'mode': 'search',
'token': token,
'ranked': int(config['ranked']),
# 'min_seeders': config['min_seeders'], 'min_leechers': config['min_leechers'],
'sort': config['sorted_by'],
'category': category_url_fragment,
'format': 'json'
}
for search_string in entry.get('search_strings', [entry['title']]):
params.pop('search_string', None)
params.pop('search_imdb', None)
if entry.get('movie_name'):
params['search_imdb'] = entry.get('imdb_id')
else:
query = normalize_unicode(search_string)
query_url_fragment = query.encode('utf8')
params['search_string'] = query_url_fragment
page = requests.get(self.base_url, params=params)
try:
r = page.json()
except ValueError:
log.debug(page.text)
break
for result in r:
entry = Entry()
entry['title'] = result.get('f')
entry['url'] = result.get('d')
entries.add(entry)
return entries
class RapidpushNotifier(object):
"""
Example::
rapidpush:
apikey: xxxxxxx (can also be a list of api keys)
[category: category, default FlexGet]
[group: device group, default no group]
[channel: the broadcast notification channel, if provided it will be send to the channel subscribers instead of
your devices, default no channel]
[priority: 0 - 6 (6 = highest), default 2 (normal)]
"""
schema = {
'type': 'object',
'properties': {
'api_key': one_or_more({'type': 'string'}),
'category': {
'type': 'string',
'default': 'Flexget'
},
'group': {
'type': 'string'
},
'channel': {
'type': 'string'
},
'priority': {
'type': 'integer',
'minimum': 0,
'maximum': 6
}
},
'additionalProperties': False,
'required': ['api_key']
}
def notify(self, title, message, config):
"""
Send a Rapidpush notification
"""
wrapper = {}
notification = {'title': title, 'message': message}
if not isinstance(config['api_key'], list):
config['api_key'] = [config['api_key']]
if config.get('channel'):
wrapper['command'] = 'broadcast'
else:
wrapper['command'] = 'notify'
notification['category'] = config['category']
if config.get('group'):
notification['group'] = config['group']
if config.get('priority') is not None:
notification['priority'] = config['priority']
wrapper['data'] = notification
for key in config['api_key']:
wrapper['apikey'] = key
try:
response = requests.post(RAPIDPUSH_URL, json=wrapper)
except RequestException as e:
raise PluginWarning(e.args[0])
else:
if response.json()['code'] > 400:
raise PluginWarning(response.json()['desc'])
class PluginPathBySpace:
"""Allows setting a field to a folder based on it's space
Path will be selected at random if multiple paths match the within
Example:
path_by_space:
select: most_free_percent # or most_free, most_used, most_used_percent, has_free
within: 9000 # within in MB or percent.
paths:
- /drive1/
- /drive2/
- /drive3/
"""
schema = {
'type': 'object',
'properties': {
'select': {
'type': 'string',
'enum': list(selector_map.keys())
},
'to_field': {
'type': 'string',
'default': 'path'
},
'paths': one_or_more({
'type': 'string',
'format': 'path'
}),
'within': {
'oneOf': [
{
'type': 'string',
'format': 'size'
},
{
'type': 'string',
'format': 'percent'
},
]
},
},
'required': ['paths', 'select'],
'additionalProperties': False,
}
@plugin.priority(250) # run before other plugins
def on_task_metainfo(self, task, config):
selector = selector_map[config['select']]
# Convert within to bytes (int) or percent (float)
within = config.get('within')
if isinstance(within, str) and '%' in within:
within = parse_percent(within)
else:
within = parse_size(within)
path = selector(config['paths'], within=within)
if path:
log.debug('Path %s selected due to (%s)' %
(path, config['select']))
for entry in task.all_entries:
entry[config['to_field']] = path
else:
task.abort('Unable to select a path based on %s' %
config['select'])
return
class PluginExec(object):
"""
Execute commands
Simple example, xecute command for entries that reach output::
exec: echo 'found {{title}} at {{url}}' > file
Advanced Example::
exec:
on_start:
phase: echo "Started"
on_input:
for_entries: echo 'got {{title}}'
on_output:
for_accepted: echo 'accepted {{title}} - {{url}} > file
You can use all (available) entry fields in the command.
"""
NAME = 'exec'
HANDLED_PHASES = ['start', 'input', 'filter', 'output', 'exit']
schema = {
'oneOf': [
one_or_more({'type': 'string'}),
{
'type': 'object',
'properties': {
'on_start': {
'$ref': '#/definitions/phaseSettings'
},
'on_input': {
'$ref': '#/definitions/phaseSettings'
},
'on_filter': {
'$ref': '#/definitions/phaseSettings'
},
'on_output': {
'$ref': '#/definitions/phaseSettings'
},
'on_exit': {
'$ref': '#/definitions/phaseSettings'
},
'fail_entries': {
'type': 'boolean'
},
'auto_escape': {
'type': 'boolean'
},
'encoding': {
'type': 'string'
},
'allow_background': {
'type': 'boolean'
},
},
'additionalProperties': False,
},
],
'definitions': {
'phaseSettings': {
'type': 'object',
'properties': {
'phase': one_or_more({'type': 'string'}),
'for_entries': one_or_more({'type': 'string'}),
'for_accepted': one_or_more({'type': 'string'}),
'for_rejected': one_or_more({'type': 'string'}),
'for_undecided': one_or_more({'type': 'string'}),
'for_failed': one_or_more({'type': 'string'}),
},
'additionalProperties': False,
}
},
}
def prepare_config(self, config):
if isinstance(config, basestring):
config = [config]
if isinstance(config, list):
config = {'on_output': {'for_accepted': config}}
if not config.get('encoding'):
config['encoding'] = io_encoding
for phase_name in config:
if phase_name.startswith('on_'):
for items_name in config[phase_name]:
if isinstance(config[phase_name][items_name], basestring):
config[phase_name][items_name] = [
config[phase_name][items_name]
]
return config
def execute_cmd(self, cmd, allow_background, encoding):
log.verbose('Executing: %s', cmd)
p = subprocess.Popen(
text_to_native_str(cmd, encoding=io_encoding),
shell=True,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
close_fds=False,
)
if not allow_background:
r, w = (p.stdout, p.stdin)
response = r.read().decode(io_encoding)
r.close()
w.close()
if response:
log.info('Stdout: %s',
response.rstrip()) # rstrip to get rid of newlines
return p.wait()
def execute(self, task, phase_name, config):
config = self.prepare_config(config)
if phase_name not in config:
log.debug('phase %s not configured' % phase_name)
return
name_map = {
'for_entries': task.entries,
'for_accepted': task.accepted,
'for_rejected': task.rejected,
'for_undecided': task.undecided,
'for_failed': task.failed,
}
allow_background = config.get('allow_background')
for operation, entries in name_map.items():
if operation not in config[phase_name]:
continue
log.debug('running phase_name: %s operation: %s entries: %s' %
(phase_name, operation, len(entries)))
for entry in entries:
for cmd in config[phase_name][operation]:
entrydict = EscapingEntry(entry) if config.get(
'auto_escape') else entry
# Do string replacement from entry, but make sure quotes get escaped
try:
cmd = render_from_entry(cmd, entrydict)
except RenderError as e:
log.error('Could not set exec command for %s: %s' %
(entry['title'], e))
# fail the entry if configured to do so
if config.get('fail_entries'):
entry.fail(
'Entry `%s` does not have required fields for string replacement.'
% entry['title'])
continue
log.debug('phase_name: %s operation: %s cmd: %s' %
(phase_name, operation, cmd))
if task.options.test:
log.info('Would execute: %s' % cmd)
else:
# Make sure the command can be encoded into appropriate encoding, don't actually encode yet,
# so logging continues to work.
try:
cmd.encode(config['encoding'])
except UnicodeEncodeError:
log.error('Unable to encode cmd `%s` to %s' %
(cmd, config['encoding']))
if config.get('fail_entries'):
entry.fail(
'cmd `%s` could not be encoded to %s.' %
(cmd, config['encoding']))
continue
# Run the command, fail entries with non-zero return code if configured to
if self.execute_cmd(
cmd, allow_background, config['encoding']
) != 0 and config.get('fail_entries'):
entry.fail('exec return code was non-zero')
# phase keyword in this
if 'phase' in config[phase_name]:
for cmd in config[phase_name]['phase']:
try:
cmd = render_from_task(cmd, task)
except RenderError as e:
log.error('Error rendering `%s`: %s' % (cmd, e))
else:
log.debug('phase cmd: %s' % cmd)
if task.options.test:
log.info('Would execute: %s' % cmd)
else:
self.execute_cmd(cmd, allow_background,
config['encoding'])
def __getattr__(self, item):
"""Creates methods to handle task phases."""
for phase in self.HANDLED_PHASES:
if item == plugin.phase_methods[phase]:
# A phase method we handle has been requested
break
else:
# We don't handle this phase
raise AttributeError(item)
def phase_handler(task, config):
self.execute(task, 'on_' + phase, config)
# Make sure we run after other plugins so exec can use their output
phase_handler.priority = 100
return phase_handler
class OutputRapidPush(object):
"""
Example::
rapidpush:
apikey: xxxxxxx (can also be a list of api keys)
[category: category, default FlexGet]
[title: title, default New release]
[group: device group, default no group]
[message: the message, default {{title}}]
[channel: the broadcast notification channel, if provided it will be send to the channel subscribers instead of
your devices, default no channel]
[priority: 0 - 6 (6 = highest), default 2 (normal)]
[notify_accepted: boolean true or false, default true]
[notify_rejected: boolean true or false, default false]
[notify_failed: boolean true or false, default false]
[notify_undecided: boolean true or false, default false]
Configuration parameters are also supported from entries (eg. through set).
"""
schema = {
'type': 'object',
'properties': {
'apikey': one_or_more({'type': 'string'}),
'category': {'type': 'string', 'default': 'Flexget'},
'title': {'type': 'string', 'default': 'New Release'},
'group': {'type': 'string', 'default': ''},
'channel': {'type': 'string', 'default': ''},
'priority': {'type': 'integer', 'default': 2},
'message': {'type': 'string', 'default': '{{title}}'},
'notify_accepted': {'type': 'boolean', 'default': True},
'notify_rejected': {'type': 'boolean', 'default': False},
'notify_failed': {'type': 'boolean', 'default': False},
'notify_undecided': {'type': 'boolean', 'default': False}
},
'additionalProperties': False,
'required': ['apikey']
}
# Run last to make sure other outputs are successful before sending notification
@plugin.priority(0)
def on_task_output(self, task, config):
# get the parameters
if config['notify_accepted']:
log.debug("Notify accepted entries")
self.process_notifications(task, task.accepted, config)
if config['notify_rejected']:
log.debug("Notify rejected entries")
self.process_notifications(task, task.rejected, config)
if config['notify_failed']:
log.debug("Notify failed entries")
self.process_notifications(task, task.failed, config)
if config['notify_undecided']:
log.debug("Notify undecided entries")
self.process_notifications(task, task.undecided, config)
# Process the given events.
def process_notifications(self, task, entries, config):
for entry in entries:
if task.options.test:
log.info("Would send RapidPush notification about: %s", entry['title'])
continue
log.info("Send RapidPush notification about: %s", entry['title'])
apikey = entry.get('apikey', config['apikey'])
if isinstance(apikey, list):
apikey = ','.join(apikey)
title = config['title']
try:
title = entry.render(title)
except RenderError as e:
log.error('Error setting RapidPush title: %s' % e)
message = config['message']
try:
message = entry.render(message)
except RenderError as e:
log.error('Error setting RapidPush message: %s' % e)
# Check if we have to send a normal or a broadcast notification.
if not config['channel']:
priority = entry.get('priority', config['priority'])
category = entry.get('category', config['category'])
try:
category = entry.render(category)
except RenderError as e:
log.error('Error setting RapidPush category: %s' % e)
group = entry.get('group', config['group'])
try:
group = entry.render(group)
except RenderError as e:
log.error('Error setting RapidPush group: %s' % e)
# Send the request
data_string = json.dumps({
'title': title,
'message': message,
'priority': priority,
'category': category,
'group': group})
data = {'apikey': apikey, 'command': 'notify', 'data': data_string}
else:
channel = config['channel']
try:
channel = entry.render(channel)
except RenderError as e:
log.error('Error setting RapidPush channel: %s' % e)
# Send the broadcast request
data_string = json.dumps({
'title': title,
'message': message,
'channel': channel})
data = {'apikey': apikey, 'command': 'broadcast', 'data': data_string}
response = task.requests.post(url, data=data, raise_status=False)
json_data = response.json()
if 'code' in json_data:
if json_data['code'] == 200:
log.debug("RapidPush message sent")
else:
log.error(json_data['desc'] + " (" + str(json_data['code']) + ")")
else:
for item in json_data:
if json_data[item]['code'] == 200:
log.debug(item + ": RapidPush message sent")
else:
log.error(item + ": " + json_data[item]['desc'] + " (" + str(json_data[item]['code']) + ")")
def __init__(self):
"""Set up the schema"""
self.schema = {
'type': 'object',
'properties': {
'username': {
'type': 'string'
},
'password': {
'type': 'string'
},
'user_agent': {
'type': 'string'
},
'search': {
'type': 'string'
},
'artist': {
'type': 'string'
},
'album': {
'type': 'string'
},
'year': {
'type': ['string', 'integer']
},
'tags': one_or_more({'type': 'string'}),
'tag_type': {
'type': 'string',
'enum': list(self._opts('tag_type').keys())
},
'encoding': {
'type': 'string',
'enum': self._opts('encoding')
},
'format': {
'type': 'string',
'enum': self._opts('format')
},
'media': {
'type': 'string',
'enum': self._opts('media')
},
'release_type': {
'type': 'string',
'enum': list(self._opts('release_type').keys())
},
'log': {
'oneOf': [{
'type': 'string',
'enum': list(self._opts('log').keys())
}, {
'type': 'boolean'
}]
},
'leech_type': {
'type': 'string',
'enum': list(self._opts('leech_type').keys())
},
'hascue': {
'type': 'boolean'
},
'scene': {
'type': 'boolean'
},
'vanityhouse': {
'type': 'boolean'
},
},
'required': ['username', 'password'],
'additionalProperties': False
}
class UrlRewriteIPTorrents:
"""
IpTorrents urlrewriter and search plugin.
iptorrents:
rss_key: xxxxxxxxx (required)
uid: xxxxxxxx (required)
password: xxxxxxxx (required)
category: HD
Category is any combination of: Movie-all, Movie-3D, Movie-480p,
Movie-4K, Movie-BD-R, Movie-BD-Rip, Movie-Cam, Movie-DVD-R,
Movie-HD-Bluray, Movie-Kids, Movie-MP4, Movie-Non-English,
Movie-Packs, Movie-Web-DL, Movie-x265, Movie-XviD,
TV-all, TV-Documentaries, TV-Sports, TV-480p, TV-BD, TV-DVD-R,
TV-DVD-Rip, TV-MP4, TV-Mobile, TV-Non-English, TV-Packs,
TV-Packs-Non-English, TV-SD-x264, TV-x264, TV-x265, TV-XVID, TV-Web-DL
"""
schema = {
'type': 'object',
'properties': {
'rss_key': {
'type': 'string'
},
'uid': {
'oneOf': [{
'type': 'integer'
}, {
'type': 'string'
}]
},
'password': {
'type': 'string'
},
'category':
one_or_more({
'oneOf': [{
'type': 'integer'
}, {
'type': 'string',
'enum': list(CATEGORIES)
}]
}),
'free': {
'type': 'boolean',
'default': False
},
},
'required': ['rss_key', 'uid', 'password'],
'additionalProperties': False,
}
# urlrewriter API
def url_rewritable(self, task, entry):
url = entry['url']
if url.startswith(BASE_URL + '/download.php/'):
return False
if url.startswith(BASE_URL + '/'):
return True
return False
# urlrewriter API
def url_rewrite(self, task, entry):
if 'url' not in entry:
logger.error("Didn't actually get a URL...")
else:
logger.debug('Got the URL: {}', entry['url'])
if entry['url'].startswith(SEARCH_URL):
# use search
results = self.search(task, entry)
if not results:
raise UrlRewritingError("No search results found")
# TODO: Search doesn't enforce close match to title, be more picky
entry['url'] = results[0]['url']
@plugin.internet(logger)
def search(self, task, entry, config=None):
"""
Search for name from iptorrents
"""
categories = config.get('category', 'All')
# Make sure categories is a list
if not isinstance(categories, list):
categories = [categories]
# If there are any text categories, turn them into their id number
categories = [
c if isinstance(c, int) else CATEGORIES[c] for c in categories
]
category_params = {str(c): '' for c in categories if str(c)}
entries = set()
for search_string in entry.get('search_strings', [entry['title']]):
search_params = {
key: value
for (key, value) in category_params.items()
}
query = normalize_unicode(search_string)
search_params.update({'q': query, 'qf': ''})
logger.debug('searching with params: {}', search_params)
if config.get('free'):
req = requests.get(FREE_SEARCH_URL,
params=search_params,
cookies={
'uid': str(config['uid']),
'pass': config['password']
})
else:
req = requests.get(SEARCH_URL,
params=search_params,
cookies={
'uid': str(config['uid']),
'pass': config['password']
})
logger.debug('full search URL: {}', req.url)
if '/u/' + str(config['uid']) not in req.text:
raise plugin.PluginError(
"Invalid cookies (user not logged in)...")
soup = get_soup(req.content, parser="html.parser")
torrents = soup.find('table', {'id': 'torrents'})
results = torrents.findAll('tr')
for torrent in results:
if torrent.th and 'ac' in torrent.th.get('class'):
# Header column
continue
if torrent.find('td', {'colspan': '99'}):
logger.debug('No results found for search {}',
search_string)
break
entry = Entry()
link = torrent.find('a', href=re.compile('download'))['href']
entry[
'url'] = f"{BASE_URL}{link}?torrent_pass={config.get('rss_key')}"
entry['title'] = torrent.find('a',
href=re.compile('details')).text
seeders = torrent.findNext('td', {
'class': 'ac t_seeders'
}).text
leechers = torrent.findNext('td', {
'class': 'ac t_leechers'
}).text
entry['torrent_seeds'] = int(seeders)
entry['torrent_leeches'] = int(leechers)
entry['torrent_availability'] = torrent_availability(
entry['torrent_seeds'], entry['torrent_leeches'])
size = torrent.findNext(
text=re.compile(r'^([\.\d]+) ([GMK]?)B$'))
size = re.search(r'^([\.\d]+) ([GMK]?)B$', size)
entry['content_size'] = parse_filesize(size.group(0))
logger.debug('Found entry {}', entry)
entries.add(entry)
return entries
class NexusPHP(object):
"""
配置示例
task_name:
rss:
url: https://www.example.com/rss.xml
other_fields:
- link
nexusphp:
cookie: 'my_cookie'
discount:
- free
- 2x
seeders:
min: 1
max: 30
leechers:
min: 1
max: 100
max_complete: 0.8
hr: no
"""
schema = {
'type': 'object',
'properties': {
'cookie': {
'type': 'string'
},
'discount':
one_or_more({
'type': 'string',
'enum': ['free', '2x', '2xfree', '30%', '50%', '2x50%']
}),
'seeders': {
'type': 'object',
'properties': {
'min': {
'type': 'integer',
'minimum': 0,
'default': 0
},
'max': {
'type': 'integer',
'minimum': 0,
'default': 100000
}
}
},
'leechers': {
'type': 'object',
'properties': {
'min': {
'type': 'integer',
'minimum': 0,
'default': 0
},
'max': {
'type': 'integer',
'minimum': 0,
'default': 100000
},
'max_complete': {
'type': 'number',
'minimum': 0,
'maximum': 1,
'default': 1
}
}
},
'hr': {
'type': 'boolean'
},
'adapter': {
'type': 'object',
'properties': {
'free': {
'type': 'string',
'default': 'free'
},
'2x': {
'type': 'strging',
'default': 'twoup'
},
'2xfree': {
'type': 'string',
'default': 'twoupfree'
},
'30%': {
'type': 'string',
'default': 'thirtypercent'
},
'50%': {
'type': 'string',
'default': 'halfdown'
},
'2x50%': {
'type': 'string',
'default': 'twouphalfdown'
}
}
},
'comment': {
'type': 'boolean'
},
'user-agent': {
'type': 'string'
}
},
'required': ['cookie']
}
@staticmethod
def build_config(config):
config = dict(config)
config.setdefault('discount', None)
config.setdefault('seeders', {'min': 0, 'max': 100000})
config.setdefault('leechers', {
'min': 0,
'max': 100000,
'max_complete': 1
})
config.setdefault('hr', True)
config.setdefault('adapter', None)
config.setdefault(
'user-agent',
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) '
'Chrome/75.0.3770.142 Safari/537.36')
return config
@plugin.priority(127)
def on_task_modify(self, task, config):
if config.get('comment', False):
for entry in task.entries:
if 'torrent' in entry and 'link' in entry:
entry['torrent'].content['comment'] = entry['link']
entry['torrent'].modified = True
def on_task_filter(self, task, config):
config = self.build_config(config)
adapter = HTTPAdapter(max_retries=5)
task.requests.mount('http://', adapter)
task.requests.mount('https://', adapter)
# 先访问一次 预防异常
headers = {
'cookie': config['cookie'],
'user-agent': config['user-agent']
}
try:
task.requests.get(task.entries[0].get('link'), headers=headers)
except:
pass
def consider_entry(_entry, _link):
try:
discount, seeders, leechers, hr = NexusPHP._get_info(
task, _link, config['cookie'], config['adapter'],
config['user-agent'])
except plugin.PluginError as e:
raise e
except Exception as e:
log.info('NexusPHP._get_info: ' + str(e))
return
seeder_max = config['seeders']['max']
seeder_min = config['seeders']['min']
leecher_max = config['leechers']['max']
leecher_min = config['leechers']['min']
if config['discount']:
if discount not in config['discount']:
_entry.reject('%s does not match discount' %
discount) # 优惠信息不匹配
return
if config['hr'] is False and hr:
_entry.reject('it is HR') # 拒绝HR
if len(seeders) not in range(seeder_min, seeder_max + 1):
_entry.reject('%d is out of range of seeder' %
len(seeders)) # 做种人数不匹配
return
if len(leechers) not in range(leecher_min, leecher_max + 1):
_entry.reject('%d is out of range of leecher' %
len(leechers)) # 下载人数不匹配
return
if len(leechers) != 0:
max_complete = max(leechers,
key=lambda x: x['completed'])['completed']
else:
max_complete = 0
if max_complete > config['leechers']['max_complete']:
_entry.reject('%f is more than max_complete' %
max_complete) # 最大完成度不匹配
return
_entry.accept()
futures = [] # 线程任务
with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
for entry in task.entries:
link = entry.get('link')
if not link:
raise plugin.PluginError(
"The rss plugin require 'other_fields' which contain 'link'. "
"For example: other_fields: - link")
futures.append(executor.submit(consider_entry, entry, link))
for f in concurrent.futures.as_completed(futures):
exception = f.exception()
if isinstance(exception, plugin.PluginError):
log.info(exception)
@staticmethod
# 解析页面，获取优惠、做种者信息、下载者信息
def info_from_page(detail_page, peer_page, discount_fn, hr_fn=None):
try:
discount = discount_fn(detail_page)
except Exception:
discount = None # 无优惠
try:
if hr_fn:
hr = hr_fn(detail_page)
else:
hr = False
for item in [
'hitandrun', 'hit_run.gif', 'Hit and Run', 'Hit & Run'
]:
if item in detail_page.text:
hr = True
break
except Exception:
hr = False # 无HR
soup = get_soup(peer_page)
tables = soup.find_all('table', limit=2)
try:
seeders = NexusPHP.get_peers(tables[0])
except IndexError:
seeders = []
try:
leechers = NexusPHP.get_peers(tables[1])
except IndexError:
leechers = []
return discount, seeders, leechers, hr
@staticmethod
def get_peers(table):
peers = []
name_index = 0
connectable_index = 1
uploaded_index = 2
downloaded_index = 4
completed_index = 7
for index, tr in enumerate(table.find_all('tr')):
try:
if index == 0:
tds = tr.find_all('td')
for i, td in enumerate(tds):
text = td.get_text()
if text in ['用户', '用戶', '会员/IP']:
name_index = i
elif text in ['可连接', '可連接', '公网']:
connectable_index = i
elif text in ['上传', '上傳', '总上传']:
uploaded_index = i
elif text in ['下载', '下載', '本次下载']:
downloaded_index = i
elif text == '完成':
completed_index = i
else:
tds = tr.find_all('td')
peers.append({
'name':
tds[name_index].get_text(),
'connectable':
True
if tds[connectable_index].get_text() != '是' else False,
'uploaded':
tds[uploaded_index].get_text(),
'downloaded':
tds[downloaded_index].get_text(),
'completed':
float(tds[completed_index].get_text().strip('%')) / 100
})
except Exception:
peers.append({
'name': '',
'connectable': False,
'uploaded': '',
'downloaded': '',
'completed': 0
})
return peers
@staticmethod
def _get_info(task, link, cookie, adapter, user_agent):
headers = {'cookie': cookie, 'user-agent': user_agent}
detail_page = task.requests.get(link, headers=headers) # 详情
detail_page.encoding = 'utf-8'
if 'totheglory' in link:
peer_url = link
else:
peer_url = link.replace('details.php', 'viewpeerlist.php', 1)
try:
peer_page = task.requests.get(peer_url,
headers=headers).text # peer详情
except:
peer_page = ''
if 'login' in detail_page.url:
raise plugin.PluginError(
"Can't access the site. Your cookie may be wrong!")
if adapter:
convert = {value: key for key, value in adapter.items()}
discount_fn = NexusPHP.generate_discount_fn(convert)
return NexusPHP.info_from_page(detail_page, peer_page, discount_fn)
sites_discount = {
'chdbits': {
'pro_free.*?</h1>': 'free',
'pro_2up.*?</h1>': '2x',
'pro_free2up.*?</h1>': '2xfree',
'pro_30pctdown.*?</h1>': '30%',
'pro_50pctdown.*?</h1>': '50%',
'pro_50pctdown2up.*?</h1>': '2x50%'
},
'u2.dmhy': {
'<td.*?top.*?pro_free.*?优惠历史.*?</td>': 'free',
'<td.*?top.*?pro_2up.*?优惠历史.*?</td>': '2x',
'<td.*?top.*?pro_free2up.*?优惠历史.*?</td>': '2xfree',
'<td.*?top.*?pro_30pctdown.*?优惠历史.*?</td>': '30%',
'<td.*?top.*?pro_50pctdown.*?优惠历史.*?</td>': '50%',
'<td.*?top.*?pro_50pctdown2up.*?优惠历史.*?</td>': '2x50%',
'<td.*?top.*?pro_custom.*?优惠历史.*?</td>': '2x'
},
'yingk': {
'span_frees': 'free',
'span_twoupls': '2x',
'span_twoupfreels': '2xfree',
'span_thirtypercentls': '30%',
'span_halfdowns': '50%',
'span_twouphalfdownls': '2x50%'
},
'totheglory': {
'本种子限时不计流量': 'free',
'本种子的下载流量计为实际流量的30%': '30%',
'本种子的下载流量会减半': '50%',
},
'hdchina': {
'pro_free.*?</h2>': 'free',
'pro_2up.*?</h2>': '2x',
'pro_free2up.*?</h2>': '2xfree',
'pro_30pctdown.*?</h2>': '30%',
'pro_50pctdown.*?</h2>': '50%',
'pro_50pctdown2up.*?</h2>': '2x50%'
}
}
for site, convert in sites_discount.items():
if site in link:
discount_fn = NexusPHP.generate_discount_fn(convert)
return NexusPHP.info_from_page(detail_page, peer_page,
discount_fn)
discount_fn = NexusPHP.generate_discount_fn({
'class=\'free\'.*?免.*?</h1>':
'free',
'class=\'twoup\'.*?2X.*?</h1>':
'2x',
'class=\'twoupfree\'.*?2X免.*?</h1>':
'2xfree',
'class=\'thirtypercent\'.*?30%.*?</h1>':
'30%',
'class=\'halfdown\'.*?50%.*?</h1>':
'50%',
'class=\'twouphalfdown\'.*?2X 50%.*?</h1>':
'2x50%'
})
return NexusPHP.info_from_page(detail_page, peer_page, discount_fn)
@staticmethod
def generate_discount_fn(convert):
def fn(page):
html = page.text.replace('\n', '')
for key, value in convert.items():
match = re.search(key, html)
if match:
return value
return None
return fn
class RTorrentInputPlugin(RTorrentPluginBase):
schema = {
'type': 'object',
'properties': {
'uri': {
'type': 'string'
},
'username': {
'type': 'string'
},
'password': {
'type': 'string'
},
'digest_auth': {
'type': 'boolean',
'default': False
},
'view': {
'type': 'string',
'default': 'main'
},
'fields':
one_or_more({
'type': 'string',
'enum': list(RTorrent.default_fields)
}),
},
'required': ['uri'],
'additionalProperties': False,
}
def on_task_input(self, task, config):
client = RTorrent(
os.path.expanduser(config['uri']),
username=config.get('username'),
password=config.get('password'),
digest_auth=config['digest_auth'],
session=task.requests,
)
fields = config.get('fields')
try:
torrents = client.torrents(config['view'], fields=fields)
except (OSError, xmlrpc_client.Error) as e:
task.abort('Could not get torrents (%s): %s' % (config['view'], e))
return
entries = []
for torrent in torrents:
entry = Entry(
title=torrent['name'],
url='%s/%s' %
(os.path.expanduser(config['uri']), torrent['hash']),
path=torrent['base_path'],
torrent_info_hash=torrent['hash'],
)
for attr, value in torrent.items():
entry[attr] = value
if 'timestamp_finished' in entry:
entry['timestamp_finished'] = datetime.fromtimestamp(
entry['timestamp_finished'])
entries.append(entry)
return entries
class EmailNotifier(object):
"""
Send an e-mail with the list of all succeeded (downloaded) entries.
Configuration options
=============== ===================================================================
Option Description
=============== ===================================================================
from The email address from which the email will be sent (required)
to The email address of the recipient (required)
smtp_host The host of the smtp server
smtp_port The port of the smtp server
smtp_username The username to use to connect to the smtp server
smtp_password The password to use to connect to the smtp server
smtp_tls Should we use TLS to connect to the smtp server
smtp_ssl Should we use SSL to connect to the smtp server
=============== ===================================================================
Config basic example::
email:
from: [email protected]
to: [email protected]
smtp_host: smtp.host.com
Config example with smtp login::
email:
from: [email protected]
to: [email protected]
smtp_host: smtp.host.com
smtp_port: 25
smtp_login: true
smtp_username: my_smtp_login
smtp_password: my_smtp_password
smtp_tls: true
GMAIL example::
from: [email protected]
to: [email protected]
smtp_host: smtp.gmail.com
smtp_port: 587
smtp_login: true
smtp_username: gmailUser
smtp_password: gmailPassword
smtp_tls: true
Default values for the config elements::
email:
smtp_host: localhost
smtp_port: 25
smtp_login: False
smtp_username:
smtp_password:
smtp_tls: False
smtp_ssl: False
"""
schema = {
'type': 'object',
'properties': {
'to': one_or_more({
'type': 'string',
'format': 'email'
}),
'from': {
'type': 'string',
'default': '*****@*****.**',
'format': 'email'
},
'smtp_host': {
'type': 'string',
'default': 'localhost'
},
'smtp_port': {
'type': 'integer',
'default': 25
},
'smtp_username': {
'type': 'string'
},
'smtp_password': {
'type': 'string'
},
'smtp_tls': {
'type': 'boolean',
'default': False
},
'smtp_ssl': {
'type': 'boolean',
'default': False
},
'html': {
'type': 'boolean',
'default': False
},
},
'required': ['to'],
'dependencies': {
'smtp_username': ['smtp_password'],
'smtp_password': ['smtp_username'],
'smtp_ssl': ['smtp_tls']
},
'additionalProperties': False,
}
def notify(self, title, message, config):
"""
Send an email notification
:param str message: message body
:param str title: message subject
:param dict config: email plugin config
"""
if not isinstance(config['to'], list):
config['to'] = [config['to']]
email = MIMEMultipart('alternative')
email['To'] = ','.join(config['to'])
email['From'] = config['from']
email['Subject'] = title
email['Date'] = formatdate(localtime=True)
content_type = 'html' if config['html'] else 'plain'
email.attach(
MIMEText(message.encode('utf-8'), content_type, _charset='utf-8'))
try:
log.debug('sending email notification to %s:%s',
config['smtp_host'], config['smtp_port'])
mail_server = smtplib.SMTP_SSL if config[
'smtp_ssl'] else smtplib.SMTP
mail_server = mail_server(config['smtp_host'], config['smtp_port'])
if config['smtp_tls']:
mail_server.ehlo()
mail_server.starttls()
mail_server.ehlo()
except (socket.error, OSError) as e:
raise PluginWarning(str(e))
try:
if config.get('smtp_username'):
# Forcing to use `str` type
log.debug('logging in to smtp server using username: %s',
config['smtp_username'])
mail_server.login(text_to_native_str(config['smtp_username']),
text_to_native_str(config['smtp_password']))
mail_server.sendmail(email['From'], config['to'],
email.as_string())
except IOError as e:
raise PluginWarning(str(e))
mail_server.quit()
class PluginTransmission(TransmissionBase):
"""
Add url from entry url to transmission
Example::
transmission:
host: localhost
port: 9091
netrc: /home/flexget/.tmnetrc
username: myusername
password: mypassword
path: the download location
Default values for the config elements::
transmission:
host: localhost
port: 9091
enabled: yes
"""
schema = {
'anyOf': [
{'type': 'boolean'},
{
'type': 'object',
'properties': {
'host': {'type': 'string'},
'port': {'type': 'integer'},
'netrc': {'type': 'string'},
'username': {'type': 'string'},
'password': {'type': 'string'},
'action': {
'type': 'string',
'enum': ['add', 'remove', 'purge', 'pause', 'resume'],
},
'path': {'type': 'string'},
'max_up_speed': {'type': 'number'},
'max_down_speed': {'type': 'number'},
'max_connections': {'type': 'integer'},
'ratio': {'type': 'number'},
'add_paused': {'type': 'boolean'},
'content_filename': {'type': 'string'},
'main_file_only': {'type': 'boolean'},
'main_file_ratio': {'type': 'number'},
'magnetization_timeout': {'type': 'integer'},
'enabled': {'type': 'boolean'},
'include_subs': {'type': 'boolean'},
'bandwidth_priority': {'type': 'number'},
'honor_limits': {'type': 'boolean'},
'include_files': one_or_more({'type': 'string'}),
'skip_files': one_or_more({'type': 'string'}),
'rename_like_files': {'type': 'boolean'},
'queue_position': {'type': 'integer'},
},
'additionalProperties': False,
},
]
}
def prepare_config(self, config):
config = TransmissionBase.prepare_config(self, config)
config.setdefault('action', 'add')
config.setdefault('path', '')
config.setdefault('main_file_only', False)
config.setdefault('magnetization_timeout', 0)
config.setdefault('include_subs', False)
config.setdefault('rename_like_files', False)
config.setdefault('include_files', [])
return config
@plugin.priority(120)
def on_task_download(self, task, config):
"""
Call download plugin to generate the temp files we will load
into deluge then verify they are valid torrents
"""
config = self.prepare_config(config)
if not config['enabled']:
return
# If the download plugin is not enabled, we need to call it to get our temp .torrent files
if 'download' not in task.config:
download = plugin.get('download', self)
for entry in task.accepted:
if entry.get('transmission_id'):
# The torrent is already loaded in deluge, we don't need to get anything
continue
if config['action'] != 'add' and entry.get('torrent_info_hash'):
# If we aren't adding the torrent new, all we need is info hash
continue
download.get_temp_file(task, entry, handle_magnets=True, fail_html=True)
@plugin.priority(135)
def on_task_output(self, task, config):
config = self.prepare_config(config)
# don't add when learning
if task.options.learn:
return
if not config['enabled']:
return
# Do not run if there is nothing to do
if not task.accepted:
return
if self.client is None:
self.client = self.create_rpc_client(config)
if self.client:
log.debug('Successfully connected to transmission.')
else:
raise plugin.PluginError("Couldn't connect to transmission.")
session_torrents = self.client.get_torrents()
for entry in task.accepted:
if task.options.test:
log.info('Would %s %s in transmission.', config['action'], entry['title'])
continue
# Compile user options into appropriate dict
options = self._make_torrent_options_dict(config, entry)
torrent_info = None
for t in session_torrents:
if t.hashString.lower() == entry.get(
'torrent_info_hash', ''
).lower() or t.id == entry.get('transmission_id'):
torrent_info = t
log.debug(
'Found %s already loaded in transmission as %s',
entry['title'],
torrent_info.name,
)
break
if not torrent_info:
if config['action'] != 'add':
log.warning(
'Cannot %s %s because it is not loaded in transmission.',
config['action'],
entry['title'],
)
continue
downloaded = not entry['url'].startswith('magnet:')
# Check that file is downloaded
if downloaded and 'file' not in entry:
entry.fail('`file` field missing?')
continue
# Verify the temp file exists
if downloaded and not os.path.exists(entry['file']):
tmp_path = os.path.join(task.manager.config_base, 'temp')
log.debug('entry: %s', entry)
log.debug('temp: %s', ', '.join(os.listdir(tmp_path)))
entry.fail("Downloaded temp file '%s' doesn't exist!?" % entry['file'])
continue
try:
if downloaded:
with open(entry['file'], 'rb') as f:
filedump = base64.b64encode(f.read()).decode('utf-8')
torrent_info = self.client.add_torrent(filedump, 30, **options['add'])
else:
# we need to set paused to false so the magnetization begins immediately
options['add']['paused'] = False
torrent_info = self.client.add_torrent(
entry['url'], timeout=30, **options['add']
)
except TransmissionError as e:
log.debug('TransmissionError', exc_info=True)
log.debug('Failed options dict: %s', options['add'])
msg = 'Error adding {} to transmission. TransmissionError: {}'.format(
entry['title'], e.message or 'N/A'
)
log.error(msg)
entry.fail(msg)
continue
log.info('"%s" torrent added to transmission', entry['title'])
# The info returned by the add call is incomplete, refresh it
torrent_info = self.client.get_torrent(torrent_info.id)
try:
total_size = torrent_info.totalSize
main_id = None
find_main_file = (
options['post'].get('main_file_only') or 'content_filename' in options['post']
)
skip_files = options['post'].get('skip_files')
# We need to index the files if any of the following are defined
if find_main_file or skip_files:
file_list = self.client.get_files(torrent_info.id)[torrent_info.id]
if options['post'].get('magnetization_timeout', 0) > 0 and not file_list:
log.debug(
'Waiting %d seconds for "%s" to magnetize',
options['post']['magnetization_timeout'],
entry['title'],
)
for _ in range(options['post']['magnetization_timeout']):
sleep(1)
file_list = self.client.get_files(torrent_info.id)[torrent_info.id]
if file_list:
total_size = self.client.get_torrent(
torrent_info.id, ['id', 'totalSize']
).totalSize
break
else:
log.warning(
'"%s" did not magnetize before the timeout elapsed, '
'file list unavailable for processing.',
entry['title'],
)
# Find files based on config
dl_list = []
skip_list = []
main_list = []
ext_list = ['*.srt', '*.sub', '*.idx', '*.ssa', '*.ass']
main_ratio = config['main_file_ratio']
if 'main_file_ratio' in options['post']:
main_ratio = options['post']['main_file_ratio']
for f in file_list:
# No need to set main_id if we're not going to need it
if find_main_file and file_list[f]['size'] > total_size * main_ratio:
main_id = f
if 'include_files' in options['post']:
if any(
fnmatch(file_list[f]['name'], mask)
for mask in options['post']['include_files']
):
dl_list.append(f)
elif options['post'].get('include_subs') and any(
fnmatch(file_list[f]['name'], mask) for mask in ext_list
):
dl_list.append(f)
if skip_files:
if any(fnmatch(file_list[f]['name'], mask) for mask in skip_files):
skip_list.append(f)
if main_id is not None:
# Look for files matching main ID title but with a different extension
if options['post'].get('rename_like_files'):
for f in file_list:
# if this filename matches main filename we want to rename it as well
fs = os.path.splitext(file_list[f]['name'])
if fs[0] == os.path.splitext(file_list[main_id]['name'])[0]:
main_list.append(f)
else:
main_list = [main_id]
if main_id not in dl_list:
dl_list.append(main_id)
elif find_main_file:
log.warning(
'No files in "%s" are > %d%% of content size, no files renamed.',
entry['title'],
main_ratio * 100,
)
# If we have a main file and want to rename it and associated files
if 'content_filename' in options['post'] and main_id is not None:
if 'download_dir' not in options['add']:
download_dir = self.client.get_session().download_dir
else:
download_dir = options['add']['download_dir']
# Get new filename without ext
file_ext = os.path.splitext(file_list[main_id]['name'])[1]
file_path = os.path.dirname(
os.path.join(download_dir, file_list[main_id]['name'])
)
filename = options['post']['content_filename']
if config['host'] == 'localhost' or config['host'] == '127.0.0.1':
counter = 1
while os.path.exists(os.path.join(file_path, filename + file_ext)):
# Try appending a (#) suffix till a unique filename is found
filename = '%s(%s)' % (
options['post']['content_filename'],
counter,
)
counter += 1
else:
log.debug(
'Cannot ensure content_filename is unique '
'when adding to a remote transmission daemon.'
)
for index in main_list:
file_ext = os.path.splitext(file_list[index]['name'])[1]
log.debug(
'File %s renamed to %s'
% (file_list[index]['name'], filename + file_ext)
)
# change to below when set_files will allow setting name, more efficient to have one call
# fl[index]['name'] = os.path.basename(pathscrub(filename + file_ext).encode('utf-8'))
try:
self.client.rename_torrent_path(
torrent_info.id,
file_list[index]['name'],
os.path.basename(str(pathscrub(filename + file_ext))),
)
except TransmissionError:
log.error('content_filename only supported with transmission 2.8+')
if options['post'].get('main_file_only') and main_id is not None:
# Set Unwanted Files
options['change']['files_unwanted'] = [
x for x in file_list if x not in dl_list
]
options['change']['files_wanted'] = dl_list
log.debug(
'Downloading %s of %s files in torrent.',
len(options['change']['files_wanted']),
len(file_list),
)
elif (
not options['post'].get('main_file_only') or main_id is None
) and skip_files:
# If no main file and we want to skip files
if len(skip_list) >= len(file_list):
log.debug(
'skip_files filter would cause no files to be downloaded; '
'including all files in torrent.'
)
else:
options['change']['files_unwanted'] = skip_list
options['change']['files_wanted'] = [
x for x in file_list if x not in skip_list
]
log.debug(
'Downloading %s of %s files in torrent.',
len(options['change']['files_wanted']),
len(file_list),
)
# Set any changed file properties
if list(options['change'].keys()):
self.client.change_torrent(torrent_info.id, 30, **options['change'])
if config['action'] == 'add':
# if add_paused was defined and set to False start the torrent;
# prevents downloading data before we set what files we want
start_paused = (
options['post']['paused']
if 'paused' in options['post']
else not self.client.get_session().start_added_torrents
)
if start_paused:
self.client.stop_torrent(torrent_info.id)
else:
self.client.start_torrent(torrent_info.id)
elif config['action'] in ('remove', 'purge'):
self.client.remove_torrent(
[torrent_info.id], delete_data=config['action'] == 'purge'
)
log.info('%sd %s from transmission', config['action'], torrent_info.name)
elif config['action'] == 'pause':
self.client.stop_torrent([torrent_info.id])
log.info('paused %s in transmission', torrent_info.name)
elif config['action'] == 'resume':
self.client.start_torrent([torrent_info.id])
log.info('resumed %s in transmission', torrent_info.name)
except TransmissionError as e:
log.debug('TransmissionError', exc_info=True)
log.debug('Failed options dict: %s', options)
msg = 'Error trying to {} {}, TransmissionError: {}'.format(
config['action'], entry['title'], e.message or 'N/A'
)
log.error(msg)
continue
def _make_torrent_options_dict(self, config, entry):
opt_dic = {}
for opt_key in (
'path',
'add_paused',
'honor_limits',
'bandwidth_priority',
'max_connections',
'max_up_speed',
'max_down_speed',
'ratio',
'main_file_only',
'main_file_ratio',
'magnetization_timeout',
'include_subs',
'content_filename',
'include_files',
'skip_files',
'rename_like_files',
'queue_position',
):
# Values do not merge config with task
# Task takes priority then config is used
if opt_key in entry:
opt_dic[opt_key] = entry[opt_key]
elif opt_key in config:
opt_dic[opt_key] = config[opt_key]
options = {'add': {}, 'change': {}, 'post': {}}
add = options['add']
if opt_dic.get('path'):
try:
path = os.path.expanduser(entry.render(opt_dic['path']))
except RenderError as e:
log.error('Error setting path for %s: %s' % (entry['title'], e))
else:
# Transmission doesn't like it when paths end in a separator
path = path.rstrip('\\/')
add['download_dir'] = text_to_native_str(pathscrub(path), 'utf-8')
# make sure we add it paused, will modify status after adding
add['paused'] = True
change = options['change']
if 'bandwidth_priority' in opt_dic:
change['bandwidthPriority'] = opt_dic['bandwidth_priority']
if 'honor_limits' in opt_dic and not opt_dic['honor_limits']:
change['honorsSessionLimits'] = False
if 'max_up_speed' in opt_dic:
change['uploadLimit'] = opt_dic['max_up_speed']
change['uploadLimited'] = True
if 'max_down_speed' in opt_dic:
change['downloadLimit'] = opt_dic['max_down_speed']
change['downloadLimited'] = True
if 'max_connections' in opt_dic:
change['peer_limit'] = opt_dic['max_connections']
if 'ratio' in opt_dic:
change['seedRatioLimit'] = opt_dic['ratio']
if opt_dic['ratio'] == -1:
# seedRatioMode:
# 0 follow the global settings
# 1 override the global settings, seeding until a certain ratio
# 2 override the global settings, seeding regardless of ratio
change['seedRatioMode'] = 2
else:
change['seedRatioMode'] = 1
if 'queue_position' in opt_dic:
change['queuePosition'] = opt_dic['queue_position']
post = options['post']
# set to modify paused status after
if 'add_paused' in opt_dic:
post['paused'] = opt_dic['add_paused']
if 'main_file_only' in opt_dic:
post['main_file_only'] = opt_dic['main_file_only']
if 'main_file_ratio' in opt_dic:
post['main_file_ratio'] = opt_dic['main_file_ratio']
if 'magnetization_timeout' in opt_dic:
post['magnetization_timeout'] = opt_dic['magnetization_timeout']
if 'include_subs' in opt_dic:
post['include_subs'] = opt_dic['include_subs']
if 'content_filename' in opt_dic:
try:
post['content_filename'] = entry.render(opt_dic['content_filename'])
except RenderError as e:
log.error('Unable to render content_filename %s: %s' % (entry['title'], e))
if 'skip_files' in opt_dic:
post['skip_files'] = opt_dic['skip_files']
if not isinstance(post['skip_files'], list):
post['skip_files'] = [post['skip_files']]
if 'include_files' in opt_dic:
post['include_files'] = opt_dic['include_files']
if not isinstance(post['include_files'], list):
post['include_files'] = [post['include_files']]
if 'rename_like_files' in opt_dic:
post['rename_like_files'] = opt_dic['rename_like_files']
return options
def on_task_learn(self, task, config):
""" Make sure all temp files are cleaned up when entries are learned """
# If download plugin is enabled, it will handle cleanup.
if 'download' not in task.config:
download = plugin.get('download', self)
download.cleanup_temp_files(task)
on_task_abort = on_task_learn
class Unique(object):
"""
Take action on entries with duplicate fields, except for the first item
Reject the second+ instance of every movie:
unique:
field:
- imdb_id
- movie_name
action: reject
"""
schema = {
'type': 'object',
'properties': {
'field': one_or_more({'type': 'string'}),
'action': {
'enum': ['accept', 'reject']
},
},
'required': ['field'],
'additionalProperties': False,
}
def prepare_config(self, config):
if isinstance(config, bool) or config is None:
config = {}
config.setdefault('action', 'reject')
if not isinstance(config['field'], list):
config['field'] = [config['field']]
return config
def extract_fields(self, entry, field_names):
return [entry[field] for field in field_names]
def should_ignore(self, item, action):
return item.accepted and action == 'accept' or item.rejected and action == 'reject'
def on_task_filter(self, task, config):
config = self.prepare_config(config)
field_names = config['field']
entries = list(task.entries)
for i, entry in enumerate(entries):
# Ignore already processed entries
if self.should_ignore(entry, config['action']):
continue
try:
entry_fields = self.extract_fields(entry, field_names)
# Ignore if a field is missing
except KeyError:
continue
# Iterate over next items, try to find a similar item
for prospect in entries[i + 1:]:
# Ignore processed prospects
if self.should_ignore(prospect, config['action']):
continue
try:
prospect_fields = self.extract_fields(
prospect, field_names)
# Ignore if a field is missing
except KeyError:
continue
if entry_fields and entry_fields == prospect_fields:
msg = 'Field {} value {} equals on {} and {}'.format(
field_names, entry_fields, entry['title'],
prospect['title'])
# Mark prospect
if config['action'] == 'accept':
prospect.accept(msg)
else:
prospect.reject(msg)
class SftpList:
"""
Generate entries from SFTP. This plugin requires the pysftp Python module and its dependencies.
Configuration:
host: Host to connect to.
port: Port the remote SSH server is listening on (default 22).
username: Username to log in as.
password: The password to use. Optional if a private key is provided.
private_key: Path to the private key (if any) to log into the SSH server.
private_key_pass: Password for the private key (if needed).
recursive: Indicates whether the listing should be recursive.
get_size: Indicates whetern to calculate the size of the remote file/directory.
WARNING: This can be very slow when computing the size of directories!
files_only: Indicates wheter to omit diredtories from the results.
dirs_only: Indicates whether to omit files from the results.
dirs: List of directories to download.
socket_timeout_sec: Socket timeout in seconds (default 15 seconds).
connection_tries: Number of times to attempt to connect before failing (default 3).
Example:
sftp_list:
host: example.com
username: Username
private_key: /Users/username/.ssh/id_rsa
recursive: False
get_size: True
files_only: False
dirs:
- '/path/to/list/'
- '/another/path/'
"""
schema = {
'type': 'object',
'properties': {
'host': {
'type': 'string'
},
'username': {
'type': 'string'
},
'password': {
'type': 'string'
},
'port': {
'type': 'integer',
'default': DEFAULT_SFTP_PORT
},
'files_only': {
'type': 'boolean',
'default': True
},
'dirs_only': {
'type': 'boolean',
'default': False
},
'recursive': {
'type': 'boolean',
'default': False
},
'get_size': {
'type': 'boolean',
'default': True
},
'private_key': {
'type': 'string'
},
'private_key_pass': {
'type': 'string'
},
'dirs': one_or_more({'type': 'string'}),
'socket_timeout_sec': {
'type': 'integer',
'default': DEFAULT_SOCKET_TIMEOUT_SEC
},
'connection_tries': {
'type': 'integer',
'default': DEFAULT_CONNECT_TRIES
},
},
'additionProperties': False,
'required': ['host', 'username'],
}
@staticmethod
def prepare_config(config: dict) -> dict:
"""
Sets defaults for the provided configuration
"""
config.setdefault('password', None)
config.setdefault('private_key', None)
config.setdefault('private_key_pass', None)
config.setdefault('dirs', ['.'])
return config
@classmethod
def on_task_input(cls, task: Task, config: dict) -> List[Entry]:
"""
Input task handler
"""
config = cls.prepare_config(config)
files_only: bool = config['files_only']
dirs_only: bool = config['dirs_only']
recursive: bool = config['recursive']
get_size: bool = config['get_size']
socket_timeout_sec: int = config['socket_timeout_sec']
connection_tries: int = config['connection_tries']
directories: List[str] = []
if files_only and dirs_only:
logger.warning(
"Both files_only and dirs_only are set. This will result in no entries being discovered."
)
if isinstance(config['dirs'], list):
directories.extend(config['dirs'])
else:
directories.append(config['dirs'])
sftp_config: SftpConfig = task_config_to_sftp_config(config)
sftp: SftpClient = sftp_connect(sftp_config, socket_timeout_sec,
connection_tries)
entries: List[Entry] = sftp.list_directories(directories, recursive,
get_size, files_only,
dirs_only)
sftp.close()
return entries
class SearchRarBG(object):
"""
RarBG search plugin.
To perform search against single category:
rarbg:
category: x264 720p
To perform search against multiple categories:
rarbg:
category:
- x264 720p
- x264 1080p
Movie categories accepted: x264 720p, x264 1080p, XviD, Full BD
TV categories accepted: HDTV, SDTV
You can use also use category ID manually if you so desire (eg. x264 720p is actually category id '45')
"""
schema = {
'type': 'object',
'properties': {
'category': one_or_more({
'oneOf': [
{'type': 'integer'},
{'type': 'string', 'enum': list(CATEGORIES)},
]}),
'sorted_by': {'type': 'string', 'enum': ['seeders', 'leechers', 'last'], 'default': 'last'},
# min_seeders and min_leechers seem to be working again
'min_seeders': {'type': 'integer', 'default': 0},
'min_leechers': {'type': 'integer', 'default': 0},
'limit': {'type': 'integer', 'enum': [25, 50, 100], 'default': 25},
'ranked': {'type': 'boolean', 'default': True},
'use_tvdb': {'type': 'boolean', 'default': False},
},
"additionalProperties": False
}
base_url = 'https://torrentapi.org/pubapi_v2.php'
def get_token(self):
# Don't use a session as tokens are not affected by domain limit
try:
r = get(self.base_url, params={'get_token': 'get_token', 'format': 'json'}).json()
token = r.get('token')
log.debug('RarBG token: %s' % token)
return token
except RequestException as e:
log.debug('Could not retrieve RarBG token: %s', e.args[0])
@plugin.internet(log)
def search(self, task, entry, config):
"""
Search for entries on RarBG
"""
categories = config.get('category', 'all')
# Ensure categories a list
if not isinstance(categories, list):
categories = [categories]
# Convert named category to its respective category id number
categories = [c if isinstance(c, int) else CATEGORIES[c] for c in categories]
category_url_fragment = ';'.join(str(c) for c in categories)
entries = set()
token = self.get_token()
if not token:
log.error('Could not retrieve token. Abandoning search.')
return entries
params = {'mode': 'search', 'token': token, 'ranked': int(config['ranked']),
'min_seeders': config['min_seeders'], 'min_leechers': config['min_leechers'],
'sort': config['sorted_by'], 'category': category_url_fragment, 'format': 'json_extended',
'app_id': 'flexget'}
for search_string in entry.get('search_strings', [entry['title']]):
params.pop('search_string', None)
params.pop('search_imdb', None)
params.pop('search_tvdb', None)
if entry.get('movie_name'):
params['search_imdb'] = entry.get('imdb_id')
else:
query = normalize_scene(search_string)
query_url_fragment = query.encode('utf8')
params['search_string'] = query_url_fragment
if config['use_tvdb']:
plugin.get_plugin_by_name('thetvdb_lookup').instance.lazy_series_lookup(entry)
params['search_tvdb'] = entry.get('tvdb_id')
log.debug('Using tvdb id %s', entry.get('tvdb_id'))
try:
page = requests.get(self.base_url, params=params)
log.debug('requesting: %s', page.url)
except RequestException as e:
log.error('RarBG request failed: %s' % e.args[0])
continue
r = page.json()
# error code 20 just means no results were found
if r.get('error_code') == 20:
searched_string = params.get('search_string') or 'imdb={0}'.format(params.get('search_imdb')) or \
'tvdb={0}'.format(params.get('tvdb_id'))
log.debug('No results found for %s', searched_string)
continue
elif r.get('error'):
log.error('Error code %s: %s', r.get('error_code'), r.get('error'))
continue
else:
for result in r.get('torrent_results'):
e = Entry()
e['title'] = result.get('title')
e['url'] = result.get('download')
e['torrent_seeds'] = int(result.get('seeders'))
e['torrent_leeches'] = int(result.get('leechers'))
e['content_size'] = int(result.get('size')) / 1024 / 1024
episode_info = result.get('episode_info')
if episode_info:
e['imdb_id'] = episode_info.get('imdb')
e['tvdb_id'] = episode_info.get('tvdb')
e['tvrage_id'] = episode_info.get('tvrage')
entries.add(e)
return entries
class OutputPushover(object):
"""
Example::
pushover:
userkey: <USER_KEY> (can also be a list of userkeys)
apikey: <API_KEY>
[device: <DEVICE_STRING>] (default: (none))
[title: <MESSAGE_TITLE>] (default: "Download started" -- accepts Jinja2)
[message: <MESSAGE_BODY>] (default: "{{series_name}} {{series_id}}" -- accepts Jinja2)
[priority: <PRIORITY>] (default = 0 -- normal = 0, high = 1, silent = -1)
[url: <URL>] (default: "{{imdb_url}}" -- accepts Jinja2)
[urltitle: <URL_TITLE>] (default: (none) -- accepts Jinja2)
[sound: <SOUND>] (default: pushover default)
Configuration parameters are also supported from entries (eg. through set).
"""
default_message = "{% if series_name is defined %}{{tvdb_series_name|d(series_name)}} " \
"{{series_id}} {{tvdb_ep_name|d('')}}{% elif imdb_name is defined %}{{imdb_name}} "\
"{{imdb_year}}{% else %}{{title}}{% endif %}"
schema = {
'type': 'object',
'properties': {
'userkey': one_or_more({'type': 'string'}),
'apikey': {
'type': 'string'
},
'device': {
'type': 'string',
'default': ''
},
'title': {
'type': 'string',
'default': "{{task}}"
},
'message': {
'type': 'string',
'default': default_message
},
'priority': {
'type': 'integer',
'default': 0
},
'url': {
'type': 'string',
'default':
'{% if imdb_url is defined %}{{imdb_url}}{% endif %}'
},
'urltitle': {
'type': 'string',
'default': ''
},
'sound': {
'type': 'string',
'default': ''
}
},
'required': ['userkey', 'apikey'],
'additionalProperties': False
}
# Run last to make sure other outputs are successful before sending notification
@plugin.priority(0)
def on_task_output(self, task, config):
# Support for multiple userkeys
userkeys = config["userkey"]
if not isinstance(userkeys, list):
userkeys = [userkeys]
# Set a bunch of local variables from the config
apikey = config["apikey"]
device = config["device"]
priority = config["priority"]
sound = config["sound"]
# Loop through the provided entries
for entry in task.accepted:
title = config["title"]
message = config["message"]
url = config["url"]
urltitle = config["urltitle"]
# Attempt to render the title field
try:
title = entry.render(title)
except RenderError as e:
log.warning("Problem rendering 'title': %s" % e)
title = "Download started"
# Attempt to render the message field
try:
message = entry.render(message)
except RenderError as e:
log.warning("Problem rendering 'message': %s" % e)
message = entry["title"]
# Attempt to render the url field
try:
url = entry.render(url)
except RenderError as e:
log.warning("Problem rendering 'url': %s" % e)
url = entry.get("imdb_url", "")
# Attempt to render the urltitle field
try:
urltitle = entry.render(urltitle)
except RenderError as e:
log.warning("Problem rendering 'urltitle': %s" % e)
urltitle = ""
for userkey in userkeys:
# Build the request
data = {
"user": userkey,
"token": apikey,
"title": title,
"message": message,
"url": url,
"url_title": urltitle
}
if device:
data["device"] = device
if priority:
data["priority"] = priority
if sound:
data["sound"] = sound
# Check for test mode
if task.options.test:
log.info("Test mode. Pushover notification would be:")
if device:
log.info(" Device: %s" % device)
else:
log.info(" Device: [broadcast]")
log.info(" Title: %s" % title)
log.info(" Message: %s" % message)
log.info(" URL: %s" % url)
log.info(" URL Title: %s" % urltitle)
log.info(" Priority: %d" % priority)
log.info(" userkey: %s" % userkey)
log.info(" apikey: %s" % apikey)
log.info(" sound: %s" % sound)
# Test mode. Skip remainder.
continue
# Make the request
try:
response = task.requests.post(pushover_url,
data=data,
raise_status=False)
except RequestException as e:
log.warning(
'Could not get response from Pushover: {}'.format(e))
return
# Check if it succeeded
request_status = response.status_code
# error codes and messages from Pushover API
if request_status == 200:
log.debug("Pushover notification sent")
elif request_status == 500:
log.debug(
"Pushover notification failed, Pushover API having issues"
)
# TODO: Implement retrying. API requests 5 seconds between retries.
elif request_status >= 400:
errors = json.loads(response.content)['errors']
log.error("Pushover API error: %s" % errors[0])
else:
log.error(
"Unknown error when sending Pushover notification")
class SearchMoreThanTV(object):
"""
MorethanTV search plugin.
"""
schema = {
'type': 'object',
'properties': {
'username': {
'type': 'string'
},
'password': {
'type': 'string'
},
'category':
one_or_more({
'type': 'string',
'enum': list(CATEGORIES.keys())
},
unique_items=True),
'order_by': {
'type': 'string',
'enum': ['seeders', 'leechers', 'time'],
'default': 'time'
},
'order_way': {
'type': 'string',
'enum': ['desc', 'asc'],
'default': 'desc'
},
'tags':
one_or_more({
'type': 'string',
'enum': TAGS
}, unique_items=True),
'all_tags': {
'type': 'boolean',
'default': True
}
},
'required': ['username', 'password'],
'additionalProperties': False
}
base_url = 'https://www.morethan.tv/'
errors = False
def get(self, url, params, username, password, force=False):
"""
Wrapper to allow refreshing the cookie if it is invalid for some reason
:param str url:
:param list params:
:param str username:
:param str password:
:param bool force: flag used to refresh the cookie forcefully ie. forgo DB lookup
:return:
"""
cookies = self.get_login_cookie(username, password, force=force)
response = requests.get(url, params=params, cookies=cookies)
if self.base_url + 'login.php' in response.url:
if self.errors:
raise plugin.PluginError(
'MoreThanTV login cookie is invalid. Login page received?')
self.errors = True
# try again
response = self.get(url, params, username, password, force=True)
else:
self.errors = False
return response
def get_login_cookie(self, username, password, force=False):
"""
Retrieves login cookie
:param str username:
:param str password:
:param bool force: if True, then retrieve a fresh cookie instead of looking in the DB
:return:
"""
if not force:
with Session() as session:
saved_cookie = session.query(MoreThanTVCookie).filter(
MoreThanTVCookie.username == username).first()
if saved_cookie and saved_cookie.expires and saved_cookie.expires >= datetime.datetime.now(
):
log.debug('Found valid login cookie')
return saved_cookie.cookie
url = self.base_url + 'login.php'
try:
log.debug('Attempting to retrieve MoreThanTV cookie')
response = requests.post(url,
data={
'username': username,
'password': password,
'login': '******',
'keeplogged': '1'
},
timeout=30)
except RequestException as e:
raise plugin.PluginError('MoreThanTV login failed: %s' % e)
if 'Your username or password was incorrect.' in response.text:
raise plugin.PluginError(
'MoreThanTV login failed: Your username or password was incorrect.'
)
with Session() as session:
expires = None
for c in requests.cookies:
if c.name == 'session':
expires = c.expires
if expires:
expires = datetime.datetime.fromtimestamp(expires)
log.debug('Saving or updating MoreThanTV cookie in db')
cookie = MoreThanTVCookie(username=username,
cookie=dict(requests.cookies),
expires=expires)
session.merge(cookie)
return cookie.cookie
@plugin.internet(log)
def search(self, task, entry, config):
"""
Search for entries on MoreThanTV
"""
params = {}
if 'category' in config:
categories = config['category'] if isinstance(
config['category'], list) else [config['category']]
for category in categories:
params[CATEGORIES[category]] = 1
if 'tags' in config:
tags = config['tags'] if isinstance(config['tags'],
list) else [config['tags']]
tags = ', '.join(tags)
params['taglist'] = tags
entries = set()
params.update({
'tags_type': int(config['all_tags']),
'order_by': config['order_by'],
'search_submit': 1,
'order_way': config['order_way'],
'action': 'basic',
'group_results': 0
})
for search_string in entry.get('search_strings', [entry['title']]):
params['searchstr'] = search_string.replace("'", "")
log.debug('Using search params: %s', params)
try:
page = self.get(self.base_url + 'torrents.php', params,
config['username'], config['password'])
log.debug('requesting: %s', page.url)
except RequestException as e:
log.error('MoreThanTV request failed: %s', e)
continue
soup = get_soup(page.content)
for result in soup.findAll('tr', attrs={'class': 'torrent'}):
group_info = result.find('td', attrs={
'class': 'big_info'
}).find('div', attrs={'class': 'group_info'})
title = group_info.find(
'a', href=re.compile('torrents.php\?id=\d+')).text
url = self.base_url + group_info.find(
'a',
href=re.compile('torrents.php\?action=download'))['href']
torrent_info = result.findAll('td',
attrs={'class': 'number_column'})
size = re.search('(\d+(?:[.,]\d+)*)\s?([KMG]B)',
torrent_info[0].text)
torrent_tags = ', '.join([
tag.text
for tag in group_info.findAll('div',
attrs={'class': 'tags'})
])
e = Entry()
e['title'] = title
e['url'] = url
e['torrent_snatches'] = int(torrent_info[1].text)
e['torrent_seeds'] = int(torrent_info[2].text)
e['torrent_leeches'] = int(torrent_info[3].text)
e['torrent_internal'] = True if group_info.find(
'span', attrs={'class': 'flag_internal'}) else False
e['torrent_fast_server'] = True if group_info.find(
'span', attrs={'class': 'flag_fast'}) else False
e['torrent_sticky'] = True if group_info.find(
'span', attrs={'class': 'flag_sticky'}) else False
e['torrent_tags'] = torrent_tags
e['content_size'] = parse_filesize(size.group(0))
entries.add(e)
return entries
class RegexExtract(object):
"""
Updates an entry with the values of regex matched named groups
Usage:
regex_extract:
field: <string>
regex:
- <regex>
[prefix]: <string>
Example:
regex_extract:
prefix: f1_
field: title
regex:
- Formula\.?1.(?P<location>*?)
"""
schema = {
'type': 'object',
'properties': {
'prefix': {
'type': 'string'
},
'field': {
'type': 'string'
},
'regex': one_or_more({
'type': 'string',
'format': 'regex'
}),
},
}
def on_task_start(self, task, config):
regex = config.get('regex')
if isinstance(regex, str):
regex = [regex]
self.regex_list = ReList(regex)
# Check the regex
try:
for _ in self.regex_list:
pass
except re.error as e:
raise plugin.PluginError('Error compiling regex: %s' % str(e))
def on_task_modify(self, task, config):
prefix = config.get('prefix')
modified = 0
for entry in task.entries:
for rx in self.regex_list:
entry_field = entry.get('title')
log.debug('Matching %s with regex: %s' % (entry_field, rx))
try:
match = rx.match(entry_field)
except re.error as e:
raise plugin.PluginError(
'Error encountered processing regex: %s' % str(e))
if match:
log.debug('Successfully matched %s' % entry_field)
data = match.groupdict()
if prefix:
for key in list(data.keys()):
data[prefix + key] = data[key]
del data[key]
log.debug('Values added to entry: %s' % data)
entry.update(data)
modified += 1
log.info('%d entries matched and modified' % modified)
"type": "string",
"pattern": "^([#&][^\x07\x2C\s]{0,200})",
"error_pattern": "channel name must start with # or & and contain no commas and whitespace",
}
schema = {
"oneOf": [
{
"type": "object",
"additionalProperties": {
"type": "object",
"properties": {
"tracker_file": {"type": "string"},
"server": {"type": "string"},
"port": {"type": "integer"},
"nickname": {"type": "string"},
"channels": one_or_more(channel_pattern),
"nickserv_password": {"type": "string"},
"invite_nickname": {"type": "string"},
"invite_message": {"type": "string"},
"task": one_or_more({"type": "string"}),
"task_re": {
"type": "object",
"additionalProperties": one_or_more(
{
"type": "object",
"properties": {"regexp": {"type": "string"}, "field": {"type": "string"}},
"required": ["regexp", "field"],
"additionalProperties": False,
}
),
},
class UrlRewriteTorrent411(object):
"""
torrent411 Urlrewriter and search Plugin.
---
RSS (Two Options)
-- RSS DOWNLOAD WITH LOGIN
rss:
url: http://www.t411.in/rss/?cat=210
username: ****
password: ****
- OR -
-- RSS NORMAL URL REWRITE (i.e.: http://www.t411.in/torrents/download/?id=12345678)
-- WARNING: NEED CUSTOM COOKIES NOT HANDLE BY THIS PLUGIN
rss:
url: http://www.t411.in/rss/?cat=210
---
SEARCH WITHIN SITE
discover:
what:
- emit_movie_queue: yes
from:
- torrent411:
username: xxxxxxxx (required)
password: xxxxxxxx (required)
category: Film
sub_category: Multi-Francais
---
Category is one of these:
Animation, Animation-Serie, Concert, Documentaire, Emission-TV,
Film, Serie-TV, Series, Spectacle, Sport, Video-clips
---
Sub-Category is any combination of:
Anglais, VFF, Muet, Multi-Francais, Multi-Quebecois,
VFQ, VFSTFR, VOSTFR, VOASTA
BDrip-BRrip-SD, Bluray-4K, Bluray-Full-Remux, DVD-R-5,
DVD-R-9, DVDrip, HDrip-1080p, HDrip-720p, HDlight-1080p,
HDlight-720p, TVrip-SD, TVripHD-1080p, TVripHD-720p,
VCD-SVCD-VHSrip, WEBrip, WEBripHD-1080p, WEBripHD-1080p
2D, 3D-Converti-Amateur, 3D-Converti-Pro, 3D-Natif
"""
schema = {
'type': 'object',
'properties': {
'username': {
'type': 'string'
},
'password': {
'type': 'string'
},
'category': {
'type': 'string'
},
'sub_category':
one_or_more({
'type': 'string',
'enum': list(SUB_CATEGORIES)
}),
},
'deprecated':
'"torrent411" plugin has been replaced by the "t411" plugin.',
'required': ['username', 'password'],
'additionalProperties': False
}
# urlrewriter API
def url_rewritable(self, task, entry):
url = entry['url']
if re.match(
r'^(https?://)?(www\.)?t411\.in/torrents/(?!download/)[-A-Za-z0-9+&@#/%|?=~_|!:,.;]+',
url):
return True
return False
# urlrewriter API
def url_rewrite(self, task, entry):
if 'url' not in entry:
log.error("Didn't actually get a URL...")
else:
url = entry['url']
log.debug("Got the URL: %s" % entry['url'])
rawdata = ""
try:
opener = urllib2.build_opener()
opener.addheaders = [('User-agent', 'Mozilla/5.0')]
response = opener.open(url)
except Exception as e:
raise UrlRewritingError("Connection Error for %s : %s" %
(url, e))
rawdata = response.read()
match = re.search(
r"<a href=\"/torrents/download/\?id=(\d*?)\">.*\.torrent</a>",
rawdata)
if match:
torrent_id = match.group(1)
log.debug("Got the Torrent ID: %s" % torrent_id)
entry[
'url'] = 'http://www.t411.in/torrents/download/?id=' + torrent_id
if 'download_auth' in entry:
auth_handler = t411Auth(*entry['download_auth'])
entry['download_auth'] = auth_handler
else:
raise UrlRewritingError("Cannot find torrent ID")
@plugin.internet(log)
def search(self, task, entry, config=None):
"""
Search for name from torrent411.
"""
url_base = 'http://www.t411.in'
if not isinstance(config, dict):
config = {}
category = config.get('category')
if category in list(CATEGORIES):
category = CATEGORIES[category]
sub_categories = config.get('sub_category')
if not isinstance(sub_categories, list):
sub_categories = [sub_categories]
filter_url = ''
if isinstance(category, int):
filter_url = '&cat=%s' % str(category)
if sub_categories[0] is not None:
sub_categories = [SUB_CATEGORIES[c] for c in sub_categories]
filter_url = filter_url + '&' + '&'.join([
urllib.quote_plus('term[%s][]' % c[0]).encode('utf-8') +
'=' + str(c[1]) for c in sub_categories
])
entries = set()
for search_string in entry.get('search_strings', [entry['title']]):
query = normalize_unicode(search_string)
url_search = ('/torrents/search/?search=%40name+' +
urllib.quote_plus(query.encode('utf-8')) +
filter_url)
opener = urllib2.build_opener()
opener.addheaders = [('User-agent', 'Mozilla/5.0')]
response = opener.open(url_base + url_search)
data = response.read()
soup = get_soup(data)
tb = soup.find("table", class_="results")
if not tb:
continue
for tr in tb.findAll('tr')[1:][:-1]:
entry = Entry()
nfo_link_res = re.search('torrents/nfo/\?id=(\d+)', str(tr))
if nfo_link_res is not None:
tid = nfo_link_res.group(1)
title_res = re.search(
'<a href=\"//www.t411.in/torrents/([-A-Za-z0-9+&@#/%|?=~_|!:,.;]+)\" title="([^"]*)">',
str(tr))
if title_res is not None:
entry['title'] = title_res.group(2).decode('utf-8')
size = tr('td')[5].contents[0]
entry[
'url'] = 'http://www.t411.in/torrents/download/?id=%s' % tid
entry['torrent_seeds'] = tr('td')[7].contents[0]
entry['torrent_leeches'] = tr('td')[8].contents[0]
entry['search_sort'] = torrent_availability(
entry['torrent_seeds'], entry['torrent_leeches'])
size = re.search('([\.\d]+) ([GMK]?)B', size)
if size:
if size.group(2) == 'G':
entry['content_size'] = int(
float(size.group(1)) * 1000**3 / 1024**2)
elif size.group(2) == 'M':
entry['content_size'] = int(
float(size.group(1)) * 1000**2 / 1024**2)
elif size.group(2) == 'K':
entry['content_size'] = int(
float(size.group(1)) * 1000 / 1024**2)
else:
entry['content_size'] = int(
float(size.group(1)) / 1024**2)
auth_handler = t411Auth(config['username'], config['password'])
entry['download_auth'] = auth_handler
entries.add(entry)
return sorted(entries,
reverse=True,
key=lambda x: x.get('search_sort'))
class BaseFileOps(object):
# Defined by subclasses
log = None
along = {
'type': 'object',
'properties': {
'files': one_or_more({'type': 'string'}),
'subdirs': one_or_more({'type': 'string'})
},
'additionalProperties': False,
'required': ['files']
}
def on_task_output(self, task, config):
if config is True:
config = {}
elif config is False:
return
for entry in task.accepted:
if 'location' not in entry:
self.log.verbose(
'Cannot handle %s because it does not have the field location.'
% entry['title'])
continue
src = entry['location']
src_isdir = os.path.isdir(src)
try:
# check location
if not os.path.exists(src):
raise plugin.PluginWarning(
'location `%s` does not exists (anymore).' % src)
if src_isdir:
if not config.get('allow_dir'):
raise plugin.PluginWarning(
'location `%s` is a directory.' % src)
elif not os.path.isfile(src):
raise plugin.PluginWarning('location `%s` is not a file.' %
src)
# search for namesakes
siblings = {} # dict of (path=ext) pairs
if not src_isdir and 'along' in config:
parent = os.path.dirname(src)
filename_no_ext = os.path.splitext(
os.path.basename(src))[0]
subdirs = [parent] + config['along'].get('subdirs', [])
for subdir in subdirs:
if subdir == parent:
abs_subdirs = [subdir]
else:
# use glob to get a list of matching dirs
abs_subdirs = glob.glob(
os.path.join(parent, os.path.normpath(subdir)))
# iterate over every dir returned by glob looking for matching ext
for abs_subdir in abs_subdirs:
if os.path.isdir(abs_subdir):
for ext in config['along']['files']:
siblings.update(
get_siblings(ext, src, filename_no_ext,
abs_subdir))
# execute action in subclasses
self.handle_entry(task, config, entry, siblings)
except OSError as err:
entry.fail(str(err))
continue
def clean_source(self, task, config, entry):
min_size = entry.get('clean_source', config.get('clean_source', -1))
if min_size < 0:
return
base_path = os.path.split(entry.get('old_location',
entry['location']))[0]
# everything here happens after a successful execution of the main action: the entry has been moved in a
# different location, or it does not exists anymore. so from here we can just log warnings and move on.
if not os.path.isdir(base_path):
self.log.warning(
'Cannot delete path `%s` because it does not exists (anymore).'
% base_path)
return
dir_size = get_directory_size(base_path) / 1024 / 1024
if dir_size >= min_size:
self.log.info(
'Path `%s` left because it exceeds safety value set in clean_source option.'
% base_path)
return
if task.options.test:
self.log.info('Would delete `%s` and everything under it.' %
base_path)
return
try:
shutil.rmtree(base_path)
self.log.info(
'Path `%s` has been deleted because was less than clean_source safe value.'
% base_path)
except Exception as err:
self.log.warning('Unable to delete path `%s`: %s' %
(base_path, err))
def handle_entry(self, task, config, entry, siblings):
raise NotImplementedError()
class EmbyRefreshLibrary:
"""
Refresh Emby Library
Example:
emby_refresh:
server:
host: http://localhost:8096
username: <username>
apikey: <apikey>
return_host: wan
when: accepted
"""
auth = None
schema = {
'type': 'object',
'properties': {
**SCHEMA_SERVER_TAG,
'when':
one_or_more({
'type':
'string',
'enum': [
'accepted', 'rejected', 'failed', 'no_entries', 'aborted',
'always'
],
}),
},
'required': ['server'],
'additionalProperties': False,
}
def login(self, config):
if self.auth and self.auth.logged:
return
if not isinstance(config, dict):
config = {}
self.auth = EmbyAuth(**config)
self.auth.login(True)
def prepare_config(self, config):
config.setdefault('when', ['always'])
when = config['when']
if when and not isinstance(when, list):
config['when'] = [when]
return
def library_refresh(self):
EmbyApiLibrary.library_refresh(self.auth)
def on_task_start(self, task, config):
self.login(config)
@plugin.internet(logger)
def on_task_exit(self, task, config):
self.login(config)
self.prepare_config(config)
conditions = [
task.accepted and 'accepted' in config['when'],
task.rejected and 'rejected' in config['when'],
task.failed and 'failed' in config['when'],
not task.all_entries and 'no_entries' in config['when'],
'always' in config['when'],
]
if any(conditions):
self.library_refresh()
def on_task_abort(self, task, config):
self.prepare_config(config)
if 'aborted' in config['when']:
self.library_refresh()
class PushoverNotifier(object):
"""
Example::
pushover:
user_key: <USER_KEY> (can also be a list of userkeys)
token: <TOKEN>
[device: <DEVICE_STRING>]
[title: <MESSAGE_TITLE>]
[message: <MESSAGE_BODY>]
[priority: <PRIORITY>]
[url: <URL>]
[url_title: <URL_TITLE>]
[sound: <SOUND>]
[retry]: <RETRY>]
[expire]: <EXPIRE>]
[callback]: <CALLBACK>]
[html]: <HTML>]
"""
schema = {
'type': 'object',
'properties': {
'user_key': one_or_more({'type': 'string'}),
'api_key': {'type': 'string', 'default': 'aPwSHwkLcNaavShxktBpgJH4bRWc3m'},
'device': one_or_more({'type': 'string'}),
'priority': {'oneOf': [
{'type': 'number', 'minimum': -2, 'maximum': 2},
{'type': 'string'}]},
'url': {'type': 'string'},
'url_title': {'type': 'string'},
'sound': {'type': 'string'},
'retry': {'type': 'integer', 'minimum': 30},
'expire': {'type': 'integer', 'maximum': 86400},
'callback': {'type': 'string'},
'html': {'type': 'boolean'}
},
'required': ['user_key'],
'additionalProperties': False
}
def notify(self, title, message, config):
"""
Sends a Pushover notification
:param str title: the message's title
:param str message: the message to send
:param dict config: The pushover config
"""
notification = {'token': config.get('api_key'), 'message': message, 'title': title,
'device': config.get('device'), 'priority': config.get('priority'), 'url': config.get('url'),
'url_title': config.get('url_title'), 'sound': config.get('sound'),
'retry': config.get('retry'), 'expire': config.get('expire'),
'callback': config.get('callback')}
# HTML parsing mode
if config.get('html'):
notification['html'] = 1
# Support multiple devices
if isinstance(notification['device'], list):
notification['device'] = ','.join(notification['device'])
# Special case, verify certain fields exists if priority is 2
priority = config.get('priority')
expire = config.get('expire')
retry = config.get('retry')
if priority == 2 and not all([expire, retry]):
log.warning('Priority set to 2 but fields "expire" and "retry" are not both present.Lowering priority to 1')
notification['priority'] = 1
if not isinstance(config['user_key'], list):
config['user_key'] = [config['user_key']]
for user in config['user_key']:
notification['user'] = user
try:
response = requests.post(PUSHOVER_URL, data=notification)
except RequestException as e:
if e.response is not None:
if e.response.status_code == 429:
reset_time = datetime.datetime.fromtimestamp(
int(e.response.headers['X-Limit-App-Reset'])).strftime('%Y-%m-%d %H:%M:%S')
error_message = 'Monthly pushover message limit reached. Next reset: %s' % reset_time
else:
error_message = e.response.json()['errors'][0]
else:
error_message = str(e)
raise PluginWarning(error_message)
reset_time = datetime.datetime.fromtimestamp(
int(response.headers['X-Limit-App-Reset'])).strftime('%Y-%m-%d %H:%M:%S')
remaining = response.headers['X-Limit-App-Remaining']
log.debug('Pushover notification sent. Notifications remaining until next reset: %s. '
'Next reset at: %s', remaining, reset_time)
channel_pattern = {
'type': 'string', 'pattern': '^([#&][^\x07\x2C\s]{0,200})',
'error_pattern': 'channel name must start with # or & and contain no commas and whitespace'
}
schema = {
'oneOf': [
{
'type': 'object',
'additionalProperties': {
'type': 'object',
'properties': {
'tracker_file': {'type': 'string'},
'server': {'type': 'string'},
'port': {'type': 'integer'},
'nickname': {'type': 'string'},
'channels': one_or_more(channel_pattern),
'nickserv_password': {'type': 'string'},
'invite_nickname': {'type': 'string'},
'invite_message': {'type': 'string'},
'task': one_or_more({
'type': 'string'
}),
'task_re': {
'type': 'object',
'additionalProperties': one_or_more({
'type': 'object',
'properties': {
'regexp': {'type': 'string'},
'field': {'type': 'string'}
},
'required': ['regexp', 'field'],
class Filesystem:
"""
Uses local path content as an input. Can use recursion if configured.
Recursion is False by default. Can be configured to true or get integer that will specify max depth in relation to
base folder.
All files/dir/symlinks are retrieved by default. Can be changed by using the 'retrieve' property.
Example 1:: Single path
filesystem: /storage/movies/
Example 2:: List of paths
filesystem:
- /storage/movies/
- /storage/tv/
Example 3:: Object with list of paths
filesystem:
path:
- /storage/movies/
- /storage/tv/
mask: '*.mkv'
Example 4::
filesystem:
path:
- /storage/movies/
- /storage/tv/
recursive: 4 # 4 levels deep from each base folder
retrieve: files # Only files will be retrieved
Example 5::
filesystem:
path:
- /storage/movies/
- /storage/tv/
recursive: yes # No limit to depth, all sub dirs will be accessed
retrieve: # Only files and dirs will be retrieved
- files
- dirs
"""
retrieval_options = ['files', 'dirs', 'symlinks']
paths = one_or_more({
'type': 'string',
'format': 'path'
},
unique_items=True)
schema = {
'oneOf': [
paths,
{
'type': 'object',
'properties': {
'path':
paths,
'mask': {
'type': 'string'
},
'regexp': {
'type': 'string',
'format': 'regex'
},
'recursive': {
'oneOf': [{
'type': 'integer',
'minimum': 2
}, {
'type': 'boolean'
}]
},
'retrieve':
one_or_more({
'type': 'string',
'enum': retrieval_options
},
unique_items=True),
},
'required': ['path'],
'additionalProperties': False,
},
]
}
def prepare_config(self, config):
from fnmatch import translate
config = config
# Converts config to a dict with a list of paths
if not isinstance(config, dict):
config = {'path': config}
if not isinstance(config['path'], list):
config['path'] = [config['path']]
config.setdefault('recursive', False)
# If mask was specified, turn it in to a regexp
if config.get('mask'):
config['regexp'] = translate(config['mask'])
# If no mask or regexp specified, accept all files
config.setdefault('regexp', '.')
# Sets the default retrieval option to files
config.setdefault('retrieve', self.retrieval_options)
return config
def create_entry(self, filepath: Path, test_mode):
"""
Creates a single entry using a filepath and a type (file/dir)
"""
filepath = filepath.absolute()
entry = Entry()
entry['location'] = str(filepath)
entry['url'] = Path(filepath).absolute().as_uri()
entry['filename'] = filepath.name
if filepath.is_file():
entry['title'] = filepath.stem
else:
entry['title'] = filepath.name
file_stat = filepath.stat()
try:
entry['timestamp'] = datetime.fromtimestamp(file_stat.st_mtime)
except Exception as e:
logger.warning('Error setting timestamp for {}: {}', filepath, e)
entry['timestamp'] = None
entry['accessed'] = datetime.fromtimestamp(file_stat.st_atime)
entry['modified'] = datetime.fromtimestamp(file_stat.st_mtime)
entry['created'] = datetime.fromtimestamp(file_stat.st_ctime)
if entry.isvalid():
if test_mode:
logger.info("Test mode. Entry includes:")
logger.info(' Title: {}', entry['title'])
logger.info(' URL: {}', entry['url'])
logger.info(' Filename: {}', entry['filename'])
logger.info(' Location: {}', entry['location'])
logger.info(' Timestamp: {}', entry['timestamp'])
return entry
else:
logger.error('Non valid entry created: {} ', entry)
return
def get_max_depth(self, recursion, base_depth):
if recursion is False:
return base_depth + 1
elif recursion is True:
return float('inf')
else:
return base_depth + recursion
@staticmethod
def get_folder_objects(folder: Path, recursion: bool):
return folder.rglob('*') if recursion else folder.iterdir()
def get_entries_from_path(self, path_list, match, recursion, test_mode,
get_files, get_dirs, get_symlinks):
entries = []
for folder in path_list:
logger.verbose('Scanning folder {}. Recursion is set to {}.',
folder, recursion)
folder = Path(folder).expanduser()
if not folder.exists():
logger.error('{} does not exist (anymore.)', folder)
continue
logger.debug('Scanning {}', folder)
base_depth = len(folder.parts)
max_depth = self.get_max_depth(recursion, base_depth)
folder_objects = self.get_folder_objects(folder, recursion)
for path_object in folder_objects:
logger.debug(
'Checking if {} qualifies to be added as an entry.',
path_object)
try:
path_object.exists()
except UnicodeError:
logger.error(
'File {} not decodable with filesystem encoding: {}',
path_object,
sys.getfilesystemencoding(),
)
continue
entry = None
object_depth = len(path_object.parts)
if object_depth <= max_depth:
if match(str(path_object)):
if ((path_object.is_dir() and get_dirs)
or (path_object.is_symlink() and get_symlinks)
or
(path_object.is_file()
and not path_object.is_symlink() and get_files)):
entry = self.create_entry(path_object, test_mode)
else:
logger.debug(
"Path object's {} type doesn't match requested object types.",
path_object,
)
if entry and entry not in entries:
entries.append(entry)
return entries
def on_task_input(self, task, config):
config = self.prepare_config(config)
path_list = config['path']
test_mode = task.options.test
match = re.compile(config['regexp'], re.IGNORECASE).match
recursive = config['recursive']
get_files = 'files' in config['retrieve']
get_dirs = 'dirs' in config['retrieve']
get_symlinks = 'symlinks' in config['retrieve']
logger.verbose('Starting to scan folders.')
return self.get_entries_from_path(path_list, match, recursive,
test_mode, get_files, get_dirs,
get_symlinks)
