def create_rpc_client(self, config):
import transmissionrpc
from transmissionrpc import TransmissionError
from transmissionrpc import HTTPHandlerError
user, password = config.get('username'), config.get('password')
try:
cli = transmissionrpc.Client(config['host'], config['port'], user, password)
except TransmissionError as e:
if isinstance(e.original, HTTPHandlerError):
if e.original.code == 111:
raise PluginError("Cannot connect to transmission. Is it running?")
elif e.original.code == 401:
raise PluginError("Username/password for transmission is incorrect. Cannot connect.")
elif e.original.code == 110:
raise PluginError("Cannot connect to transmission: Connection timed out.")
else:
raise PluginError("Error connecting to transmission: %s" % e.original.message)
else:
raise PluginError("Error connecting to transmission: %s" % e.message)
return cli
def add_to_deluge11(self, task, config):
"""Add torrents to deluge using deluge 1.1.x api."""
try:
from deluge.ui.client import sclient
except:
raise PluginError('Deluge module required', log)
sclient.set_core_uri()
for entry in task.accepted:
try:
before = sclient.get_session_state()
except Exception, (errno, msg):
raise PluginError('Could not communicate with deluge core. %s' % msg, log)
if task.manager.options.test:
return
opts = {}
path = entry.get('path', config['path'])
if path:
try:
opts['download_location'] = os.path.expanduser(entry.render(path))
except RenderError, e:
log.error('Could not set path for %s: %s' % (entry['title'], e))
def get_token(self, refresh=False):
if refresh or not self.token:
try:
response = requests.get(
self.base_url,
params={'get_token': 'get_token', 'format': 'json', 'app_id': 'flexget'},
).json()
self.token = response.get('token')
logger.debug('RarBG token: {}', self.token)
except RequestException as e:
logger.opt(exception=True).debug('Could not retrieve RarBG token')
raise PluginError('Could not retrieve token: %s' % e)
return self.token
def authenticate(self):
"""Authenticates a session with imdb, and grabs any IDs needed for getting/modifying list."""
r = self._session.get(
'https://www.imdb.com/ap/signin?openid.return_to=https%3A%2F%2Fwww.imdb.com%2Fap-signin-'
'handler&openid.identity=http%3A%2F%2Fspecs.openid.net%2Fauth%2F2.0%2Fidentifier_select&'
'openid.assoc_handle=imdb_mobile_us&openid.mode=checkid_setup&openid.claimed_id=http%3A%'
'2F%2Fspecs.openid.net%2Fauth%2F2.0%2Fidentifier_select&openid.ns=http%3A%2F%2Fspecs.ope'
'nid.net%2Fauth%2F2.0')
soup = get_soup(r.content)
inputs = soup.select('form#ap_signin_form input')
data = dict(
(i['name'], i.get('value')) for i in inputs if i.get('name'))
data['email'] = self.config['login']
data['password'] = self.config['password']
d = self._session.post('https://www.imdb.com/ap/signin', data=data)
# Get user id by extracting from redirect url
r = self._session.head('http://www.imdb.com/profile',
allow_redirects=False)
if not r.headers.get('location') or 'login' in r.headers['location']:
raise plugin.PluginError(
'Login to imdb failed. Check your credentials.')
self.user_id = re.search('ur\d+(?!\d)', r.headers['location']).group()
# Get list ID
if self.config['list'] == 'watchlist':
data = {'consts[]': 'tt0133093', 'tracking_tag': 'watchlistRibbon'}
wl_data = self._session.post(
'http://www.imdb.com/list/_ajax/watchlist_has',
data=data).json()
try:
self.list_id = wl_data['list_id']
except KeyError:
raise PluginError(
'No list ID could be received. Please initialize list by '
'manually adding an item to it and try again')
elif self.config['list'] in IMMUTABLE_LISTS or self.config[
'list'].startswith('ls'):
self.list_id = self.config['list']
else:
data = {'tconst': 'tt0133093'}
list_data = self._session.post(
'http://www.imdb.com/list/_ajax/wlb_dropdown',
data=data).json()
for li in list_data['items']:
if li['wlb_text'] == self.config['list']:
self.list_id = li['data_list_id']
break
else:
raise plugin.PluginError('Could not find list %s' %
self.config['list'])
self._authenticated = True
def __setitem__(self, key, value):
# Enforce unicode compatibility. Check for all subclasses of basestring, so that NavigableStrings are also cast
if isinstance(value, basestring) and not type(value) == unicode:
try:
value = unicode(value)
except UnicodeDecodeError:
raise EntryUnicodeError(key, value)
# url and original_url handling
if key == 'url':
if not isinstance(value, basestring):
raise PluginError('Tried to set %r url to %r' %
(self.get('title'), value))
self.setdefault('original_url', value)
# title handling
if key == 'title':
if not isinstance(value, basestring):
raise PluginError('Tried to set title to %r' % value)
# TODO: HACK! Implement via plugin once #348 (entry events) is implemented
# enforces imdb_url in same format
if key == 'imdb_url' and isinstance(value, basestring):
imdb_id = extract_id(value)
if imdb_id:
value = make_url(imdb_id)
else:
log.debug('Tried to set imdb_id to invalid imdb url: %s' %
value)
value = None
try:
log.trace('ENTRY SET: %s = %r' % (key, value))
except Exception as e:
log.debug('trying to debug key `%s` value threw exception: %s' %
(key, e))
dict.__setitem__(self, key, value)
def on_task_filter(self, task, config):
config = self.prepare_config(config)
for item in config.get('lists'):
for plugin_name, plugin_config in item.items():
try:
thelist = plugin.get_plugin_by_name(
plugin_name).instance.get_list(plugin_config)
except AttributeError:
raise PluginError(
'Plugin %s does not support list interface' %
plugin_name)
for entry in task.entries:
if entry in thelist:
entry.accept()
def setup(manager):
if not 'email' in manager.config:
return
config = prepare_config(manager.config['email'])
config['global'] = True
global task_content
task_content = {}
for task in manager.tasks.itervalues():
task.config.setdefault('email', {})
try:
merge_dict_from_to(config, task.config['email'])
except MergeException, exc:
raise PluginError('Failed to merge email config to task %s due to %s' % (task.name, exc))
task.config.setdefault('email', config)
def extract_entry_from_soup(self, soup):
table = soup.find('div', {'id': 'main_table'})
if table is None:
raise PluginError(
'Could not fetch results table from Fuzer, aborting')
log.trace('fuzer results table: %s', table)
table = table.find('table', {'class': 'table_info'})
if len(table.find_all('tr')) == 1:
log.debug('No search results were returned from Fuzer, continuing')
return []
entries = []
for tr in table.find_all("tr"):
if not tr.get('class') or 'colhead_dark' in tr.get('class'):
continue
name = tr.find('div', {'class': 'main_title'}).find('a').text
torrent_name = re.search(
'\\n(.*)',
tr.find('div', {
'style': 'float: right;'
}).find('a')['title']).group(1)
attachment_link = tr.find('div', {
'style': 'float: right;'
}).find('a')['href']
attachment_id = re.search('attachmentid=(\d+)',
attachment_link).group(1)
raw_size = tr.find_all('td',
{'class': 'inline_info'})[0].text.strip()
seeders = int(tr.find_all('td', {'class': 'inline_info'})[2].text)
leechers = int(tr.find_all('td', {'class': 'inline_info'})[3].text)
e = Entry()
e['title'] = name
final_url = 'https://www.fuzer.me/rss/torrent.php/{}/{}/{}/{}'.format(
attachment_id, self.user_id, self.rss_key, torrent_name)
log.debug('RSS-ified download link: %s', final_url)
e['url'] = final_url
e['torrent_seeds'] = seeders
e['torrent_leeches'] = leechers
e['search_sort'] = torrent_availability(e['torrent_seeds'],
e['torrent_leeches'])
size = re.search('(\d+(?:[.,]\d+)*)\s?([KMGTP]B)', raw_size)
e['content_size'] = parse_filesize(size.group(0))
entries.append(e)
return entries
def items(self):
if self._items is None:
log.debug('fetching items from IMDB')
try:
r = self.session.get('http://www.imdb.com/list/export?list_id=%s&author_id=%s' %
(self.list_id, self.user_id), cookies=self.cookies)
except RequestException as e:
raise PluginError(e.args[0])
lines = r.iter_lines(decode_unicode=True)
# Throw away first line with headers
next(lines)
self._items = []
for row in csv_reader(lines):
log.debug('parsing line from csv: %s', ', '.join(row))
if not len(row) == 16:
log.debug('no movie row detected, skipping. %s', ', '.join(row))
continue
entry = Entry({
'title': '%s (%s)' % (row[5], row[11]) if row[11] != '????' else '%s' % row[5],
'url': row[15],
'imdb_id': row[1],
'imdb_url': row[15],
'imdb_list_position': int(row[0]),
'imdb_list_created': datetime.strptime(row[2], '%a %b %d %H:%M:%S %Y') if row[2] else None,
'imdb_list_modified': datetime.strptime(row[3], '%a %b %d %H:%M:%S %Y') if row[3] else None,
'imdb_list_description': row[4],
'imdb_name': row[5],
'imdb_year': int(row[11]) if row[11] != '????' else None,
'imdb_score': float(row[9]) if row[9] else None,
'imdb_user_score': float(row[8]) if row[8] else None,
'imdb_votes': int(row[13]) if row[13] else None,
'imdb_genres': [genre.strip() for genre in row[12].split(',')]
})
item_type = row[6].lower()
name = row[5]
year = int(row[11]) if row[11] != '????' else None
if item_type in MOVIE_TYPES:
entry['movie_name'] = name
entry['movie_year'] = year
elif item_type in SERIES_TYPES:
entry['series_name'] = name
entry['series_year'] = year
elif item_type in OTHER_TYPES:
entry['title'] = name
else:
log.verbose('Unknown IMDB type entry received: %s. Skipping', item_type)
continue
self._items.append(entry)
return self._items
def check_login(self, task, config):
url = config.get('api', self.DEFAULT_API)
if not self.session:
# Login
post = {
'username': config['username'],
'password': config['password']
}
result = query_api(url, "login", post)
response = json.loads(result.read())
if not response:
raise PluginError('Login failed', log)
self.session = response.replace('"', '')
else:
try:
query_api(url, 'getServerVersion', {'session': self.session})
except HTTPError, e:
if e.code == 403: # Forbidden
self.session = None
return self.check_login(task, config)
else:
raise PluginError('HTTP Error %s' % e, log)
def _request(self, action, page=None, **kwargs):
"""
Make an AJAX request to a given action page
Adapted from https://github.com/isaaczafuta/whatapi
"""
ajaxpage = "https://ssl.what.cd/ajax.php"
params = {}
# Filter params and map config values -> api values
for k, v in list(kwargs.items()):
params[self._key(k)] = self._getval(k, v)
# Params other than the searching ones
params['action'] = action
if page:
params['page'] = page
r = self.session.get(ajaxpage, params=params, allow_redirects=False)
if r.status_code != 200:
raise PluginError("What.cd returned a non-200 status code")
try:
json_response = r.json()
if json_response['status'] != "success":
# Try to deal with errors returned by the API
error = json_response.get('error', json_response.get('status'))
if not error or error == "failure":
error = json_response.get('response', str(json_response))
raise PluginError("What.cd gave a failure response: "
"'{}'".format(error))
return json_response['response']
except (ValueError, TypeError, KeyError) as e:
raise PluginError("What.cd returned an invalid response")
def update_base_url(self):
url = None
for mirror in MIRRORS:
try:
response = self.requests.get(mirror, timeout=2)
if response.ok:
url = mirror
break
except RequestException as err:
log.debug('Connection error. %s', str(err))
if url:
return url
else:
raise PluginError('Host unreachable.')
def on_task_learn(self, task, config):
if not config['remove_on_match']:
return
for item in config['from']:
for plugin_name, plugin_config in item.items():
try:
thelist = plugin.get_plugin_by_name(plugin_name).instance.get_list(plugin_config)
except AttributeError:
raise PluginError('Plugin %s does not support list interface' % plugin_name)
if task.manager.options.test and thelist.online:
log.info('`%s` is marked as online, would remove accepted items outside of --test mode.',
plugin_name)
continue
log.verbose('removing accepted entries from %s - %s', plugin_name, plugin_config)
thelist -= task.accepted
def request(self, no_login=False, **params):
"""Make an AJAX request to the API
If `no_login` is True, logging in will not be attempted if the request
is redirected to the login page.
Adapted from https://github.com/isaaczafuta/whatapi
"""
if 'action' not in params:
raise ValueError("An 'action' is required when making a request")
ajaxpage = "{}/ajax.php".format(self.base_url)
r = self._session.get(ajaxpage, params=params, allow_redirects=False, raise_status=True)
if not no_login and r.is_redirect and r.next.url == "{}/login.php".format(self.base_url):
logger.warning("Redirected to login page, reauthenticating and trying again")
self.authenticate(force=True)
return self.request(no_login=True, **params)
if r.status_code != 200:
raise PluginError("{} returned a non-200 status code".format(self.base_url))
try:
json_response = r.json()
if json_response['status'] != "success":
# Try to deal with errors returned by the API
error = json_response.get('error', json_response.get('status'))
if not error or error == "failure":
error = json_response.get('response', str(json_response))
raise PluginError(
"{} gave a failure response of '{}'".format(self.base_url, error)
)
return json_response['response']
except (ValueError, TypeError, KeyError):
raise PluginError("{} returned an invalid response".format(self.base_url))
def clear(self, task, config):
for item in config['what']:
for plugin_name, plugin_config in item.items():
try:
thelist = plugin.get_plugin_by_name(plugin_name).instance.get_list(plugin_config)
except AttributeError:
raise PluginError('Plugin %s does not support list interface' % plugin_name)
if thelist.immutable:
raise plugin.PluginError(thelist.immutable)
if config['phase'] == task.current_phase:
if task.manager.options.test and thelist.online:
log.info('would have cleared all items from %s - %s', plugin_name, plugin_config)
continue
log.verbose('clearing all items from %s - %s', plugin_name, plugin_config)
thelist.clear()
def __setitem__(self, key, value):
# Enforce unicode compatibility.
if PY2 and isinstance(value, native_str):
# Allow Python 2's implicit string decoding, but fail now instead of when entry fields are used.
# If encoding is anything but ascii, it should be decoded it to text before setting an entry field
try:
value = value.decode('ascii')
except UnicodeDecodeError:
raise EntryUnicodeError(key, value)
elif isinstance(value, bytes):
raise EntryUnicodeError(key, value)
# Coerce any enriched strings (such as those returned by BeautifulSoup) to plain strings to avoid serialization
# troubles.
elif isinstance(value, text_type) and type(value) != text_type: # pylint: disable=unidiomatic-typecheck
value = text_type(value)
# url and original_url handling
if key == 'url':
if not isinstance(value, (str, LazyLookup)):
raise PluginError('Tried to set %r url to %r' %
(self.get('title'), value))
self.setdefault('original_url', value)
# title handling
if key == 'title':
if not isinstance(value, (str, LazyLookup)):
raise PluginError('Tried to set title to %r' % value)
self.setdefault('original_title', value)
try:
log.trace('ENTRY SET: %s = %r' % (key, value))
except Exception as e:
log.debug('trying to debug key `%s` value threw exception: %s' %
(key, e))
super(Entry, self).__setitem__(key, value)
def set_credential(self, username=None, session=None):
"""
Set REST client credential from database
:param username: if set, account's credential will be used.
:return:
"""
query = session.query(Credential)
if username:
query = query.filter(Credential.username == username)
credential = query.first()
if credential is None:
raise PluginError('You cannot use t411 plugin without credentials. '
'Please set credential with "flexget t411 add-auth <username> <password>".')
self.__set_credential(credential.username, credential.password, credential.api_token)
def on_task_start(self, task, config):
url = config['url']
username = config['username']
password = config['password']
try:
response = task.requests.send(construct_request(url, username=username, password=password))
if not response.ok:
raise RequestException(str(response))
cookies = collect_cookies(response)
if len(get_valid_cookies(cookies)) < 1:
raise RequestException('No recognized WordPress cookies found. Perhaps username/password is invalid?')
task.requests.add_cookiejar(cookies)
except RequestException as err:
log.error('%s', err)
raise PluginError('WordPress Authentication at %s failed' % (url,))
def on_task_output(self, task, config):
for item in config:
for plugin_name, plugin_config in item.iteritems():
if task.manager.options.test:
log.info(
'Would remove accepted items from `%s` outside of --test mode.'
% plugin_name)
continue
try:
thelist = plugin.get_plugin_by_name(
plugin_name).instance.get_list(plugin_config)
except AttributeError:
raise PluginError(
'Plugin %s does not support list interface' %
plugin_name)
thelist -= task.accepted
def _get_access_token_n_update_db(self, session):
access_token, has_new_access_token = self._get_access_token(
session, self._corp_id, self._corp_secret)
logger.debug('access_token={}', access_token)
if not access_token:
raise PluginError('no access token found')
else:
if not access_token.access_token:
logger.warning(
'no access_token found for corp_id: {} and corp_secret: {}',
self._corp_id, self._corp_secret)
if has_new_access_token:
self._update_db(session, access_token)
return access_token
def _send_images(self, access_token):
media_id = self._get_media_id(access_token)
if media_id is None:
return
data = {
"touser": self._to_user,
"msgtype": "image",
"agentid": self._agent_id,
"image": {
"media_id": media_id
}
}
response_json = self._request('post', _POST_MESSAGE_URL.format(access_token=access_token.access_token),
json=data).json()
if response_json.get('errcode') != 0:
raise PluginError(response_json)
def on_task_filter(self, task, config):
for item in config:
for plugin_name, plugin_config in item.items():
try:
thelist = plugin.get_plugin_by_name(
plugin_name).instance.get_list(plugin_config)
except AttributeError:
raise PluginError(
'Plugin %s does not support list interface' %
plugin_name)
cached_items = []
for entry in task.entries:
result = thelist.get(entry)
if result and result not in cached_items:
entry.accept()
cached_items.append(result)
def get_template(templatename, pluginname=None):
"""Loads a template from disk. Looks in both included plugins and users custom plugin dir."""
if not templatename.endswith('.template'):
templatename += '.template'
locations = []
if pluginname:
locations.append(pluginname + '/' + templatename)
locations.append(templatename)
for location in locations:
try:
return environment.get_template(location)
except TemplateNotFound:
pass
else:
raise PluginError('Template not found: %s (%s)' %
(templatename, pluginname))
def execute_inputs(self, config, task):
"""
:param config: Discover config
:param task: Current task
:return: List of pseudo entries created by inputs under `what` configuration
"""
entries = []
entry_titles = set()
entry_urls = set()
# run inputs
for item in config['what']:
for input_name, input_config in item.items():
input = get_plugin_by_name(input_name)
if input.api_ver == 1:
raise PluginError('Plugin %s does not support API v2' %
input_name)
method = input.phase_handlers['input']
try:
result = method(task, input_config)
except PluginError as e:
log.warning('Error during input plugin %s: %s' %
(input_name, e))
continue
if not result:
log.warning('Input %s did not return anything' %
input_name)
continue
for entry in result:
urls = ([entry['url']] if entry.get('url') else
[]) + entry.get('urls', [])
if any(url in entry_urls for url in urls):
log.debug(
'URL for `%s` already in entry list, skipping.' %
entry['title'])
continue
if entry['title'] in entry_titles:
log.verbose('Ignored duplicate title `%s`' %
entry['title']) # TODO: should combine?
continue
entries.append(entry)
entry_titles.add(entry['title'])
entry_urls.update(urls)
return entries
def create_entries_from_query(self, url, task):
"""Fetch feed and fill entries from"""
logger.info('Fetching URL: {}', url)
try:
response = task.requests.get(url)
except RequestException as e:
raise PluginError("Failed fetching '{}': {}".format(url, e))
entries = []
root = ElementTree.fromstring(response.content)
for item in root.findall('.//item'):
entry = Entry()
enclosure = item.find(
"enclosure[@type='application/x-bittorrent']")
if enclosure is None:
logger.warning(
"Item '{}' does not contain a bittorent enclosure.",
item.title.string)
continue
else:
entry['url'] = enclosure.attrib['url']
try:
entry['content_size'] = int(
enclosure.attrib['length']) // (2**20)
except ValueError:
entry['content_size'] = 0
entry['type'] = enclosure.attrib['type']
ns = {'torznab': 'http://torznab.com/schemas/2015/feed'}
self._parse_torznab_attrs(entry, item.findall('torznab:attr', ns))
for child in item.iter():
if child.tag in [
'{http://torznab.com/schemas/2015/feed}attr',
'enclosure'
]:
continue
else:
if child.tag in ['description', 'title'] and child.text:
entry[child.tag] = child.text
entries.append(entry)
return entries
def on_task_filter(self, task, config):
fields = config['fields']
action = config['action']
match_entries = []
# TODO: xxx
# we probably want to have common "run and combine inputs" function sometime soon .. this code is in
# few places already (discover, inputs, ...)
# code written so that this can be done easily ...
for item in config['from']:
for input_name, input_config in item.iteritems():
input = get_plugin_by_name(input_name)
if input.api_ver == 1:
raise PluginError('Plugin %s does not support API v2' %
input_name)
method = input.phase_handlers['input']
try:
result = method(task, input_config)
except PluginError as e:
log.warning('Error during input plugin %s: %s' %
(input_name, e))
continue
if result:
match_entries.extend(result)
else:
log.warning('Input %s did not return anything' %
input_name)
continue
# perform action on intersecting entries
for entry in task.entries:
for generated_entry in match_entries:
log.trace('checking if %s matches %s' %
(entry['title'], generated_entry['title']))
common = self.entry_intersects(entry, generated_entry, fields)
if common:
msg = 'intersects with %s on field(s) %s' % \
(generated_entry['title'], ', '.join(common))
if action == 'reject':
entry.reject(msg)
if action == 'accept':
entry.accept(msg)
def get_user_id_and_hidden_value(self, cookies=None):
try:
if cookies:
self._session.cookies = cookiejar_from_dict(cookies)
# We need to allow for redirects here as it performs 1-2 redirects before reaching the real profile url
response = self._session.get('https://www.imdb.com/profile', allow_redirects=True)
except RequestException as e:
raise PluginError(str(e))
user_id_match = re.search('ur\d+(?!\d)', response.url)
if user_id_match:
# extract the hidden form value that we need to do post requests later on
try:
soup = get_soup(response.text)
self.hidden_value = soup.find('input', attrs={'id': '49e6c'})['value']
except Exception as e:
log.warning('Unable to locate the hidden form value ''49e6c''. Without it, you might not be able to '
'add or remove items. %s', e)
return user_id_match.group() if user_id_match else None
def on_task_filter(self, task, config):
for item in config:
for plugin_name, plugin_config in item.items():
try:
thelist = plugin.get_plugin_by_name(
plugin_name).instance.get_list(plugin_config)
except AttributeError:
raise PluginError(
'Plugin %s does not support list interface' %
plugin_name)
if task.manager.options.test and thelist.online:
log.info(
'`%s` is marked as online, would accept and remove items outside of --test mode.',
plugin_name)
continue
for entry in task.entries:
if entry in thelist:
entry.accept()
thelist.discard(entry)
def on_task_input(self, task, config):
entries = []
entry_titles = set()
entry_urls = set()
for item in config:
for input_name, input_config in item.iteritems():
input = get_plugin_by_name(input_name)
if input.api_ver == 1:
raise PluginError('Plugin %s does not support API v2' %
input_name)
method = input.phase_handlers['input']
try:
result = method(task, input_config)
except PluginError as e:
log.warning('Error during input plugin %s: %s' %
(input_name, e))
continue
if not result:
msg = 'Input %s did not return anything' % input_name
if getattr(task, 'no_entries_ok', False):
log.verbose(msg)
else:
log.warning(msg)
continue
for entry in result:
if entry['title'] in entry_titles:
log.debug(
'Title `%s` already in entry list, skipping.' %
entry['title'])
continue
urls = ([entry['url']] if entry.get('url') else
[]) + entry.get('urls', [])
if any(url in entry_urls for url in urls):
log.debug(
'URL for `%s` already in entry list, skipping.' %
entry['title'])
continue
entries.append(entry)
entry_titles.add(entry['title'])
entry_urls.update(urls)
return entries
def on_task_filter(self, task, config):
for item in config['from']:
for plugin_name, plugin_config in item.items():
try:
thelist = plugin.get_plugin_by_name(plugin_name).instance.get_list(plugin_config)
except AttributeError:
raise PluginError('Plugin %s does not support list interface' % plugin_name)
already_accepted = []
for entry in task.entries:
result = thelist.get(entry)
if not result:
continue
if config['action'] == 'accept':
if config['single_match']:
if result not in already_accepted:
already_accepted.append(result)
entry.accept()
else:
entry.accept()
elif config['action'] == 'reject':
entry.reject()
