from flexget.utils.soup import get_soup from flexget.utils.requests import Session from flexget.utils.tools import str_to_int log = logging.getLogger('utils.imdb') # IMDb delivers a version of the page which is unparsable to unknown (and some known) user agents, such as requests' # Spoof the old urllib user agent to keep results consistent requests = Session() requests.headers.update({'User-Agent': 'Python-urllib/2.6'}) #requests.headers.update({'User-Agent': random.choice(USERAGENTS)}) # this makes most of the titles to be returned in english translation, but not all of them requests.headers.update({'Accept-Language': 'en-US,en;q=0.8'}) # give imdb a little break between requests (see: http://flexget.com/ticket/129#comment:1) requests.set_domain_delay('imdb.com', '3 seconds') def is_imdb_url(url): """Tests the url to see if it's for imdb.com.""" if not isinstance(url, basestring): return # Probably should use urlparse. return re.match(r'https?://[^/]*imdb\.com/', url) def extract_id(url): """Return IMDb ID of the given URL. Return None if not valid or if URL is not a string.""" if not isinstance(url, basestring): return m = re.search(r'((?:nm|tt)[\d]{7})', url)
from flexget.utils.requests import Session from flexget.utils.tools import str_to_int log = logging.getLogger('utils.imdb') # IMDb delivers a version of the page which is unparsable to unknown (and some known) user agents, such as requests' # Spoof the old urllib user agent to keep results consistent requests = Session() requests.headers.update({'User-Agent': 'Python-urllib/2.6'}) #requests.headers.update({'User-Agent': random.choice(USERAGENTS)}) # this makes most of the titles to be returned in english translation, but not all of them requests.headers.update({'Accept-Language': 'en-US,en;q=0.8'}) # give imdb a little break between requests (see: http://flexget.com/ticket/129#comment:1) requests.set_domain_delay('imdb.com', '3 seconds') def is_imdb_url(url): """Tests the url to see if it's for imdb.com.""" if not isinstance(url, basestring): return # Probably should use urlparse. return re.match(r'https?://[^/]*imdb\.com/', url) def extract_id(url): """Return IMDb ID of the given URL. Return None if not valid or if URL is not a string.""" if not isinstance(url, basestring): return m = re.search(r'((?:nm|tt)[\d]{7})', url)
from __future__ import unicode_literals, division, absolute_import import logging import urllib from flexget import plugin from flexget.entry import Entry from flexget.event import event from flexget.config_schema import one_or_more from flexget.utils.requests import Session, get from flexget.utils.search import normalize_unicode log = logging.getLogger('rarbg') requests = Session() requests.set_domain_delay('torrentapi.org', '10.3 seconds') # they only allow 1 request per 10 seconds CATEGORIES = { 'all': 0, # Movies 'x264 720p': 45, 'x264 1080p': 44, 'XviD': 14, 'Full BD': 42, # TV 'HDTV': 41, 'SDTV': 18 }
import logging import re from flexget import plugin from flexget.entry import Entry from flexget.event import event from flexget.plugin import get_plugin_by_name from flexget.utils.cached_input import cached from flexget.utils.requests import RequestException, Session from flexget.utils.soup import get_soup log = logging.getLogger('letterboxd') logging.getLogger('api_tmdb').setLevel(logging.CRITICAL) requests = Session(max_retries=5) requests.set_domain_delay('letterboxd.com', '1 seconds') base_url = 'http://letterboxd.com' SLUGS = { 'default': { 'p_slug': '/%(user)s/list/%(list)s/', 'f_slug': 'data-film-slug' }, 'diary': { 'p_slug': '/%(user)s/films/diary/', 'f_slug': 'data-film-slug' }, 'likes': { 'p_slug': '/%(user)s/likes/films/', 'f_slug': 'data-film-link' },
class InputWhatCD(object): """A plugin that searches what.cd == Usage: All parameters except `username` and `password` are optional. whatcd: username: password: user_agent: (A custom user-agent for the client to report. It is NOT A GOOD IDEA to spoof a browser with this. You are responsible for your account.) search: (general search filter) artist: (artist name) album: (album name) year: (album year) encoding: (encoding specifics - 192, 320, lossless, etc.) format: (MP3, FLAC, AAC, etc.) media: (CD, DVD, vinyl, Blu-ray, etc.) release_type: (album, soundtrack, EP, etc.) log: (log specification - true, false, '100%', or '<100%') hascue: (has a cue file - true or false) scene: (is a scene release - true or false) vanityhouse: (is a vanity house release - true or false) leech_type: ('freeleech', 'neutral', 'either', or 'normal') tags: (a list of tags to match - drum.and.bass, new.age, blues, etc.) tag_type: (match 'any' or 'all' of the items in `tags`) """ # Aliases for config -> api params ALIASES = { "artist": "artistname", "album": "groupname", "leech_type": "freetorrent", "release_type": "releaseType", "tags": "taglist", "tag_type": "tags_type", "search": "searchstr", "log": "haslog", } # API parameters # None means a raw value entry (no validation) # A dict means a choice with a mapping for the API # A list is just a choice with no mapping PARAMS = { "searchstr": None, "taglist": None, "artistname": None, "groupname": None, "year": None, "tags_type": { "any": 0, "all": 1, }, "encoding": [ "192", "APS (VBR)", "V2 (VBR)", "V1 (VBR)", "256", "APX (VBR)", "V0 (VBR)", "320", "lossless", "24bit lossless", "V8 (VBR)" ], "format": [ "MP3", "FLAC", "AAC", "AC3", "DTS" ], "media": [ "CD", "DVD", "vinyl", "soundboard", "SACD", "DAT", "cassette", "WEB", "Blu-ray" ], "releaseType": { "album": 1, "soundtrack": 3, "EP": 5, "anthology": 6, "compilation": 7, "DJ mix": 8, "single": 9, "live album": 11, "remix": 13, "bootleg": 14, "interview": 15, "mixtape": 16, "unknown": 21, "concert recording": 22, "demo": 23 }, "haslog": { "False": 0, "True": 1, "100%": 100, "<100%": -1 }, "freetorrent": { "freeleech": 1, "neutral": 2, "either": 3, "normal": 0, }, "hascue": { "False": 0, "True": 1, }, "scene": { "False": 0, "True": 1, }, "vanityhouse": { "False": 0, "True": 1, } } def _key(self, key): """Gets the API key name from the entered key""" try: if key in self.ALIASES: return self.ALIASES[key] elif key in self.PARAMS: return key return None except KeyError: return None def _opts(self, key): """Gets the options for the specified key""" temp = self._key(key) try: return self.PARAMS[temp] except KeyError: return None def _getval(self, key, val): """Gets the value for the specified key""" # No alias or param by that name if self._key(key) is None: return None opts = self._opts(key) if opts is None: if isinstance(val, list): return ",".join(val) return val elif isinstance(opts, dict): # Options, translate the input to output # The str cast converts bools to 'True'/'False' for use as keys return opts[str(val)] else: # List of options, check it's in the list if val not in opts: return None return val def __init__(self): """Set up the schema""" self.schema = { 'type': 'object', 'properties': { 'username': {'type': 'string'}, 'password': {'type': 'string'}, 'user_agent': {'type': 'string'}, 'search': {'type': 'string'}, 'artist': {'type': 'string'}, 'album': {'type': 'string'}, 'year': {'type': ['string', 'integer']}, 'tags': one_or_more({'type': 'string'}), 'tag_type': {'type': 'string', 'enum': self._opts('tag_type').keys()}, 'encoding': {'type': 'string', 'enum': self._opts('encoding')}, 'format': {'type': 'string', 'enum': self._opts('format')}, 'media': {'type': 'string', 'enum': self._opts('media')}, 'release_type': {'type': 'string', 'enum': self._opts('release_type').keys()}, 'log': {'oneOf': [{'type': 'string', 'enum': self._opts('log').keys()}, {'type': 'boolean'}]}, 'leech_type': {'type': 'string', 'enum': self._opts('leech_type').keys()}, 'hascue': {'type': 'boolean'}, 'scene': {'type': 'boolean'}, 'vanityhouse': {'type': 'boolean'}, }, 'required': ['username', 'password'], 'additionalProperties': False } def _login(self, config): """ Log in and store auth data from the server Adapted from https://github.com/isaaczafuta/whatapi """ data = { 'username': config['username'], 'password': config['password'], 'keeplogged': 1, } r = self.session.post("https://ssl.what.cd/login.php", data=data, allow_redirects=False) if r.status_code != 302 or r.headers.get('location') != "index.php": raise PluginError("Failed to log in to What.cd") accountinfo = self._request("index") self.authkey = accountinfo["authkey"] self.passkey = accountinfo["passkey"] log.info("Logged in to What.cd") def _request(self, action, **kwargs): """ Make an AJAX request to a given action page Adapted from https://github.com/isaaczafuta/whatapi """ ajaxpage = 'https://ssl.what.cd/ajax.php' params = {} # Filter params and map config values -> api values for k, v in kwargs.iteritems(): key = self._key(k) if key is not None: params[key] = self._getval(k, v) # Params other than the searching ones params['action'] = action if 'page' in kwargs: params['page'] = kwargs['page'] r = self.session.get(ajaxpage, params=params, allow_redirects=False) if r.status_code != 200: raise PluginError("What.cd returned a non-200 status code") try: json_response = r.json() if json_response['status'] != "success": # Try to deal with errors returned by the API error = json_response.get('error', json_response.get('status')) if not error or error == "failure": error = json_response.get('response') if not error: error = str(json_response) raise PluginError("What.cd gave a failure response: " "'{0}'".format(error)) return json_response['response'] except (ValueError, TypeError, KeyError) as e: raise PluginError("What.cd returned an invalid response") @cached('whatcd') @plugin.internet(log) def on_task_input(self, task, config): """Search on What.cd""" self.session = Session() user_agent = config.get('user_agent') if user_agent: # Using a custom user agent self.session.headers.update({"User-Agent": user_agent}) # From the API docs: "Refrain from making more than five (5) requests every ten (10) seconds" self.session.set_domain_delay('ssl.what.cd', '2 seconds') # Login self._login(config) # Perform the query results = [] page = 1 while True: result = self._request("browse", page=page, **config) if not result['results']: break results.extend(result["results"]) pages = result['pages'] page = result['currentPage'] log.info("Got {0} of {1} pages".format(page, pages)) if page >= pages: break page += 1 # Logged in and made a request successfully, it's ok if nothing matches task.no_entries_ok = True # Parse the needed information out of the response entries = [] for result in results: # Get basic information on the release info = dict((k, result[k]) for k in ('artist', 'groupName', 'groupYear')) # Releases can have multiple download options for tor in result['torrents']: temp = info.copy() temp.update(dict((k, tor[k]) for k in ('media', 'encoding', 'format', 'torrentId'))) entries.append(Entry( title="{artist} - {groupName} - {groupYear} " "({media} - {format} - {encoding})-{torrentId}.torrent".format(**temp), url="https://what.cd/torrents.php?action=download&" "id={0}&authkey={1}&torrent_pass={2}".format(temp['torrentId'], self.authkey, self.passkey), torrent_seeds=tor['seeders'], torrent_leeches=tor['leechers'], # Size is given in bytes, convert it content_size=int(tor['size'] / (1024**2) * 100) / 100 )) return entries
import logging import re from flexget import plugin from flexget.entry import Entry from flexget.event import event from flexget.plugin import get_plugin_by_name from flexget.utils.cached_input import cached from flexget.utils.requests import RequestException, Session from flexget.utils.soup import get_soup log = logging.getLogger('letterboxd') logging.getLogger('api_tmdb').setLevel(logging.CRITICAL) requests = Session(max_retries=5) requests.set_domain_delay('letterboxd.com', '1 seconds') base_url = 'http://letterboxd.com' SLUGS = { 'default': { 'p_slug': '/%(user)s/list/%(list)s/', 'f_slug': 'data-film-slug'}, 'diary': { 'p_slug': '/%(user)s/films/diary/', 'f_slug': 'data-film-slug'}, 'likes': { 'p_slug': '/%(user)s/likes/films/', 'f_slug': 'data-film-link'}, 'rated': { 'p_slug': '/%(user)s/films/ratings/', 'f_slug': 'data-film-slug'},
from __future__ import unicode_literals, division, absolute_import import logging import urllib from flexget import plugin from flexget.entry import Entry from flexget.event import event from flexget.config_schema import one_or_more from flexget.utils.requests import Session from flexget.utils.search import normalize_unicode log = logging.getLogger('rarbg') requests = Session() requests.set_domain_delay( 'torrentapi.org', '10.3 seconds') # they only allow 1 request per 10 seconds CATEGORIES = { 'all': 0, # Movies 'x264 720p': 45, 'x264 1080p': 44, 'XviD': 14, 'Full BD': 42, # TV 'HDTV': 41, 'SDTV': 18 }
class InputWhatCD(object): """A plugin that searches what.cd == Usage: All parameters except `username` and `password` are optional. whatcd: username: password: user_agent: (A custom user-agent for the client to report. It is NOT A GOOD IDEA to spoof a browser with this. You are responsible for your account.) search: (general search filter) artist: (artist name) album: (album name) year: (album year) encoding: (encoding specifics - 192, 320, lossless, etc.) format: (MP3, FLAC, AAC, etc.) media: (CD, DVD, vinyl, Blu-ray, etc.) release_type: (album, soundtrack, EP, etc.) log: (log specification - true, false, '100%', or '<100%') hascue: (has a cue file - true or false) scene: (is a scene release - true or false) vanityhouse: (is a vanity house release - true or false) leech_type: ('freeleech', 'neutral', 'either', or 'normal') tags: (a list of tags to match - drum.and.bass, new.age, blues, etc.) tag_type: (match 'any' or 'all' of the items in `tags`) """ # Aliases for config -> api params ALIASES = { "artist": "artistname", "album": "groupname", "leech_type": "freetorrent", "release_type": "releaseType", "tags": "taglist", "tag_type": "tags_type", "search": "searchstr", "log": "haslog", } # API parameters # None means a raw value entry (no validation) # A dict means a choice with a mapping for the API # A list is just a choice with no mapping PARAMS = { "searchstr": None, "taglist": None, "artistname": None, "groupname": None, "year": None, "tags_type": { "any": 0, "all": 1, }, "encoding": [ "192", "APS (VBR)", "V2 (VBR)", "V1 (VBR)", "256", "APX (VBR)", "V0 (VBR)", "320", "lossless", "24bit lossless", "V8 (VBR)" ], "format": ["MP3", "FLAC", "AAC", "AC3", "DTS"], "media": [ "CD", "DVD", "vinyl", "soundboard", "SACD", "DAT", "cassette", "WEB", "Blu-ray" ], "releaseType": { "album": 1, "soundtrack": 3, "EP": 5, "anthology": 6, "compilation": 7, "DJ mix": 8, "single": 9, "live album": 11, "remix": 13, "bootleg": 14, "interview": 15, "mixtape": 16, "unknown": 21, "concert recording": 22, "demo": 23 }, "haslog": { "False": 0, "True": 1, "100%": 100, "<100%": -1 }, "freetorrent": { "freeleech": 1, "neutral": 2, "either": 3, "normal": 0, }, "hascue": { "False": 0, "True": 1, }, "scene": { "False": 0, "True": 1, }, "vanityhouse": { "False": 0, "True": 1, } } def _key(self, key): """Gets the API key name from the entered key""" try: if key in self.ALIASES: return self.ALIASES[key] elif key in self.PARAMS: return key return None except KeyError: return None def _opts(self, key): """Gets the options for the specified key""" temp = self._key(key) try: return self.PARAMS[temp] except KeyError: return None def _getval(self, key, val): """Gets the value for the specified key""" # No alias or param by that name if self._key(key) is None: return None opts = self._opts(key) if opts is None: if isinstance(val, list): return ",".join(val) return val elif isinstance(opts, dict): # Options, translate the input to output # The str cast converts bools to 'True'/'False' for use as keys return opts[str(val)] else: # List of options, check it's in the list if val not in opts: return None return val def __init__(self): """Set up the schema""" self.schema = { 'type': 'object', 'properties': { 'username': { 'type': 'string' }, 'password': { 'type': 'string' }, 'user_agent': { 'type': 'string' }, 'search': { 'type': 'string' }, 'artist': { 'type': 'string' }, 'album': { 'type': 'string' }, 'year': { 'type': ['string', 'integer'] }, 'tags': one_or_more({'type': 'string'}), 'tag_type': { 'type': 'string', 'enum': self._opts('tag_type').keys() }, 'encoding': { 'type': 'string', 'enum': self._opts('encoding') }, 'format': { 'type': 'string', 'enum': self._opts('format') }, 'media': { 'type': 'string', 'enum': self._opts('media') }, 'release_type': { 'type': 'string', 'enum': self._opts('release_type').keys() }, 'log': { 'oneOf': [{ 'type': 'string', 'enum': self._opts('log').keys() }, { 'type': 'boolean' }] }, 'leech_type': { 'type': 'string', 'enum': self._opts('leech_type').keys() }, 'hascue': { 'type': 'boolean' }, 'scene': { 'type': 'boolean' }, 'vanityhouse': { 'type': 'boolean' }, }, 'required': ['username', 'password'], 'additionalProperties': False } def _login(self, config): """ Log in and store auth data from the server Adapted from https://github.com/isaaczafuta/whatapi """ data = { 'username': config['username'], 'password': config['password'], 'keeplogged': 1, } r = self.session.post("https://ssl.what.cd/login.php", data=data, allow_redirects=False) if r.status_code != 302 or r.headers.get('location') != "index.php": raise PluginError("Failed to log in to What.cd") accountinfo = self._request("index") self.authkey = accountinfo["authkey"] self.passkey = accountinfo["passkey"] log.info("Logged in to What.cd") def _request(self, action, **kwargs): """ Make an AJAX request to a given action page Adapted from https://github.com/isaaczafuta/whatapi """ ajaxpage = 'https://ssl.what.cd/ajax.php' params = {} # Filter params and map config values -> api values for k, v in kwargs.iteritems(): key = self._key(k) if key is not None: params[key] = self._getval(k, v) # Params other than the searching ones params['action'] = action if 'page' in kwargs: params['page'] = kwargs['page'] r = self.session.get(ajaxpage, params=params, allow_redirects=False) if r.status_code != 200: raise PluginError("What.cd returned a non-200 status code") try: json_response = r.json() if json_response['status'] != "success": # Try to deal with errors returned by the API error = json_response.get('error', json_response.get('status')) if not error or error == "failure": error = json_response.get('response') if not error: error = str(json_response) raise PluginError("What.cd gave a failure response: " "'{0}'".format(error)) return json_response['response'] except (ValueError, TypeError) as e: raise PluginError("What.cd returned an invalid response") @cached('whatcd') @plugin.internet(log) def on_task_input(self, task, config): """Search on What.cd""" self.session = Session() # From the API docs: "Refrain from making more than five (5) requests every ten (10) seconds" self.session.set_domain_delay('ssl.what.cd', '2 seconds') # Login self._login(config) # Perform the query results = [] page = 1 while True: result = self._request("browse", page=page, **config) if not result['results']: break results.extend(result["results"]) pages = result['pages'] page = result['currentPage'] log.info("Got {0} of {1} pages".format(page, pages)) if page >= pages: break page += 1 # Logged in and made a request successfully, it's ok if nothing matches task.no_entries_ok = True # Parse the needed information out of the response entries = [] for result in results: # Get basic information on the release info = dict( (k, result[k]) for k in ('artist', 'groupName', 'groupYear')) # Releases can have multiple download options for tor in result['torrents']: temp = info.copy() temp.update( dict( (k, tor[k]) for k in ('media', 'encoding', 'format', 'torrentId'))) entries.append( Entry( title="{artist} - {groupName} - {groupYear} " "({media} - {format} - {encoding})-{torrentId}.torrent" .format(**temp), url="https://what.cd/torrents.php?action=download&" "id={0}&authkey={1}&torrent_pass={2}".format( temp['torrentId'], self.authkey, self.passkey), torrent_seeds=tor['seeders'], torrent_leeches=tor['leechers'], # Size is given in bytes, convert it content_size=int(tor['size'] / (1024**2) * 100) / 100)) return entries
from __future__ import unicode_literals, division, absolute_import import logging import urllib from flexget import plugin from flexget.entry import Entry from flexget.event import event from flexget.config_schema import one_or_more from flexget.utils.requests import Session, get from flexget.utils.search import normalize_unicode log = logging.getLogger('rarbg') requests = Session() requests.set_domain_delay( 'torrentapi.org', '2.1 seconds') # they only allow 1 request per 2 seconds CATEGORIES = { 'all': 0, # Movies 'x264 720p': 45, 'x264 1080p': 44, 'XviD': 14, 'Full BD': 42, # TV 'HDTV': 41, 'SDTV': 18 }
from __future__ import unicode_literals, division, absolute_import import logging import urllib from flexget import plugin from flexget.entry import Entry from flexget.event import event from flexget.config_schema import one_or_more from flexget.utils.requests import Session, get from flexget.utils.search import normalize_unicode log = logging.getLogger('rarbg') requests = Session() requests.set_domain_delay('torrentapi.org', '2.1 seconds') # they only allow 1 request per 2 seconds CATEGORIES = { 'all': 0, # Movies 'x264 720p': 45, 'x264 1080p': 44, 'XviD': 14, 'Full BD': 42, # TV 'HDTV': 41, 'SDTV': 18 }