示例#1
0
    def search(self, query, comparator=StringComparator(), config=None):
        """
        Search for name from piratebay.
        """
        if not isinstance(config, dict):
            config = {}
        sort = SORT.get(config.get('sort_by', 'seeds'))
        if config.get('sort_reverse'):
            sort += 1
        if isinstance(config.get('category'), int):
            category = config['category']
        else:
            category = CATEGORIES.get(config.get('category', 'all'))
        filter_url = '/0/%d/%d' % (sort, category)

        comparator.set_seq1(query)
        query = comparator.search_string()
        # urllib.quote will crash if the unicode string has non ascii characters, so encode in utf-8 beforehand
        url = 'http://thepiratebay.se/search/' + urllib.quote(query.encode('utf-8')) + filter_url
        log.debug('Using %s as piratebay search url' % url)
        page = requests.get(url).content
        soup = get_soup(page)
        entries = []
        for link in soup.find_all('a', attrs={'class': 'detLink'}):
            comparator.set_seq2(link.contents[0])
            log.debug('name: %s' % comparator.a)
            log.debug('found name: %s' % comparator.b)
            log.debug('confidence: %s' % comparator.ratio())
            if not comparator.matches():
                continue
            entry = Entry()
            entry['title'] = link.contents[0]
            entry['url'] = 'http://thepiratebay.se' + link.get('href')
            tds = link.parent.parent.parent.find_all('td')
            entry['torrent_seeds'] = int(tds[-2].contents[0])
            entry['torrent_leeches'] = int(tds[-1].contents[0])
            entry['search_ratio'] = comparator.ratio()
            entry['search_sort'] = torrent_availability(entry['torrent_seeds'], entry['torrent_leeches'])
            # Parse content_size
            size = link.find_next(attrs={'class': 'detDesc'}).contents[0]
            size = re.search('Size ([\.\d]+)\xa0([GMK])iB', size)
            if size:
                if size.group(2) == 'G':
                    entry['content_size'] = int(float(size.group(1)) * 1000 ** 3 / 1024 ** 2)
                elif size.group(2) == 'M':
                    entry['content_size'] = int(float(size.group(1)) * 1000 ** 2 / 1024 ** 2)
                else:
                    entry['content_size'] = int(float(size.group(1)) * 1000 / 1024 ** 2)
            entries.append(entry)

        if not entries:
            dashindex = query.rfind('-')
            if dashindex != -1:
                return self.search(query[:dashindex], comparator=comparator)
            else:
                raise PluginWarning('No close matches for %s' % query, log, log_once=True)

        entries.sort(reverse=True, key=lambda x: x.get('search_sort'))

        return entries
示例#2
0
def get_first_result(tmdb_function, value):
    if isinstance(value, unicode):
        value = value.encode('utf-8')
    if isinstance(value, basestring):
        value = quote(value, safe=b'')
    print "TMDB_FUNCT" + tmdb_function
    if tmdb_function == 'imdbLookup':
    	url = '%s/3/movie/%s?&api_key=%s' % (server, value, api_key)
    elif tmdb_function == 'search':
    	url = '%s/3/search/movie?query=%s&api_key=%s&page=1' % (server, value, api_key)
    elif tmdb_function == 'getInfo':
        url = '%s/3/movie/%s?&api_key=%s' % (server, value, api_key)
        print url
    else:
        print value
    try:
        data = requests.get(url)
    except requests.RequestException:
        log.warning('Request failed %s' % url)
        return
    try:
        result = data.json()
    except ValueError:
        log.warning('TMDb returned invalid json.')
        return
    # Make sure there is a valid result to return
    if len(result) and result.get('id'):
            return result
示例#3
0
def query_member_id(api_key, user_token, login_name):
    """
    Get the member id of a member identified by its login name.

    :param string api_key: Api key requested at https://www.betaseries.com/api
    :param string user_token: obtained with a call to create_token()
    :param string login_name: The login name of the member
    :return: Id of the member identified by its login name or `None` if not found
    """
    r = requests.get(
        API_URL_PREFIX + 'members/search',
        params={'login': login_name},
        headers={
            'Accept': 'application/json',
            'X-BetaSeries-Version': '2.1',
            'X-BetaSeries-Key': api_key,
            'X-BetaSeries-Token': user_token,
        },
    )
    assert r.status_code == 200, "Bad HTTP status code: %s" % r.status_code
    j = r.json()
    error_list = j['errors']
    for err in error_list:
        log.error(str(err))
    found_id = None
    if not error_list:
        for candidate in j['users']:
            if candidate['login'] == login_name:
                found_id = candidate['id']
                break
    return found_id
示例#4
0
文件: pogcal.py 项目: Anaerin/Flexget
    def on_task_input(self, task, config):
        try:
            r = requests.post('http://www.pogdesign.co.uk/cat/', data={'username': config['username'], 'password': config['password'], 'sub_login': '******'}, allow_redirects=True)
            if 'U / P Invalid' in r.text:
                raise plugin.PluginError('Invalid username/password for pogdesign.')
            page = requests.get('http://www.pogdesign.co.uk/cat/showselect.php', cookies=r.cookies)
        except requests.RequestException as e:
            raise plugin.PluginError('Error retrieving source: %s' % e)
        soup = BeautifulSoup(page.text)
        entries = []
        for row in soup.find_all('label', {'class': 'label_check'}):
            if row.find(attrs={'checked': 'checked'}):
                t = row.text
                if t.endswith('[The]'):
                    t = 'The ' + t[:-6]

                # Make certain names friendlier
                if t in self.name_map:
                    t = self.name_map[t]

                e = Entry()
                e['title'] = t
                url = row.find_next('a', {'class': 'selectsummary'})
                e['url'] = 'http://www.pogdesign.co.uk' + url['href']
                entries.append(e)
        return entries
示例#5
0
    def url_rewrite(self, task, entry):
        url = entry['url']
        for (scheme, netloc) in EZTV_MIRRORS:
            try:
                _, _, path, params, query, fragment = urlparse(url)
                url = urlunparse((scheme, netloc, path, params, query, fragment))
                page = requests.get(url).content
            except RequestException as e:
                log.debug('Eztv mirror `%s` seems to be down', url)
                continue
            break

        if not page:
            raise UrlRewritingError('No mirrors found for url %s' % entry['url'])

        log.debug('Eztv mirror `%s` chosen', url)
        try:
            soup = get_soup(page)
            mirrors = soup.find('a', attrs={'class': re.compile(r'download_\d')})
            if not mirrors:
                raise UrlRewritingError('Unable to locate download link from url %s'
                                        % url)
            entry['urls'] = [m.get('href') for m in mirrors]
            entry['url'] = mirrors[0].get('href')
        except Exception as e:
            raise UrlRewritingError(e)
示例#6
0
def get_http_seeds(url, info_hash):
    url = get_scrape_url(url, info_hash)
    if not url:
        log.debug('if not url is true returning 0')
        return 0
    log.debug('Checking for seeds from %s' % url)

    try:
        data = bdecode(requests.get(url).text.encode('utf-8')).get('files')
    except RequestException as e:
        log.debug('Error scraping: %s' % e)
        return 0
    except SyntaxError as e:
        log.warning('Error decoding tracker response: %s' % e)
        return 0
    except BadStatusLine as e:
        log.warning('Error BadStatusLine: %s' % e)
        return 0
    except IOError as e:
        log.warning('Server error: %s' % e)
        return 0
    if not data:
        log.debug('No data received from tracker scrape.')
        return 0
    log.debug('get_http_seeds is returning: %s' % list(data.values())[0]['complete'])
    return list(data.values())[0]['complete']
示例#7
0
def query_member_id(api_key, user_token, login_name):
    """
    Get the member id of a member identified by its login name.

    :param string api_key: Api key requested at http://www.betaseries.com/api
    :param string user_token: obtained with a call to create_token()
    :param string login_name: The login name of the member
    :return: Id of the member identified by its login name or `None` if not found
    """
    r = requests.get(
        API_URL_PREFIX + "members/search",
        params={"login": login_name},
        headers={
            "Accept": "application/json",
            "X-BetaSeries-Version": "2.1",
            "X-BetaSeries-Key": api_key,
            "X-BetaSeries-Token": user_token,
        },
    )
    assert r.status_code == 200, "Bad HTTP status code: %s" % r.status_code
    j = r.json()
    error_list = j["errors"]
    for err in error_list:
        log.error(str(err))
    found_id = None
    if not error_list:
        for candidate in j["users"]:
            if candidate["login"] == login_name:
                found_id = candidate["id"]
                break
    return found_id
示例#8
0
    def entries_from_search(self, name, url=None):
        """Parses torrent download url from search results"""
        name = normalize_unicode(name)
        if not url:
            url = 'http://www.newtorrents.info/search/%s' % quote(
                name.encode('utf-8'), safe=b':/~?=&%'
            )

        logger.debug('search url: {}', url)

        html = requests.get(url).text
        # fix </SCR'+'IPT> so that BS does not crash
        # TODO: should use beautifulsoup massage
        html = re.sub(r'(</SCR.*?)...(.*?IPT>)', r'\1\2', html)

        soup = get_soup(html)
        # saving torrents in dict
        torrents = []
        for link in soup.find_all('a', attrs={'href': re.compile('down.php')}):
            torrent_url = 'http://www.newtorrents.info%s' % link.get('href')
            release_name = link.parent.next.get('title')
            # quick dirty hack
            seed = link.find_next('td', attrs={'class': re.compile('s')}).renderContents()
            if seed == 'n/a':
                seed = 0
            else:
                try:
                    seed = int(seed)
                except ValueError:
                    logger.warning(
                        'Error converting seed value ({}) from newtorrents to integer.', seed
                    )
                    seed = 0

            # TODO: also parse content_size and peers from results
            torrents.append(
                Entry(
                    title=release_name,
                    url=torrent_url,
                    torrent_seeds=seed,
                    torrent_availability=torrent_availability(seed, 0),
                )
            )
        # sort with seed number Reverse order
        torrents.sort(reverse=True, key=lambda x: x.get('torrent_availability', 0))
        # choose the torrent
        if not torrents:
            dashindex = name.rfind('-')
            if dashindex != -1:
                return self.entries_from_search(name[:dashindex])
            else:
                return torrents
        else:
            if len(torrents) == 1:
                logger.debug('found only one matching search result.')
            else:
                logger.debug(
                    'search result contains multiple matches, sorted {} by most seeders', torrents
                )
            return torrents
    def search(self, query, comparator=StringComparator(), config=None):
        """
        Search for name from prelist.
        """
        if not isinstance(config, dict):
            config = {}
       

        comparator.set_seq1(query)
        query = comparator.search_string()
        # urllib.quote will crash if the unicode string has non ascii characters, so encode in utf-8 beforehand
        url = 'http://www.prelist.ws/?search=' + urllib.quote(query.encode('utf-8')) 
        log.debug('Using %s as prelist search url' % url)
        page = requests.get(url).content
        soup = get_soup(page)
        entries = []
        for link in soup.find_all('a',attrs={'href' : re.compile('search=')}):
            log.debug('name: %s' % link)

            comparator.set_seq2(link.contents[0])
            log.debug('name: %s' % comparator.a)
            log.debug('found name: %s' % comparator.b)
            log.debug('confidence: %s' % comparator.ratio())
            if not comparator.matches():
                continue
            entry = Entry()
            entry['title'] = link.contents[0]
            entry['url'] = 'http://www.prelist.ws/' + link.get('href')
            entries.append(entry)


        return entries
示例#10
0
    def entries_from_search(self, name, url=None):
        """Parses torrent download url from search results"""
        name = normalize_unicode(name)
        if not url:
            url = 'http://www.newtorrents.info/search/%s' % quote(
                name.encode('utf-8'), safe=b':/~?=&%'
            )

        log.debug('search url: %s' % url)

        html = requests.get(url).text
        # fix </SCR'+'IPT> so that BS does not crash
        # TODO: should use beautifulsoup massage
        html = re.sub(r'(</SCR.*?)...(.*?IPT>)', r'\1\2', html)

        soup = get_soup(html)
        # saving torrents in dict
        torrents = []
        for link in soup.find_all('a', attrs={'href': re.compile('down.php')}):
            torrent_url = 'http://www.newtorrents.info%s' % link.get('href')
            release_name = link.parent.next.get('title')
            # quick dirty hack
            seed = link.find_next('td', attrs={'class': re.compile('s')}).renderContents()
            if seed == 'n/a':
                seed = 0
            else:
                try:
                    seed = int(seed)
                except ValueError:
                    log.warning(
                        'Error converting seed value (%s) from newtorrents to integer.' % seed
                    )
                    seed = 0

            # TODO: also parse content_size and peers from results
            torrents.append(
                Entry(
                    title=release_name,
                    url=torrent_url,
                    torrent_seeds=seed,
                    torrent_availability=torrent_availability(seed, 0),
                )
            )
        # sort with seed number Reverse order
        torrents.sort(reverse=True, key=lambda x: x.get('torrent_availability', 0))
        # choose the torrent
        if not torrents:
            dashindex = name.rfind('-')
            if dashindex != -1:
                return self.entries_from_search(name[:dashindex])
            else:
                return torrents
        else:
            if len(torrents) == 1:
                log.debug('found only one matching search result.')
            else:
                log.debug(
                    'search result contains multiple matches, sorted %s by most seeders' % torrents
                )
            return torrents
示例#11
0
    def search(self, task, entry, config=None):
        """
        Search for name from iptorrents
        """

        categories = config.get('category', 'All')
        # Make sure categories is a list
        if not isinstance(categories, list):
            categories = [categories]

        # If there are any text categories, turn them into their id number
        categories = [c if isinstance(c, int) else CATEGORIES[c]
                      for c in categories]
        filter_url = '&'.join((str(c) + '=') for c in categories)

        entries = set()

        for search_string in entry.get('search_strings', [entry['title']]):
            query = normalize_unicode(search_string)
            query = quote_plus(query.encode('utf8'))

            url = "{base_url}/t?{filter}&q={query}&qf=".format(base_url=BASE_URL, filter=filter_url, query=query)
            log.debug('searching with url: %s' % url)
            req = requests.get(url, cookies={'uid': str(config['uid']), 'pass': config['password']})

            if '/u/' + str(config['uid']) not in req.text:
                raise plugin.PluginError("Invalid cookies (user not logged in)...")

            soup = get_soup(req.content, parser="html.parser")
            torrents = soup.find('table', {'id': 'torrents'})

            results = torrents.findAll('tr')
            for torrent in results:
                if torrent.th and 'ac' in torrent.th.get('class'):
                    # Header column
                    continue
                if torrent.find('td', {'colspan': '99'}):
                    log.debug('No results found for search %s', search_string)
                    return
                entry = Entry()
                link = torrent.find('a', href=re.compile('download'))['href']
                entry['url'] = "{base}{link}?torrent_pass={key}".format(
                    base=BASE_URL, link=link, key=config.get('rss_key'))
                entry['title'] = torrent.find('a', href=re.compile('details')).text

                seeders = torrent.findNext('td', {'class': 'ac t_seeders'}).text
                leechers = torrent.findNext('td', {'class': 'ac t_leechers'}).text
                entry['torrent_seeds'] = int(seeders)
                entry['torrent_leeches'] = int(leechers)
                entry['search_sort'] = torrent_availability(entry['torrent_seeds'],
                                                            entry['torrent_leeches'])

                size = torrent.findNext(text=re.compile('^([\.\d]+) ([GMK]?)B$'))
                size = re.search('^([\.\d]+) ([GMK]?)B$', size)

                entry['content_size'] = parse_filesize(size.group(0))
                log.debug('Found entry %s', entry)
                entries.add(entry)

        return entries
示例#12
0
文件: irc.py 项目: AlinaKay/Flexget
 def retrieve_tracker_config(cls, tracker_config_file):
     """
     Will attempt to retrieve the .tracker file from disk or github. Returns the parsed XML.
     :param tracker_config_file: URL or path to .tracker file
     :return: parsed XML
     """
     tracker_config_file = os.path.expanduser(tracker_config_file)
     if os.path.exists(tracker_config_file):
         return cls.read_tracker_config(tracker_config_file)
     elif re.match('^(?![\\\/]).+\.tracker$', tracker_config_file):
         tracker_basepath = os.path.abspath(os.path.join(manager.config_base, 'trackers'))
         save_path = os.path.join(tracker_basepath, tracker_config_file)
         if not os.path.exists(tracker_basepath):
             try:
                 os.mkdir(tracker_basepath)
             except IOError as e:
                 raise TrackerFileError(e)
         if not os.path.exists(save_path):
             log.info('Tracker file not found on disk. Attempting to fetch tracker config file from Github.')
             url = 'https://raw.githubusercontent.com/autodl-community/autodl-trackers/master/' + tracker_config_file
             try:
                 r = requests.get(url)
                 with io.open(save_path, 'wb') as tracker_file:
                     for chunk in r.iter_content(8192):
                         tracker_file.write(chunk)
             except (requests.RequestException, IOError) as e:
                 raise TrackerFileError(e)
         return cls.read_tracker_config(save_path)
     else:
         raise TrackerFileError('Unable to open tracker config file %s', tracker_config_file)
示例#13
0
 def get_file(self, only_cached=False):
     """Makes sure the poster is downloaded to the local cache (in userstatic folder) and
     returns the path split into a list of directory and file components"""
     from flexget.manager import manager
     base_dir = os.path.join(manager.config_base, 'userstatic')
     if self.file and os.path.isfile(os.path.join(base_dir, self.file)):
         return self.file.split(os.sep)
     elif only_cached:
         return
     # If we don't already have a local copy, download one.
     log.debug('Downloading poster %s' % self.url)
     dirname = os.path.join('tmdb', 'posters', str(self.movie_id))
     # Create folders if they don't exist
     fullpath = os.path.join(base_dir, dirname)
     if not os.path.isdir(fullpath):
         os.makedirs(fullpath)
     filename = os.path.join(dirname, posixpath.basename(self.url))
     thefile = file(os.path.join(base_dir, filename), 'wb')
     thefile.write(requests.get(self.url).content)
     self.file = filename
     # If we are detached from a session, update the db
     if not Session.object_session(self):
         session = Session()
         try:
             poster = session.query(TMDBPoster).filter(TMDBPoster.db_id == self.db_id).first()
             if poster:
                 poster.file = filename
         finally:
             session.close()
     return filename.split(os.sep)
示例#14
0
def query_series(api_key, user_token, member_name=None):
    """
    Get the list of series followed by the authenticated user

    :param string api_key: Api key requested at http://www.betaseries.com/api
    :param string user_token: Obtained with a call to create_token()
    :param string member_name: [optional] A member name to get the list of series from. If None, will query the member
        for whom the user_token was for
    :return: List of serie titles or empty list
    """
    params = {}
    if member_name:
        member_id = query_member_id(api_key, user_token, member_name)
        if member_id:
            params = {'id': member_id}
        else:
            log.error("member %r not found" % member_name)
            return []
    r = requests.get(API_URL_PREFIX + 'members/infos', params=params, headers={
        'Accept': 'application/json',
        'X-BetaSeries-Version': '2.1',
        'X-BetaSeries-Key': api_key,
        'X-BetaSeries-Token': user_token,
    })
    assert r.status_code == 200, "Bad HTTP status code: %s" % r.status_code
    j = r.json()
    error_list = j['errors']
    for err in error_list:
        log.error(str(err))
    if not error_list:
        return [x['title'] for x in j['member']['shows'] if x['user']['archived'] is False]
    else:
        return []
示例#15
0
文件: plex.py 项目: ratoaq2/Flexget
 def plex_get_accesstoken(self, config, globalaccesstoken=""):
     accesstoken = None
     if not globalaccesstoken:
         globalaccesstoken = self.plex_get_globalaccesstoken(config)
     if config['server'] in ('localhost', '127.0.0.1'):
         log.debug('Server using localhost. Global Token will be used')
         return globalaccesstoken
     try:
         r = requests.get(
             "https://my.plexapp.com/pms/servers?X-Plex-Token=%s" %
             globalaccesstoken)
     except requests.RequestException as e:
         raise plugin.PluginError(
             "Could not get servers from my.plexapp.com using "
             "authentication-token: %s. (%s)" % (globalaccesstoken, e))
     dom = parseString(r.text)
     for node in dom.getElementsByTagName('Server'):
         if config['server'] in (node.getAttribute('address'),
                                 node.getAttribute('localAddresses')):
             accesstoken = node.getAttribute('accessToken')
             log.debug("Got plextoken: %s" % accesstoken)
     if not accesstoken:
         raise plugin.PluginError('Could not retrieve accesstoken for %s.' %
                                  config['server'])
     else:
         return accesstoken
示例#16
0
    def on_task_input(self, task, config):
        try:
            r = requests.post('http://www.pogdesign.co.uk/cat/',
                              data={
                                  'username': config['username'],
                                  'password': config['password'],
                                  'sub_login': '******'
                              },
                              allow_redirects=True)
            if 'U / P Invalid' in r.text:
                raise plugin.PluginError(
                    'Invalid username/password for pogdesign.')
            page = requests.get(
                'http://www.pogdesign.co.uk/cat/showselect.php',
                cookies=r.cookies)
        except requests.RequestException as e:
            raise plugin.PluginError('Error retrieving source: %s' % e)
        soup = BeautifulSoup(page.text)
        entries = []
        for row in soup.find_all('label', {'class': 'label_check'}):
            if row.find(attrs={'checked': 'checked'}):
                t = row.text
                if t.endswith('[The]'):
                    t = 'The ' + t[:-6]

                # Make certain names friendlier
                if t in self.name_map:
                    t = self.name_map[t]

                e = Entry()
                e['title'] = t
                url = row.find_next('a', {'class': 'selectsummary'})
                e['url'] = 'http://www.pogdesign.co.uk' + url['href']
                entries.append(e)
        return entries
示例#17
0
def query_series(api_key, user_token, member_name=None):
    """
    Get the list of series followed by the authenticated user

    :param string api_key: Api key requested at http://www.betaseries.com/api
    :param string user_token: Obtained with a call to create_token()
    :param string member_name: [optional] A member name to get the list of series from. If None, will query the member
        for whom the user_token was for
    :return: List of serie titles or empty list
    """
    params = {}
    if member_name:
        member_id = query_member_id(api_key, user_token, member_name)
        if member_id:
            params = {'id': member_id}
        else:
            log.error("member %r not found" % member_name)
            return []
    r = requests.get(API_URL_PREFIX + 'members/infos', params=params, headers={
        'Accept': 'application/json',
        'X-BetaSeries-Version': '2.1',
        'X-BetaSeries-Key': api_key,
        'X-BetaSeries-Token': user_token,
    })
    assert r.status_code == 200, "Bad HTTP status code: %s" % r.status_code
    j = r.json()
    error_list = j['errors']
    for err in error_list:
        log.error(str(err))
    if not error_list:
        return [x['title'] for x in j['member']['shows'] if x['user']['archived'] is False]
    else:
        return []
示例#18
0
 def retrieve_tracker_config(cls, tracker_config_file):
     """
     Will attempt to retrieve the .tracker file from disk or github. Returns the parsed XML.
     :param tracker_config_file: URL or path to .tracker file
     :return: parsed XML
     """
     tracker_config_file = os.path.expanduser(tracker_config_file)
     if os.path.exists(tracker_config_file):
         return cls.read_tracker_config(tracker_config_file)
     elif re.match('^(?![\\\/]).+\.tracker$', tracker_config_file):
         tracker_basepath = os.path.abspath(os.path.join(manager.config_base, 'trackers'))
         save_path = os.path.join(tracker_basepath, tracker_config_file)
         if not os.path.exists(tracker_basepath):
             try:
                 os.mkdir(tracker_basepath)
             except IOError as e:
                 raise TrackerFileError(e)
         if not os.path.exists(save_path):
             log.info('Tracker file not found on disk. Attempting to fetch tracker config file from Github.')
             url = 'https://raw.githubusercontent.com/autodl-community/autodl-trackers/master/' + tracker_config_file
             try:
                 r = requests.get(url)
                 with io.open(save_path, 'wb') as tracker_file:
                     for chunk in r.iter_content(8192):
                         tracker_file.write(chunk)
             except (requests.RequestException, IOError) as e:
                 raise TrackerFileError(e)
         return cls.read_tracker_config(save_path)
     else:
         raise TrackerFileError('Unable to open tracker config file %s', tracker_config_file)
示例#19
0
文件: plex.py 项目: Flexget/Flexget
 def plex_get_accesstoken(self, config, globalaccesstoken=""):
     accesstoken = None
     if not globalaccesstoken:
         globalaccesstoken = self.plex_get_globalaccesstoken(config)
     if config['server'] in ('localhost', '127.0.0.1'):
         log.debug('Server using localhost. Global Token will be used')
         return globalaccesstoken
     try:
         r = requests.get(
             "https://my.plexapp.com/pms/servers?X-Plex-Token=%s" % globalaccesstoken
         )
     except requests.RequestException as e:
         raise plugin.PluginError(
             "Could not get servers from my.plexapp.com using "
             "authentication-token: %s. (%s)" % (globalaccesstoken, e)
         )
     dom = parseString(r.text)
     for node in dom.getElementsByTagName('Server'):
         if config['server'] in (
             node.getAttribute('address'),
             node.getAttribute('localAddresses'),
         ):
             accesstoken = node.getAttribute('accessToken')
             log.debug("Got plextoken: %s", accesstoken)
     if not accesstoken:
         raise plugin.PluginError('Could not retrieve accesstoken for %s.' % config['server'])
     else:
         return accesstoken
示例#20
0
    def search(self, task, entry, config=None):
        """
        Search for name from iptorrents
        """

        categories = config.get('category', 'All')
        # Make sure categories is a list
        if not isinstance(categories, list):
            categories = [categories]

        # If there are any text categories, turn them into their id number
        categories = [c if isinstance(c, int) else CATEGORIES[c]
                      for c in categories]
        filter_url = '&'.join((str(c) + '=') for c in categories)

        entries = set()

        for search_string in entry.get('search_strings', [entry['title']]):
            query = normalize_unicode(search_string)
            query = quote_plus(query.encode('utf8'))

            url = "{base_url}/t?{filter}&q={query}&qf=".format(base_url=BASE_URL, filter=filter_url, query=query)
            log.debug('searching with url: %s' % url)
            req = requests.get(url, cookies={'uid': str(config['uid']), 'pass': config['password']})

            if '/u/' + str(config['uid']) not in req.text:
                raise plugin.PluginError("Invalid cookies (user not logged in)...")

            soup = get_soup(req.content, parser="html.parser")
            torrents = soup.find('table', {'id': 'torrents'})

            results = torrents.findAll('tr')
            for torrent in results:
                if torrent.th and 'ac' in torrent.th.get('class'):
                    # Header column
                    continue
                if torrent.find('td', {'colspan': '99'}):
                    log.debug('No results found for search %s', search_string)
                    break
                entry = Entry()
                link = torrent.find('a', href=re.compile('download'))['href']
                entry['url'] = "{base}{link}?torrent_pass={key}".format(
                    base=BASE_URL, link=link, key=config.get('rss_key'))
                entry['title'] = torrent.find('a', href=re.compile('details')).text

                seeders = torrent.findNext('td', {'class': 'ac t_seeders'}).text
                leechers = torrent.findNext('td', {'class': 'ac t_leechers'}).text
                entry['torrent_seeds'] = int(seeders)
                entry['torrent_leeches'] = int(leechers)
                entry['search_sort'] = torrent_availability(entry['torrent_seeds'],
                                                            entry['torrent_leeches'])

                size = torrent.findNext(text=re.compile('^([\.\d]+) ([GMK]?)B$'))
                size = re.search('^([\.\d]+) ([GMK]?)B$', size)

                entry['content_size'] = parse_filesize(size.group(0))
                log.debug('Found entry %s', entry)
                entries.add(entry)

        return entries
示例#21
0
def query_member_id(api_key, user_token, login_name):
    """
    Get the member id of a member identified by its login name.

    :param string api_key: Api key requested at http://www.betaseries.com/api
    :param string user_token: obtained with a call to create_token()
    :param string login_name: The login name of the member
    :return: Id of the member identified by its login name or `None` if not found
    """
    r = requests.get(API_URL_PREFIX + 'members/search', params={
        'login': login_name
    }, headers={
        'Accept': 'application/json',
        'X-BetaSeries-Version': '2.1',
        'X-BetaSeries-Key': api_key,
        'X-BetaSeries-Token': user_token,
    })
    assert r.status_code == 200, "Bad HTTP status code: %s" % r.status_code
    j = r.json()
    error_list = j['errors']
    for err in error_list:
        log.error(str(err))
    found_id = None
    if not error_list:
        for candidate in j['users']:
            if candidate['login'] == login_name:
                found_id = candidate['id']
                break
    return found_id
示例#22
0
 def get_file(self, only_cached=False):
     """Makes sure the poster is downloaded to the local cache (in userstatic folder) and
     returns the path split into a list of directory and file components"""
     from flexget.manager import manager
     base_dir = os.path.join(manager.config_base, 'userstatic')
     if self.file and os.path.isfile(os.path.join(base_dir, self.file)):
         return self.file.split(os.sep)
     elif only_cached:
         return
     # If we don't already have a local copy, download one.
     log.debug('Downloading poster %s' % self.url)
     dirname = os.path.join('tmdb', 'posters', str(self.movie_id))
     # Create folders if they don't exist
     fullpath = os.path.join(base_dir, dirname)
     if not os.path.isdir(fullpath):
         os.makedirs(fullpath)
     filename = os.path.join(dirname, posixpath.basename(self.url))
     thefile = file(os.path.join(base_dir, filename), 'wb')
     thefile.write(requests.get(self.url).content)
     self.file = filename
     # If we are detached from a session, update the db
     if not Session.object_session(self):
         session = Session()
         poster = session.query(TMDBPoster).filter(TMDBPoster.db_id == self.db_id).first()
         if poster:
             poster.file = filename
             session.commit()
         session.close()
     return filename.split(os.sep)
示例#23
0
def get_http_seeds(url, info_hash):
    url = get_scrape_url(url, info_hash)
    if not url:
        log.debug('if not url is true returning 0')
        return 0
    log.debug('Checking for seeds from %s' % url)

    try:
        data = bdecode(requests.get(url).content).get('files')
    except RequestException as e:
        log.debug('Error scraping: %s', e)
        return 0
    except SyntaxError as e:
        log.warning('Error decoding tracker response: %s', e)
        return 0
    except BadStatusLine as e:
        log.warning('Error BadStatusLine: %s', e)
        return 0
    except IOError as e:
        log.warning('Server error: %s', e)
        return 0
    if not data:
        log.debug('No data received from tracker scrape.')
        return 0
    log.debug('get_http_seeds is returning: %s', list(data.values())[0]['complete'])
    return list(data.values())[0]['complete']
示例#24
0
    def search(self, task, entry, config=None):
        """
        Search for name from piratebay.
        """
        if not isinstance(config, dict):
            config = {}
        sort = SORT.get(config.get('sort_by', 'seeds'))
        if config.get('sort_reverse'):
            sort += 1
        if isinstance(config.get('category'), int):
            category = config['category']
        else:
            category = CATEGORIES.get(config.get('category', 'all'))
        filter_url = '/0/%d/%d' % (sort, category)

        entries = set()
        for search_string in entry.get('search_strings', [entry['title']]):
            query = normalize_unicode(search_string)
            # TPB search doesn't like dashes
            query = query.replace('-', ' ')
            # urllib.quote will crash if the unicode string has non ascii characters, so encode in utf-8 beforehand
            url = 'http://thepiratebay.%s/search/%s%s' % (
                CUR_TLD, quote(query.encode('utf-8')), filter_url)
            log.debug('Using %s as piratebay search url' % url)
            page = requests.get(url).content
            soup = get_soup(page)
            for link in soup.find_all('a', attrs={'class': 'detLink'}):
                entry = Entry()
                entry['title'] = self.extract_title(link)
                if not entry['title']:
                    log.error(
                        'Malformed search result. No title or url found. Skipping.'
                    )
                    continue
                entry['url'] = 'http://thepiratebay.%s%s' % (CUR_TLD,
                                                             link.get('href'))
                tds = link.parent.parent.parent.find_all('td')
                entry['torrent_seeds'] = int(tds[-2].contents[0])
                entry['torrent_leeches'] = int(tds[-1].contents[0])
                entry['search_sort'] = torrent_availability(
                    entry['torrent_seeds'], entry['torrent_leeches'])
                # Parse content_size
                size = link.find_next(attrs={'class': 'detDesc'}).contents[0]
                size = re.search('Size ([\.\d]+)\xa0([GMK])iB', size)
                if size:
                    if size.group(2) == 'G':
                        entry['content_size'] = int(
                            float(size.group(1)) * 1000**3 / 1024**2)
                    elif size.group(2) == 'M':
                        entry['content_size'] = int(
                            float(size.group(1)) * 1000**2 / 1024**2)
                    else:
                        entry['content_size'] = int(
                            float(size.group(1)) * 1000 / 1024**2)
                entries.add(entry)

        return sorted(entries,
                      reverse=True,
                      key=lambda x: x.get('search_sort'))
示例#25
0
def bluray_request(endpoint, **params) -> Any:
    full_url = BASE_URL + endpoint
    response = requests.get(full_url, params=params)
    if response.content:
        try:
            return response.json(strict=False)
        except JSONDecodeError as e:
            raise PluginError('Could decode json from response blu-ray api')
示例#26
0
文件: pogcal.py 项目: Donavan/Flexget
 def on_task_input(self, task, config):
     try:
         r = requests.post('http://www.pogdesign.co.uk/cat/', data={'username': config['username'], 'password': config['password'], 'sub_login': '******'}, allow_redirects=True)
         if 'U / P Invalid' in r.text:
             raise plugin.PluginError('Invalid username/password for pogdesign.')
         page = requests.get('http://www.pogdesign.co.uk/cat/showselect.php', cookies=r.cookies)
     except requests.RequestException, e:
         raise plugin.PluginError('Error retrieving source: %s' % e)
示例#27
0
    def parse_downloads(self, series_url, search_title):
        page = requests.get(series_url).content
        try:
            soup = get_soup(page)
        except Exception as e:
            raise UrlRewritingError(e)

        urls = []
        # find all titles
        episode_titles = self.find_all_titles(search_title)
        if not episode_titles:
            raise UrlRewritingError('Unable to find episode')

        for ep_title in episode_titles:
            # find matching download
            episode_title = soup.find('strong',
                                      text=re.compile(ep_title, re.I))
            if not episode_title:
                continue

            # find download container
            episode = episode_title.parent
            if not episode:
                continue

            # find episode language
            episode_lang = episode.find_previous(
                'strong', text=re.compile('Sprache')).next_sibling
            if not episode_lang:
                log.warning('No language found for: %s', series_url)
                continue

            # filter language
            if not self.check_language(episode_lang):
                log.warning('languages not matching: %s <> %s',
                            self.config['language'], episode_lang)
                continue

            # find download links
            links = episode.find_all('a')
            if not links:
                log.warning('No links found for: %s', series_url)
                continue

            for link in links:
                if not link.has_attr('href'):
                    continue

                url = link['href']
                pattern = (
                    r'http:\/\/download\.serienjunkies\.org.*%s_.*\.html' %
                    self.config['hoster'])

                if re.match(pattern, url) or self.config['hoster'] == 'all':
                    urls.append(url)
                else:
                    continue
        return urls
示例#28
0
    def search(self, task, entry, config=None):
        """
        Search for name from iptorrents
        """

        categories = config.get('category', 'all')
        # Make sure categories is a list
        if not isinstance(categories, list):
            categories = [categories]

        # If there are any text categories, turn them into their id number
        categories = [c if isinstance(c, int) else CATEGORIES[c]
                      for c in categories]
        filter_url = '&'.join((str(c) + '=') for c in categories)

        entries = set()

        for search_string in entry.get('search_strings', [entry['title']]):
            query = normalize_unicode(search_string)
            query = quote_plus(query.encode('utf8'))

            url = "{base_url}/t?{filter}&q={query}&qf=".format(base_url=BASE_URL, filter=filter_url, query=query)
            log.debug('searching with url: %s' % url)
            req = requests.get(url, cookies={'uid': str(config['uid']), 'pass': config['password']})

            if '/u/' + str(config.get('uid')) not in req.content:
                raise plugin.PluginError("Invalid cookies (user not logged in)...")

            soup = get_soup(req.content, parser="html.parser")
            torrents = soup.find('table', {'class': 'torrents'})

            for torrent in torrents.findAll('a', href=re.compile('\.torrent$')):
                entry = Entry()
                entry['url'] = "{base}{link}?torrent_pass={key}".format(
                    base=BASE_URL, link=torrent['href'], key=config.get('rss_key'))
                entry['title'] = torrent.findPrevious("a", attrs={'class': 't_title'}).text

                seeders = torrent.findNext('td', {'class': 'ac t_seeders'}).text
                leechers = torrent.findNext('td', {'class': 'ac t_leechers'}).text
                entry['torrent_seeds'] = int(seeders)
                entry['torrent_leeches'] = int(leechers)
                entry['search_sort'] = torrent_availability(entry['torrent_seeds'],
                                                            entry['torrent_leeches'])

                size = torrent.findNext(text=re.compile('^([\.\d]+) ([GMK]?)B$'))
                size = re.search('^([\.\d]+) ([GMK]?)B$', size)
                if size:
                    if size.group(2) == 'G':
                        entry['content_size'] = int(float(size.group(1)) * 1000 ** 3 / 1024 ** 2)
                    elif size.group(2) == 'M':
                        entry['content_size'] = int(float(size.group(1)) * 1000 ** 2 / 1024 ** 2)
                    elif size.group(2) == 'K':
                        entry['content_size'] = int(float(size.group(1)) * 1000 / 1024 ** 2)
                    else:
                        entry['content_size'] = int(float(size.group(1)) / 1024 ** 2)
                entries.add(entry)

        return entries
示例#29
0
    def search(self, task, entry, config=None):
        """
            Search for entries on PublicHD
        """

        categories = config.get('category', 'all')
        # Ensure categories a list
        if not isinstance(categories, list):
            categories = [categories]
        # Convert named category to its respective category id number
        categories = [c if isinstance(c, int) else CATEGORIES[c] for c in categories]
        category_url_fragment = '&category=%s' % urllib.quote(';'.join(str(c) for c in categories))

        base_url = 'http://publichd.se/index.php?page=torrents&active=0'

        entries = set()
        for search_string in entry.get('search_strings', [entry['title']]):
            query = normalize_unicode(search_string)
            query_url_fragment = '&search=' + urllib.quote(query.encode('utf8'))

            # http://publichd.se/index.php?page=torrents&active=0&category=5;15&search=QUERY
            url = (base_url + category_url_fragment + query_url_fragment)
            log.debug('PublicHD search url: %s' % url)

            page = requests.get(url).content
            soup = get_soup(page)

            for result in soup.findAll('a', href=re.compile('page=torrent-details')):
                entry = Entry()
                entry['title'] = result.text
                # Expand the selection to whole row
                result = result.findPrevious('tr')
                download_url = result.find('a', href=re.compile('\.torrent$'))['href']
                torrent_hash = re.search(r'/([0-9a-fA-F]{5,40})/', download_url).group(1)

                entry['url'] = 'http://publichd.se/download.php?id=%s' % torrent_hash

                seeds, leeches = result.findAll('td', text=re.compile('^\d+$'))
                entry['torrent_seeds'] = int(seeds.text)
                entry['torrent_leeches'] = int(leeches.text)
                entry['search_sort'] = torrent_availability(entry['torrent_seeds'], entry['torrent_leeches'])
                size = result.find("td", text=re.compile('(\d+(?:[.,]\d+)*)\s?([KMG]B)')).text
                size = re.search('(\d+(?:[.,]\d+)*)\s?([KMG]B)', size)

                if size:
                    if size.group(2) == 'GB':
                        entry['content_size'] = int(float(size.group(1).replace(',', '')) * 1000 ** 3 / 1024 ** 2)
                    elif size.group(2) == 'MB':
                        entry['content_size'] = int(float(size.group(1).replace(',', '')) * 1000 ** 2 / 1024 ** 2)
                    elif size.group(2) == 'KB':
                        entry['content_size'] = int(float(size.group(1).replace(',', '')) * 1000 / 1024 ** 2)
                    else:
                        entry['content_size'] = int(float(size.group(1).replace(',', '')) / 1024 ** 2)

                entries.add(entry)

        return entries
示例#30
0
    def search(self, task, entry, config=None):
        """
            Search for entries on PublicHD
        """

        categories = config.get('category', 'all')
        # Ensure categories a list
        if not isinstance(categories, list):
            categories = [categories]
        # Convert named category to its respective category id number
        categories = [c if isinstance(c, int) else CATEGORIES[c] for c in categories]
        category_url_fragment = '&category=%s' % urllib.parse.quote(';'.join(str(c) for c in categories))

        base_url = 'http://publichd.se/index.php?page=torrents&active=0'

        entries = set()
        for search_string in entry.get('search_strings', [entry['title']]):
            query = normalize_unicode(search_string)
            query_url_fragment = '&search=' + urllib.parse.quote(query.encode('utf8'))

            # http://publichd.se/index.php?page=torrents&active=0&category=5;15&search=QUERY
            url = (base_url + category_url_fragment + query_url_fragment)
            log.debug('PublicHD search url: %s' % url)

            page = requests.get(url).content
            soup = get_soup(page)

            for result in soup.findAll('a', href=re.compile('page=torrent-details')):
                entry = Entry()
                entry['title'] = result.text
                # Expand the selection to whole row
                result = result.findPrevious('tr')
                download_url = result.find('a', href=re.compile('\.torrent$'))['href']
                torrent_hash = re.search(r'/([0-9a-fA-F]{5,40})/', download_url).group(1)

                entry['url'] = 'http://publichd.se/download.php?id=%s' % torrent_hash

                seeds, leeches = result.findAll('td', text=re.compile('^\d+$'))
                entry['torrent_seeds'] = int(seeds.text)
                entry['torrent_leeches'] = int(leeches.text)
                entry['search_sort'] = torrent_availability(entry['torrent_seeds'], entry['torrent_leeches'])
                size = result.find("td", text=re.compile('(\d+(?:[.,]\d+)*)\s?([KMG]B)')).text
                size = re.search('(\d+(?:[.,]\d+)*)\s?([KMG]B)', size)

                if size:
                    if size.group(2) == 'GB':
                        entry['content_size'] = int(float(size.group(1).replace(',', '')) * 1000 ** 3 / 1024 ** 2)
                    elif size.group(2) == 'MB':
                        entry['content_size'] = int(float(size.group(1).replace(',', '')) * 1000 ** 2 / 1024 ** 2)
                    elif size.group(2) == 'KB':
                        entry['content_size'] = int(float(size.group(1).replace(',', '')) * 1000 / 1024 ** 2)
                    else:
                        entry['content_size'] = int(float(size.group(1).replace(',', '')) / 1024 ** 2)

                entries.add(entry)

        return entries
示例#31
0
 def parse_download_page(self, url):
     try:
         page = requests.get(url).content
         soup = get_soup(page, "html.parser")
         download_link = soup.findAll(href=re.compile("redirect|redirectlink"))
         download_href = download_link[0]["href"]
         return download_href
     except Exception:
         raise UrlRewritingError("Unable to locate torrent from url %s" % url)
示例#32
0
 def get_token(self):
     # Don't use a session as tokens are not affected by domain limit
     try:
         r = get(self.base_url, params={'get_token': 'get_token', 'format': 'json'}).json()
         token = r.get('token')
         log.debug('RarBG token: %s' % token)
         return token
     except RequestException as e:
         log.debug('Could not retrieve RarBG token: %s', e.args[0])
示例#33
0
    def search(self, task, entry, config=None):
        """
        Search for name from torrentday.
        """

        categories = config.get('category', 'all')
        # Make sure categories is a list
        if not isinstance(categories, list):
            categories = [categories]
        # If there are any text categories, turn them into their id number
        categories = [c if isinstance(c, int) else CATEGORIES[c] for c in categories]
        params = { 'cata': 'yes', 'c%s' % ','.join(str(c) for c in categories): 1, 'clear-new': 1}
        entries = set()
        for search_string in entry.get('search_strings', [entry['title']]):

            url = 'https://www.torrentday.com/browse.php'
            params['search'] = normalize_unicode(search_string).replace(':', '')
            cookies = { 'uid': config['uid'], 'pass': config['passkey'], '__cfduid': config['cfduid'] }

            try:
                page = requests.get(url, params=params, cookies=cookies).content
            except RequestException as e:
                raise PluginError('Could not connect to torrentday: %s' % e)

            soup = get_soup(page)

            for tr in soup.find_all('tr', { 'class': 'browse' }):
                entry = Entry()
                # find the torrent names
                title = tr.find('a', { 'class': 'torrentName' })
                entry['title'] = title.contents[0]
                log.debug('title: %s', title.contents[0])

                # find download link
                torrent_url = tr.find('td', { 'class': 'dlLinksInfo' })
                torrent_url = torrent_url.find('a').get('href')

                # construct download URL
                torrent_url = ( 'https://www.torrentday.com/' + torrent_url + '?torrent_pass='******'rss_key'] )
                log.debug('RSS-ified download link: %s', torrent_url)
                entry['url'] = torrent_url

                # us tr object for seeders/leechers
                seeders, leechers = tr.find_all('td', { 'class': ['seedersInfo', 'leechersInfo']})
                entry['torrent_seeds'] = int(seeders.contents[0].replace(',', ''))
                entry['torrent_leeches'] = int(leechers.contents[0].replace(',', ''))
                entry['search_sort'] = torrent_availability(entry['torrent_seeds'], entry['torrent_leeches'])

                # use tr object for size
                size = tr.find('td', text=re.compile('([\.\d]+) ([TGMKk]?)B')).contents[0]
                size = re.search('([\.\d]+) ([TGMKk]?)B', str(size))

                entry['content_size'] = parse_filesize(size.group(0))

                entries.add(entry)

        return sorted(entries, reverse=True, key=lambda x: x.get('search_sort'))
示例#34
0
 def get_token(self):
     # Don't use a session as tokens are not affected by domain limit
     r = get('https://torrentapi.org/pubapi.php', params={'get_token': 'get_token', 'format': 'json'})
     token = None
     try:
         token = r.json().get('token')
     except ValueError:
         log.error('Could not retrieve RARBG token.')
     log.debug('RarBG token: %s' % token)
     return token
    def search(self, task, entry, config=None):
        """
            Search for entries on Serienjunkies
        """
        base_url = 'http://serienjunkies.org/search/'
        mull = {
            "Dauer:", "Download:", "Uploader:", u"Größe:", u"Tonhöhe:",
            "Sprache:", "Format:", "HQ-Cover:"
        }
        self.config = task.config.get('searchSerienjunkies') or {}
        self.config.setdefault('hoster', DEFHOS)
        self.config.setdefault('language', DEFLANG)

        entries = set()
        for search_string in entry.get('search_strings', [entry['title']]):
            query = normalize_unicode(search_string)
            query_url_fragment = urllib.quote(query.encode('utf8'))

            # http://serienjunkies.org/search/QUERY
            url = (base_url + query_url_fragment)
            log.debug('Serienjunkies search url: %s' % url)

            page = requests.get(url).content
            soup = get_soup(page)
            hoster = self.config['hoster']
            if self.config['language'] == 'english':
                english = True
            else:
                english = None
            for p in soup.find_all('p'):
                entry = Entry()
                if p.strong is not None and p.strong.text not in mull:
                    if english:
                        try:
                            if not p.strong.find(text=re.compile(
                                    "german", flags=re.IGNORECASE)):
                                link = p.find(
                                    text=re.compile(hoster)).find_previous('a')
                                entry['title'] = p.strong.text
                                entry['url'] = link.get('href')
                                entries.add(entry)
                        except:
                            pass
                    else:
                        try:
                            if p.strong.find(text=re.compile(
                                    "german", flags=re.IGNORECASE)):
                                link = p.find(
                                    text=re.compile(hoster)).find_previous('a')
                                entry['title'] = p.strong.text
                                entry['url'] = link.get('href')
                                entries.add(entry)
                        except:
                            pass
        return entries
示例#36
0
    def parse_download(self, series_url, search_title, config, entry):
        page = requests.get(series_url).content
        try:
            soup = get_soup(page)
        except Exception as e:
            raise UrlRewritingError(e)

        config = config or {}
        config.setdefault('hoster', 'ul')
        config.setdefault('language', 'en')

        # find matching download
        episode_title = soup.find('strong', text=search_title)
        if not episode_title:
            raise UrlRewritingError('Unable to find episode')

        # find download container
        episode = episode_title.parent
        if not episode:
            raise UrlRewritingError('Unable to find episode container')

        # find episode language
        episode_lang = episode.find_previous('strong', text=re.compile('Sprache')).next_sibling
        if not episode_lang:
            raise UrlRewritingError('Unable to find episode language')

        # filter language
        if config['language'] in ['de', 'both']:
            if not re.search('german|deutsch', episode_lang, flags=re.IGNORECASE):
                entry.reject('Language doesn\'t match selected')
                return series_url
        elif config['language'] in ['en', 'both']:
            if not re.search('englisc?h', episode_lang, flags=re.IGNORECASE):
                entry.reject('Language doesn\'t match selected')
                return series_url

        # find download links
        links = episode.find_all('a')
        if not links:
            raise UrlRewritingError('Unable to find download links')

        for link in links:
            if not link.has_attr('href'):
                continue

            url = link['href']
            pattern = 'http:\/\/download\.serienjunkies\.org.*%s_.*\.html' % config['hoster']

            if re.match(pattern, url):
                return url
            else:
                log.verbose('Hoster doesn\'t match selected')
                continue
        
        raise UrlRewritingError('URL-Rewriting failed, enable verbose logging for details.')
示例#37
0
 def parse_download_page(self, url):
     try:
         page = requests.get(url).content
         soup = get_soup(page, 'html.parser')
         download_link = soup.findAll(href=re.compile('redirect.php'))
         download_href = download_link[0]['href']
         return download_href[download_href.index('url=') + 4:]
     except Exception:
         raise UrlRewritingError(
             'Unable to locate torrent from url %s' % url
         )
示例#38
0
 def parse_download_page(self, url):
     try:
         page = requests.get(url).content
         soup = get_soup(page, 'html.parser')
         download_link = soup.findAll(
             href=re.compile('redirect|redirectlink'))
         download_href = download_link[0]['href']
         return download_href
     except Exception:
         raise UrlRewritingError('Unable to locate torrent from url %s' %
                                 url)
    def search(self, arg_entry, config=None):
        """
        Search for name from piratebay.
        """
        if not isinstance(config, dict):
            config = {}
        sort = SORT.get(config.get('sort_by', 'seeds'))
        if config.get('sort_reverse'):
            sort += 1
        if isinstance(config.get('category'), int):
            category = config['category']
        else:
            category = CATEGORIES.get(config.get('category', 'all'))
        filter_url = '/0/%d/%d' % (sort, category)

        entries = set()
        for search_string in arg_entry.get('search_string',
                                           [arg_entry['title']]):
            query = normalize_unicode(search_string)
            # TPB search doesn't like dashes
            query = query.replace('-', ' ')
            # urllib.quote will crash if the unicode string has non ascii characters, so encode in utf-8 beforehand
            url = 'http://thepiratebay.%s/search/%s%s' % (
                CUR_TLD, urllib.quote(query.encode('utf-8')), filter_url)
            log.debug('Using %s as piratebay search url' % url)
            page = requests.get(url).content
            soup = get_soup(page)
            for link in soup.find_all('a', attrs={'class': 'detLink'}):
                entry = Entry()
                entry['title'] = link.contents[0]
                entry['url'] = 'http://thepiratebay.%s%s' % (CUR_TLD,
                                                             link.get('href'))
                tds = link.parent.parent.parent.find_all('td')
                entry['torrent_seeds'] = int(tds[-2].contents[0])
                entry['torrent_leeches'] = int(tds[-1].contents[0])
                entry['search_sort'] = torrent_availability(
                    entry['torrent_seeds'], entry['torrent_leeches'])
                # Parse content_size
                size = link.find_next(attrs={'class': 'detDesc'}).contents[0]
                size = re.search('Size ([\.\d]+)\xa0([GMK])iB', size)
                if size:
                    if size.group(2) == 'G':
                        entry['content_size'] = int(
                            float(size.group(1)) * 1000**3 / 1024**2)
                    elif size.group(2) == 'M':
                        entry['content_size'] = int(
                            float(size.group(1)) * 1000**2 / 1024**2)
                    else:
                        entry['content_size'] = int(
                            float(size.group(1)) * 1000 / 1024**2)
                entries.add(entry)

        return sorted(
            entries, reverse=True, key=lambda x: x.get('search_sort'))
示例#40
0
 def parse_download_page(self, url):
     page = requests.get(url).content
     try:
         soup = get_soup(page)
         tag_div = soup.find('div', attrs={'class': 'download'})
         if not tag_div:
             raise UrlRewritingError('Unable to locate download link from url %s' % url)
         tag_a = tag_div.find('a')
         torrent_url = tag_a.get('href')
         return torrent_url
     except Exception, e:
         raise UrlRewritingError(e)
示例#41
0
def get_json(url):
    try:
        log.debug('fetching json at %s' % url)
        data = requests.get(url)
        result = data.json()
    except URLError as e:
        log.warning('Request failed %s' % url)
        return
    except ValueError:
        log.warning('Rotten Tomatoes returned invalid json at: %s' % url)
        return
    return result
示例#42
0
def get_json(url):
    try:
        log.debug('fetching json at %s' % url)
        data = requests.get(url)
        result = data.json()
    except URLError as e:
        log.warning('Request failed %s' % url)
        return
    except ValueError:
        log.warning('Rotten Tomatoes returned invalid json at: %s' % url)
        return
    return result
示例#43
0
    def parse_downloads(self, series_url, search_title):
        page = requests.get(series_url).content
        try:
            soup = get_soup(page)
        except Exception as e:
            raise UrlRewritingError(e)

        urls = []
        # find all titles
        episode_titles = self.find_all_titles(search_title)
        if not episode_titles:
            raise UrlRewritingError("Unable to find episode")

        for ep_title in episode_titles:
            # find matching download
            episode_title = soup.find("strong", text=re.compile(ep_title, re.I))
            if not episode_title:
                continue

            # find download container
            episode = episode_title.parent
            if not episode:
                continue

            # find episode language
            episode_lang = episode.find_previous("strong", text=re.compile("Sprache")).next_sibling
            if not episode_lang:
                log.warning("No language found for: %s", series_url)
                continue

            # filter language
            if not self.check_language(episode_lang):
                log.warning("languages not matching: %s <> %s", self.config["language"], episode_lang)
                continue

            # find download links
            links = episode.find_all("a")
            if not links:
                log.warning("No links found for: %s", series_url)
                continue

            for link in links:
                if not link.has_attr("href"):
                    continue

                url = link["href"]
                pattern = "http:\/\/download\.serienjunkies\.org.*%s_.*\.html" % self.config["hoster"]

                if re.match(pattern, url) or self.config["hoster"] == "all":
                    urls.append(url)
                else:
                    continue
        return urls
示例#44
0
    def search(self, entry, config=None):
        config = self.process_config(config)
        feed = REPUTATIONS[config['reputation']]
        entries = set()
        for search_string in entry.get('search_strings', [entry['title']]):
            query = normalize_unicode(search_string +
                                      config.get('extra_terms', ''))
            for domain in ['eu', 'me', 'ch', 'in']:
                # urllib.quote will crash if the unicode string has non ascii characters, so encode in utf-8 beforehand
                url = 'http://torrentz.%s/%s?q=%s' % (
                    domain, feed, urllib.quote(query.encode('utf-8')))
                log.debug('requesting: %s' % url)
                try:
                    r = requests.get(url)
                    break
                except requests.RequestException as err:
                    log.warning('torrentz.%s failed. Error: %s' %
                                (domain, err))
            else:
                raise plugin.PluginError(
                    'Error getting torrentz search results')

            if not r.content.strip():
                raise plugin.PluginError(
                    'No data from %s. Maybe torrentz is blocking the FlexGet User-Agent'
                    % url)

            rss = feedparser.parse(r.content)

            if rss.get('bozo_exception'):
                raise plugin.PluginError('Got bozo_exception (bad rss feed)')

            for item in rss.entries:
                m = re.search(
                    r'Size: ([\d]+) Mb Seeds: ([,\d]+) Peers: ([,\d]+) Hash: ([a-f0-9]+)',
                    item.description, re.IGNORECASE)
                if not m:
                    log.debug('regexp did not find seeds / peer data')
                    continue

                entry = Entry()
                entry['title'] = item.title
                entry['url'] = item.link
                entry['content_size'] = int(m.group(1))
                entry['torrent_seeds'] = int(m.group(2).replace(',', ''))
                entry['torrent_leeches'] = int(m.group(3).replace(',', ''))
                entry['torrent_info_hash'] = m.group(4).upper()
                entry['search_sort'] = torrent_availability(
                    entry['torrent_seeds'], entry['torrent_leeches'])
                entries.add(entry)

        log.debug('Search got %d results' % len(entries))
        return entries
示例#45
0
    def parse_downloads(self, series_url, search_title):
        page = requests.get(series_url).content
        try:
            soup = get_soup(page)
        except Exception as e:
            raise UrlRewritingError(e)
            
        urls = []
        # find all titles
        episode_titles = self.find_all_titles(search_title)
        if not episode_titles:
            raise UrlRewritingError('Unable to find episode')
        
        for ep_title in episode_titles:
            # find matching download
            episode_title = soup.find('strong', text=re.compile(ep_title, re.I))
            if not episode_title:
                continue
                
            # find download container
            episode = episode_title.parent
            if not episode:
                continue

            # find episode language
            episode_lang = episode.find_previous('strong', text=re.compile('Sprache')).next_sibling
            if not episode_lang:
                log.warning('No language found for: %s' % series_url)
                continue

            # filter language
            if not self.check_language(episode_lang):
                log.warning('languages not matching: %s <> %s' % (self.config['language'], episode_lang))
                continue

            # find download links
            links = episode.find_all('a')
            if not links:
                log.warning('No links found for: %s' % series_url)
                continue

            for link in links:
                if not link.has_attr('href'):
                    continue

                url = link['href']
                pattern = 'http:\/\/download\.serienjunkies\.org.*%s_.*\.html' % self.config['hoster']

                if re.match(pattern, url) or self.config['hoster'] == 'all':
                    urls.append(url)
                else:
                    continue
        return urls
示例#46
0
文件: yts.py 项目: pospqsjac/Flexget
    def search(self, task, entry, config=None):
        entries = set()
        search_strings = [
            normalize_unicode(s)
            for s in entry.get('search_strings', [entry['title']])
        ]
        for search_string in search_strings:
            url = 'https://yts.am/api/v2/list_movies.json?query_term=%s' % (
                quote(search_string.encode('utf-8')))

            log.debug('requesting: %s' % url)

            try:
                result = requests.get(url)
                try:
                    data = result.json()
                except ValueError:
                    log.debug('Could not decode json from response: %s',
                              result.text)
                    raise plugin.PluginError('Error getting result from yts.')
            except requests.RequestException as e:
                raise plugin.PluginError(
                    'Could not retrieve query from yts (%s)' % e.args[0])
            if not data['status'] == 'ok':
                raise plugin.PluginError('failed to query YTS')

            try:
                if data['data']['movie_count'] > 0:
                    for item in data['data']['movies']:
                        for torrent in item['torrents']:
                            entry = Entry()
                            entry['title'] = item['title']
                            entry['year'] = item['year']
                            entry['url'] = torrent['url']
                            entry['content_size'] = parse_filesize(
                                str(torrent['size_bytes']) + "b")
                            entry['torrent_seeds'] = torrent['seeds']
                            entry['torrent_leeches'] = torrent['peers']
                            entry['torrent_info_hash'] = torrent['hash']
                            entry[
                                'torrent_availability'] = torrent_availability(
                                    entry['torrent_seeds'],
                                    entry['torrent_leeches'])
                            entry['quality'] = torrent['quality']
                            entry['imdb_id'] = item['imdb_code']
                            if entry.isvalid():
                                entries.add(entry)
            except Exception:
                log.debug('invalid return structure from YTS')

        log.debug('Search got %d results' % len(entries))
        return entries
示例#47
0
 def get_url_content(self, url, params={}, method = "GET", json = False):
     if method is "GET":
         result = requests.get(url, params=params)
     elif method is "POST":
         result = requests.post(url, data=params)
     
     if json:
         if result.content:
             return result.json()
         else:
             return {}
     else:
         return BeautifulSoup(result.content, "html5lib")
示例#48
0
    def get_url_content(self, url, params={}, method="GET", json=False):
        if method is "GET":
            result = requests.get(url, params=params)
        elif method is "POST":
            result = requests.post(url, data=params)

        if json:
            if result.content:
                return result.json()
            else:
                return {}
        else:
            return BeautifulSoup(result.content, "html5lib")
示例#49
0
文件: thexem.py 项目: sisfs/dotfiles
 def on_task_metainfo(self, task, config): # NOQA
     if not config:
         return
     for entry in task.entries:
         if entry.get('series_id_type') == 'ep':
             if 'tvdb_id' in entry:
                 log.debug('The entry has a tvdb_id and will be used for mapping')
             else:
                 log.debug('The entry doesn\'t have tvdb_id, will check xem\'s list of shows')
                 response = requests.get('http://thexem.de/map/allNames?origin=tvdb&defaultNames=1&season=eq%s' % entry['series_season'])
                 shownames = json.loads(response.content)
                 for tvdb_id in shownames['data'].keys():
                     if entry['series_name'] in shownames['data'][tvdb_id]:
                         entry['tvdb_id'] = tvdb_id
                         log.debug('The tvdb_id for %s is %s', entry['series_name'], entry['tvdb_id'])
                         break
                 if 'tvdb_id' not in entry:
                     log.debug('An tvdb_id was not found, will search the xem\'s site')
                     response = requests.get('http://thexem.de/search?q=%s' % entry['series_name'])
                     if response.url.startswith('http://thexem.de/xem/show/'):
                         soup = get_soup(response.content)
                         try:
                             entry['tvdb_id'] = soup.findAll('a', {'href': re.compile('^http://thetvdb.com/\?tab=series')})[0].next
                             log.debug('The tvdb_id for %s is %s', entry['series_name'], entry['tvdb_id'])
                         except:
                             pass
                     if 'tvdb_id' not in entry:
                         log.error('Unable to find a tvdb_id for %s, manually specify a tvdb_id using set', entry['series_name'])
             response = requests.get('http://thexem.de/map/all?id=%s&origin=tvdb' % entry['tvdb_id'])
             episode_map = json.loads(response.content)
             for episode_entry in episode_map['data']:
                 if episode_entry[config['source']]['season'] == entry['series_season'] and episode_entry[config['source']]['episode'] == entry['series_episode']:
                     log.debug('An XEM entry was found for %s, %s episode S%02dE%02d maps to %s episode S%02dE%02d' % (entry['series_name'], config['source'], entry['series_season'], entry['series_episode'], config['destination'], episode_entry[config['destination']]['season'], episode_entry[config['destination']]['episode']))
                     if 'description' in entry:
                         entry['description'] = entry['description'].replace('Season: %s; Episode: %s' % (entry['series_season'], entry['series_episode']), 'Season: %s; Episode: %s' % (episode_entry[config['destination']]['season'], episode_entry[config['destination']]['episode']))
                     entry['series_season'] = episode_entry[config['destination']]['season']
                     entry['series_episode'] = episode_entry[config['destination']]['episode']
                     entry['series_id'] = 'S%02dE%02d' % (episode_entry[config['destination']]['season'], episode_entry[config['destination']]['episode'])
                     break
示例#50
0
 def get_token(self):
     # Don't use a session as tokens are not affected by domain limit
     r = get(self.base_url,
             params={
                 'get_token': 'get_token',
                 'format': 'json'
             })
     token = None
     try:
         token = r.json().get('token')
     except ValueError:
         log.error('Could not retrieve RARBG token.')
     log.debug('RarBG token: %s' % token)
     return token
示例#51
0
 def parse_download_page(self, url):
     page = requests.get(url,verify=False).content
     try:
         soup = get_soup(page)
         tag_div = soup.find('div', attrs={'class': 'download'})
         if not tag_div:
             raise UrlRewritingError('Unable to locate download link from url %s' % url)
         tag_a = tag_div.find('a')
         torrent_url = tag_a.get('href')
         # URL is sometimes missing the schema
         if torrent_url.startswith('//'):
             torrent_url = 'http:' + torrent_url
         return torrent_url
     except Exception as e:
         raise UrlRewritingError(e)
示例#52
0
def tvmaze_lookup(lookup_type, lookup_values):
    """
    Build the URL and return the reply from TVMaze API
    :param lookup_type: Selects the endpoint that will be used
    :param lookup_values: A list of values to be used in the URL
    :return: A JSON reply from the API
    """
    lookup_url = BASE_URL + TVMAZE_ENDPOINTS[lookup_type].format(
        *lookup_values)
    log.debug('querying tvmaze API with the following URL: %s', lookup_url)
    try:
        result = requests.get(lookup_url).json()
    except HTTPError as e:
        raise LookupError(e)
    return result
示例#53
0
 def __call__(self, parser, namespace, values, option_string=None):
     # Print the version number
     console('%s' % self.version)
     # Check for latest version from server
     try:
         page = requests.get('http://download.flexget.com/latestversion')
     except requests.RequestException:
         console('Error getting latest version number from download.flexget.com')
     else:
         ver = page.text.strip()
         if self.version == ver:
             console('You are on the latest release.')
         else:
             console('Latest release: %s' % ver)
     parser.exit()
示例#54
0
def tvmaze_lookup(lookup_url, **kwargs):
    """
    Build the URL and return the reply from TVMaze API

    :param lookup_type: Selects the endpoint that will be used
    :param lookup_values: A list of values to be used in the URL
    :return: A JSON reply from the API
    """
    url = BASE_URL + lookup_url
    log.debug('querying tvmaze API with the following URL: %s', url)
    try:
        result = requests.get(url, **kwargs).json()
    except RequestException as e:
        raise LookupError(e.args[0])
    return result
示例#55
0
 def url_from_page(self, url):
     """Parses torrent url from newtorrents download page"""
     try:
         page = requests.get(url)
         data = page.text
     except requests.RequestException:
         raise UrlRewritingError('URLerror when retrieving page')
     p = re.compile(r"copy\(\'(.*)\'\)", re.IGNORECASE)
     f = p.search(data)
     if not f:
         # the link in which plugin relies is missing!
         raise UrlRewritingError(
             'Failed to get url from download page. Plugin may need a update.'
         )
     else:
         return f.group(1)
示例#56
0
文件: pogcal.py 项目: s-m-b/Flexget
 def on_task_input(self, task, config):
     try:
         r = requests.post('http://www.pogdesign.co.uk/cat/',
                           data={
                               'username': config['username'],
                               'password': config['password'],
                               'sub_login': '******'
                           },
                           allow_redirects=True)
         if 'U / P Invalid' in r.text:
             raise plugin.PluginError(
                 'Invalid username/password for pogdesign.')
         page = requests.get(
             'http://www.pogdesign.co.uk/cat/showselect.php',
             cookies=r.cookies)
     except requests.RequestException, e:
         raise plugin.PluginError('Error retrieving source: %s' % e)
示例#57
0
    def parse_download_page(self, url, task):
        log.verbose('Descargas2020 URL: %s', url)

        try:
            page = requests.get(url)
        except requests.exceptions.RequestException as e:
            raise UrlRewritingError(e)
        try:
            soup = get_soup(page.text)
        except Exception as e:
            raise UrlRewritingError(e)

        torrent_id = None
        url_format = DESCARGAS2020_TORRENT_FORMAT

        torrent_id_prog = re.compile(
            "(?:parametros\s*=\s*\n?)\s*{\s*\n(?:\s*'\w+'\s*:.*\n)+\s*'(?:torrentID|id)"
            "'\s*:\s*'(\d+)'")
        torrent_ids = soup.findAll(text=torrent_id_prog)
        if torrent_ids:
            match = torrent_id_prog.search(torrent_ids[0])
            if match:
                torrent_id = match.group(1)
        if not torrent_id:
            log.debug('torrent ID not found, searching openTorrent script')
            torrent_id_prog = re.compile(
                'function openTorrent.*\n.*\{.*(\n.*)+window\.location\.href =\s*\"(.*\/\d+_-.*[^\/])\/?\";'
            )
            torrent_ids = soup.findAll(text=torrent_id_prog)
            if torrent_ids:
                match = torrent_id_prog.search(torrent_ids[0])
                if match:
                    torrent_id = match.group(2)
                    return torrent_id.replace('descargar-torrent',
                                              'download') + '.torrent'

        if not torrent_id:
            raise UrlRewritingError('Unable to locate torrent ID from url %s' %
                                    url)

        return url_format.format(torrent_id)