def inject_series(self, release_name): self.execute_task( 'inject_series', options={'inject': [Entry(title=release_name, url='')]})
def search(self, task, entry, config=None): """ Search for name from torrentleech. """ request_headers = {'User-Agent': 'curl/7.54.0'} rss_key = config['rss_key'] # build the form request: data = {'username': config['username'], 'password': config['password']} # POST the login form: try: login = task.requests.post( 'https://www.torrentleech.org/user/account/login/', data=data, headers=request_headers, allow_redirects=True) except RequestException as e: raise PluginError('Could not connect to torrentleech: %s', str(e)) if not isinstance(config, dict): config = {} # sort = SORT.get(config.get('sort_by', 'seeds')) # if config.get('sort_reverse'): # sort += 1 categories = config.get('category', 'all') # Make sure categories is a list if not isinstance(categories, list): categories = [categories] # If there are any text categories, turn them into their id number categories = [ c if isinstance(c, int) else CATEGORIES[c] for c in categories ] filter_url = '/categories/{}'.format(','.join( str(c) for c in categories)) entries = set() for search_string in entry.get('search_strings', [entry['title']]): query = normalize_unicode(search_string).replace(":", "") # urllib.quote will crash if the unicode string has non ascii characters, # so encode in utf-8 beforehand url = ('https://www.torrentleech.org/torrents/browse/list/query/' + quote(query.encode('utf-8')) + filter_url) log.debug('Using %s as torrentleech search url', url) results = task.requests.get(url, headers=request_headers, cookies=login.cookies).json() for torrent in results['torrentList']: entry = Entry() entry['download_headers'] = request_headers entry['title'] = torrent['name'] # construct download URL torrent_url = 'https://www.torrentleech.org/rss/download/{}/{}/{}'.format( torrent['fid'], rss_key, torrent['filename']) log.debug('RSS-ified download link: %s', torrent_url) entry['url'] = torrent_url # seeders/leechers entry['torrent_seeds'] = torrent['seeders'] entry['torrent_leeches'] = torrent['leechers'] entry['search_sort'] = torrent_availability( entry['torrent_seeds'], entry['torrent_leeches']) entry['content_size'] = parse_filesize( str(torrent['size']) + ' b') entries.add(entry) return sorted(entries, reverse=True, key=lambda x: x.get('search_sort'))
def search(self, task, entry, config): """ Search for entries on Limetorrents """ if not isinstance(config, dict): config = {'category': config} order_by = '' if isinstance(config.get('order_by'), str): if config['order_by'] != 'date': order_by = '{0}/1'.format(config['order_by']) category = 'all' if isinstance(config.get('category'), str): category = '{0}'.format(config['category']) entries = set() for search_string in entry.get('search_strings', [entry['title']]): # No special characters - use dashes instead of %20 cleaned_search_string = clean_symbols(search_string).replace( ' ', '-') query = 'search/{0}/{1}/{2}'.format( category, cleaned_search_string.encode('utf8'), order_by) logger.debug( 'Using search: {}; category: {}; ordering: {}', cleaned_search_string, category, order_by or 'default', ) try: page = task.requests.get(self.base_url + query) logger.debug('requesting: {}', page.url) except RequestException as e: logger.error('Limetorrents request failed: {}', e) continue soup = get_soup(page.content) if soup.find('a', attrs={'class': 'csprite_dl14'}) is not None: for link in soup.findAll('a', attrs={'class': 'csprite_dl14'}): row = link.find_parent('tr') info_url = str(link.get('href')) # Get the title from the URL as it's complete versus the actual Title text which gets cut off title = str(link.next_sibling.get('href')) title = title[:title.rfind('-torrent')].replace('-', ' ') title = title[1:] data = row.findAll('td', attrs={'class': 'tdnormal'}) size = str(data[1].text).replace(',', '') seeds = int( row.find('td', attrs={ 'class': 'tdseed' }).text.replace(',', '')) leeches = int( row.find('td', attrs={ 'class': 'tdleech' }).text.replace(',', '')) size = parse_filesize(size) e = Entry() e['url'] = info_url e['title'] = title e['torrent_seeds'] = seeds e['torrent_leeches'] = leeches e['torrent_availability'] = torrent_availability( e['torrent_seeds'], e['torrent_leeches']) e['content_size'] = size entries.add(e) return entries
def post(self, session=None): """ Execute task and stream results """ data = request.json for task in data.get('tasks'): if task.lower() not in [ t.lower() for t in self.manager.user_config.get('tasks', {}).keys() ]: raise NotFoundError('task %s does not exist' % task) queue = ExecuteLog() output = queue if data.get('loglevel') else None stream = ( True if any( arg[0] in ['progress', 'summary', 'loglevel', 'entry_dump'] for arg in data.items() if arg[1] ) else False ) loglevel = data.pop('loglevel', None) # This emulates the CLI command of using `--now` and `no-cache` options = { 'interval_ignore': data.pop('now', None), 'nocache': data.pop('no_cache', None), 'allow_manual': True, } for option, value in data.items(): options[option] = value if data.get('inject'): entries = [] for item in data.get('inject'): entry = Entry() entry['url'] = item['url'] if not item.get('title'): try: value, params = cgi.parse_header( requests.head(item['url']).headers['Content-Disposition'] ) entry['title'] = params['filename'] except KeyError: raise BadRequest( 'No title given, and couldn\'t get one from the URL\'s HTTP response' ) else: entry['title'] = item.get('title') if item.get('force'): entry['immortal'] = True if item.get('accept'): entry.accept(reason='accepted by API inject') if item.get('fields'): for key, value in item.get('fields').items(): entry[key] = value entries.append(entry) options['inject'] = entries executed_tasks = self.manager.execute(options=options, output=output, loglevel=loglevel) tasks_queued = [] for task_id, task_name, task_event in executed_tasks: tasks_queued.append({'id': task_id, 'name': task_name, 'event': task_event}) _streams[task_id] = {'queue': queue, 'last_update': datetime.now(), 'args': data} if not stream: return jsonify( {'tasks': [{'id': task['id'], 'name': task['name']} for task in tasks_queued]} ) def stream_response(): # First return the tasks to execute yield '{"stream": [' yield json.dumps( {'tasks': [{'id': task['id'], 'name': task['name']} for task in tasks_queued]} ) + ',\n' while True: try: yield queue.get(timeout=1) + ',\n' continue except Empty: pass if queue.empty() and all([task['event'].is_set() for task in tasks_queued]): for task in tasks_queued: del _streams[task['id']] break yield '{}]}' return Response(stream_response(), mimetype='text/event-stream')
def entries_from_search(self, name, url=None, comparator=StringComparator(cutoff=0.9)): """Parses torrent download url from search results""" comparator.set_seq1(name) name = comparator.search_string() if not url: url = 'http://www.newtorrents.info/search/%s' % urllib.quote( name.encode('utf-8'), safe=':/~?=&%') log.debug('search url: %s' % url) html = urlopener(url, log).read() # fix </SCR'+'IPT> so that BS does not crash # TODO: should use beautifulsoup massage html = re.sub(r'(</SCR.*?)...(.*?IPT>)', r'\1\2', html) soup = get_soup(html) # saving torrents in dict torrents = [] for link in soup.find_all('a', attrs={'href': re.compile('down.php')}): torrent_url = 'http://www.newtorrents.info%s' % link.get('href') release_name = link.parent.next.get('title') # quick dirty hack seed = link.find_next('td', attrs={ 'class': re.compile('s') }).renderContents() if seed == 'n/a': seed = 0 else: try: seed = int(seed) except ValueError: log.warning( 'Error converting seed value (%s) from newtorrents to integer.' % seed) seed = 0 #TODO: also parse content_size and peers from results if comparator.matches(release_name): torrents.append( Entry(title=release_name, url=torrent_url, torrent_seeds=seed, search_ratio=comparator.ratio(), search_sort=torrent_availability(seed, 0))) else: log.debug('rejecting search result: %s !~ %s' % (release_name, name)) # sort with seed number Reverse order torrents.sort(reverse=True, key=lambda x: x.get('search_sort', 0)) # choose the torrent if not torrents: dashindex = name.rfind('-') if dashindex != -1: return self.entries_from_search(name[:dashindex], comparator=comparator) else: raise PluginWarning('No matches for %s' % name, log, log_once=True) else: if len(torrents) == 1: log.debug('found only one matching search result.') else: log.debug( 'search result contains multiple matches, sorted %s by most seeders' % torrents) return torrents
def search(self, task, entry, config=None): """ Search for name from torrent411. """ url_base = 'http://www.t411.in' if not isinstance(config, dict): config = {} category = config.get('category') if category in list(CATEGORIES): category = CATEGORIES[category] sub_categories = config.get('sub_category') if not isinstance(sub_categories, list): sub_categories = [sub_categories] filter_url = '' if isinstance(category, int): filter_url = '&cat=%s' % str(category) if sub_categories[0] is not None: sub_categories = [SUB_CATEGORIES[c] for c in sub_categories] filter_url = filter_url + '&' + '&'.join([ urllib.quote_plus('term[%s][]' % c[0]).encode('utf-8') + '=' + str(c[1]) for c in sub_categories ]) entries = set() for search_string in entry.get('search_strings', [entry['title']]): query = normalize_unicode(search_string) url_search = ('/torrents/search/?search=%40name+' + urllib.quote_plus(query.encode('utf-8')) + filter_url) opener = urllib2.build_opener() opener.addheaders = [('User-agent', 'Mozilla/5.0')] response = opener.open(url_base + url_search) data = response.read() soup = get_soup(data) tb = soup.find("table", class_="results") if not tb: continue for tr in tb.findAll('tr')[1:][:-1]: entry = Entry() nfo_link_res = re.search('torrents/nfo/\?id=(\d+)', str(tr)) if nfo_link_res is not None: tid = nfo_link_res.group(1) title_res = re.search( '<a href=\"//www.t411.in/torrents/([-A-Za-z0-9+&@#/%|?=~_|!:,.;]+)\" title="([^"]*)">', str(tr)) if title_res is not None: entry['title'] = title_res.group(2).decode('utf-8') size = tr('td')[5].contents[0] entry[ 'url'] = 'http://www.t411.in/torrents/download/?id=%s' % tid entry['torrent_seeds'] = tr('td')[7].contents[0] entry['torrent_leeches'] = tr('td')[8].contents[0] entry['search_sort'] = torrent_availability( entry['torrent_seeds'], entry['torrent_leeches']) size = re.search('([\.\d]+) ([GMK]?)B', size) if size: if size.group(2) == 'G': entry['content_size'] = int( float(size.group(1)) * 1000**3 / 1024**2) elif size.group(2) == 'M': entry['content_size'] = int( float(size.group(1)) * 1000**2 / 1024**2) elif size.group(2) == 'K': entry['content_size'] = int( float(size.group(1)) * 1000 / 1024**2) else: entry['content_size'] = int( float(size.group(1)) / 1024**2) auth_handler = t411Auth(config['username'], config['password']) entry['download_auth'] = auth_handler entries.add(entry) return sorted(entries, reverse=True, key=lambda x: x.get('search_sort'))
def test_next_series_episodes_with_uncompleted_season(self, execute_task): execute_task( 'inject_series', options={'inject': [Entry(title='My Show 1 S02 480p', url='')]}) task = execute_task('test_next_series_episodes_with_unaccepted_season') assert task.find_entry(title='My Show 1 S02E01')
def search(self, task, entry, config): """ Search for entries on RarBG """ categories = config.get('category', 'all') # Ensure categories a list if not isinstance(categories, list): categories = [categories] # Convert named category to its respective category id number categories = [c if isinstance(c, int) else CATEGORIES[c] for c in categories] category_url_fragment = ';'.join(str(c) for c in categories) entries = set() token = self.get_token() if not token: log.error('Could not retrieve token. Abandoning search.') return entries params = {'mode': 'search', 'token': token, 'ranked': int(config['ranked']), 'min_seeders': config['min_seeders'], 'min_leechers': config['min_leechers'], 'sort': config['sorted_by'], 'category': category_url_fragment, 'format': 'json_extended', 'app_id': 'flexget'} for search_string in entry.get('search_strings', [entry['title']]): params.pop('search_string', None) params.pop('search_imdb', None) params.pop('search_tvdb', None) if entry.get('movie_name'): params['search_imdb'] = entry.get('imdb_id') else: query = normalize_scene(search_string) query_url_fragment = query.encode('utf8') params['search_string'] = query_url_fragment if config['use_tvdb']: plugin.get_plugin_by_name('thetvdb_lookup').instance.lazy_series_lookup(entry) params['search_tvdb'] = entry.get('tvdb_id') log.debug('Using tvdb id %s', entry.get('tvdb_id')) try: page = requests.get(self.base_url, params=params) log.debug('requesting: %s', page.url) except RequestException as e: log.error('RarBG request failed: %s' % e.args[0]) continue r = page.json() # error code 20 just means no results were found if r.get('error_code') == 20: searched_string = params.get('search_string') or 'imdb={0}'.format(params.get('search_imdb')) or \ 'tvdb={0}'.format(params.get('tvdb_id')) log.debug('No results found for %s', searched_string) continue elif r.get('error'): log.error('Error code %s: %s', r.get('error_code'), r.get('error')) continue else: for result in r.get('torrent_results'): e = Entry() e['title'] = result.get('title') e['url'] = result.get('download') e['torrent_seeds'] = int(result.get('seeders')) e['torrent_leeches'] = int(result.get('leechers')) e['content_size'] = int(result.get('size')) / 1024 / 1024 episode_info = result.get('episode_info') if episode_info: e['imdb_id'] = episode_info.get('imdb') e['tvdb_id'] = episode_info.get('tvdb') e['tvrage_id'] = episode_info.get('tvrage') entries.add(e) return entries
def on_task_input(self, task, config): # Let details plugin know that it is ok if this task doesn't produce any entries task.no_entries_ok = True filename = os.path.expanduser(config['file']) encoding = config.get('encoding', 'utf-8') with Session() as session: db_pos = (session.query(TailPosition).filter( TailPosition.task == task.name).filter( TailPosition.filename == filename).first()) if db_pos: last_pos = db_pos.position else: last_pos = 0 with open(filename, 'r', encoding=encoding, errors='replace') as file: if task.options.tail_reset == filename or task.options.tail_reset == task.name: if last_pos == 0: logger.info('Task {} tail position is already zero', task.name) else: logger.info('Task {} tail position ({}) reset to zero', task.name, last_pos) last_pos = 0 if os.path.getsize(filename) < last_pos: logger.info( 'File size is smaller than in previous execution, resetting to beginning of the file' ) last_pos = 0 file.seek(last_pos) logger.debug('continuing from last position {}', last_pos) entry_config = config.get('entry') format_config = config.get('format', {}) # keep track what fields have been found used = {} entries = [] entry = Entry() # now parse text for line in file: if not line: break for field, regexp in entry_config.items(): # log.debug('search field: %s regexp: %s' % (field, regexp)) match = re.search(regexp, line) if match: # check if used field detected, in such case start with new entry if field in used: if entry.isvalid(): logger.info( 'Found field {} again before entry was completed. Adding current incomplete, but valid entry and moving to next.', field, ) self.format_entry(entry, format_config) entries.append(entry) else: logger.info( 'Invalid data, entry field {} is already found once. Ignoring entry.', field, ) # start new entry entry = Entry() used = {} # add field to entry entry[field] = match.group(1) used[field] = True logger.debug('found field: {} value: {}', field, entry[field]) # if all fields have been found if len(used) == len(entry_config): # check that entry has at least title and url if not entry.isvalid(): logger.info( 'Invalid data, constructed entry is missing mandatory fields (title or url)' ) else: self.format_entry(entry, format_config) entries.append(entry) logger.debug('Added entry {}', entry) # start new entry entry = Entry() used = {} last_pos = file.tell() if db_pos: db_pos.position = last_pos else: session.add( TailPosition(task=task.name, filename=filename, position=last_pos)) return entries
def get(self, entry): with Session() as session: match = self._entry_query(session=session, entry=entry, approved=True) return Entry(match.entry) if match else None
def test_encoding(self): e = Entry('title', 'url') with pytest.raises(EntryUnicodeError): e['invalid'] = b'\x8e'
def getter(self): return Entry(json.loads(getattr(self, name), decode_datetime=True))
def search(self, task, entry, config): """ Search for entries on AwesomeHD """ # need lxml to parse xml try: import lxml # noqa except ImportError as e: log.debug('Error importing lxml: %s', e) raise plugin.DependencyError( 'awesomehd', 'lxml', 'lxml module required. ImportError: %s' % e) config = self.prepare_config(config) # set a domain limit, but allow the user to overwrite it if 'awesome-hd.me' not in task.requests.domain_limiters: task.requests.add_domain_limiter( TimedLimiter('awesome-hd.me', '5 seconds')) entries = set() # Can only search for imdb if not entry.get('imdb_id'): log.debug('Skipping entry %s because of missing imdb id', entry['title']) return entries # Standard search params params = { 'passkey': config['passkey'], 'internal': int(config['only_internal']), 'action': 'imdbsearch', 'imdb': entry['imdb_id'] } try: response = task.requests.get(self.base_url + 'searchapi.php', params=params).content except RequestException as e: log.error('Failed to search for imdb id %s: %s', entry['imdb_id'], e) return entries try: soup = get_soup(response, 'xml') if soup.find('error'): log.error(soup.find('error').get_text()) return entries except Exception as e: log.error('Failed to parse xml result for imdb id %s: %s', entry['imdb_id'], e) return entries authkey = soup.find('authkey').get_text() for result in soup.find_all('torrent'): # skip audio releases for now if not result.find('resolution').get_text(): log.debug('Skipping audio release') continue e = Entry() e['imdb_id'] = result.find('imdb').get_text() e['torrent_id'] = int(result.find('id').get_text()) e['uploaded_at'] = dateutil_parse(result.find('time').get_text()) e['content_size'] = parse_filesize('{} b'.format( result.find('size').get_text())) e['torrent_snatches'] = int(result.find('snatched').get_text()) e['torrent_seeds'] = int(result.find('seeders').get_text()) e['torrent_leeches'] = int(result.find('leechers').get_text()) e['release_group'] = result.find('releasegroup').get_text() e['freeleech_percent'] = int( (1 - float(result.find('freeleech').get_text())) * 100) e['encode_status'] = result.find('encodestatus').get_text() e['subtitles'] = result.find('subtitles').get_text().split(', ') e['url'] = self.base_url + 'torrents.php?action=download&id={}&authkey={}&torrent_pass={}'\ .format(e['torrent_id'], authkey, config['passkey']) # Generate a somewhat sensible title audio = result.find('audioformat').get_text().replace( 'AC-3', 'AC3') # normalize a bit source = result.find('media').get_text() encoder = result.find('encoding').get_text() # calling a WEB-DL a remux is pretty redundant if 'WEB' in source.upper(): encoder = re.sub('REMUX', '', encoder, flags=re.IGNORECASE).strip() e['title'] = '{movie_name} {year} {resolution} {source} {audio} {encoder}-{release_group}'\ .format(movie_name=result.find('name').get_text(), year=result.find('year').get_text(), resolution=result.find('resolution').get_text(), source=source, audio=audio, encoder=encoder, release_group=e['release_group']) entries.add(e) return entries
def on_task_input(self, task, config): if not task.requests.cookies: username = config['username'] password = config['password'] log.debug("Logging in to %s ..." % URL) params = { 'username': username, 'password': password, 'action': 'Login' } try: loginsrc = task.requests.post(URL + 'login.php', data=params) if 'login' in loginsrc.url: raise plugin.PluginWarning( ('Login to myepisodes.com failed, please check ' 'your account data or see if the site is down.'), log, ) except RequestException as e: raise plugin.PluginError("Error logging in to myepisodes: %s" % e) page = task.requests.get(URL + "myshows/manage/").content try: soup = get_soup(page) except Exception as e: raise plugin.PluginError( "Unable to parse myepisodes.com page: %s" % (e, )) entries = [] def show_list(select_id): return soup.find('select', {'id': select_id}).findAll('option') options = show_list('shows') if config['include_ignored']: options = chain(options, show_list('ignored_shows')) for option in options: name = option.text if config.get('strip_dates'): # Remove year from end of name if present name = re.sub(r'\s+\(\d{4}\)$', '', name) showid = option.get('value') url = '%sviews.php?type=epsbyshow&showid=%s' % (URL, showid) entry = Entry() entry['title'] = name entry['url'] = url entry['series_name'] = name entry['myepisodes_id'] = showid if entry.isvalid(): entries.append(entry) else: log.debug('Invalid entry created? %s' % entry) if not entries: log.warning( "No shows found on myepisodes.com list. Maybe you need to add some first?" ) return entries
def _parse_tiles(self, task, config, tiles, series_info): max_age = config.get('max_episode_age_days') download_premium = config.get('download_premium') entries = [] if tiles is not None: for tile in tiles: # there is only one list_item per tile for list_item in get_soup(tile).findAll( 'div', class_='npo-asset-tile-container'): episode_id = list_item['data-id'] premium = 'npo-premium-content' in list_item['class'] log.debug('Parsing episode: %s', episode_id) url = list_item.find('a')['href'] # Check if the URL found to the episode matches the expected pattern if len(url.split('/')) != 6: log.verbose( 'Skipping %s, the URL has an unexpected pattern: %s', episode_id, url) continue # something is wrong; skip this episode episode_name = list_item.find('h2') if episode_name: title = '{} ({})'.format( next(episode_name.stripped_strings), episode_id) else: title = '{}'.format(episode_id) timer = '0' timerdiv = list_item.find('div', class_='npo-asset-tile-timer') if timerdiv: # some entries are missing a running time timer = next(timerdiv.stripped_strings) remove_url = list_item.find('div', class_='npo-asset-tile-delete') if premium and not download_premium: log.debug( 'Skipping %s, no longer available without premium', title) continue entry_date = url.split('/')[-2] entry_date = self._parse_date(entry_date) if max_age >= 0 and date.today() - entry_date > timedelta( days=max_age): log.debug('Skipping %s, aired on %s', title, entry_date) continue e = Entry(series_info) e['url'] = url e['title'] = title e['series_name'] = series_info['npo_name'] e['series_name_plain'] = self._convert_plain( series_info['npo_name']) e['series_date'] = entry_date e['series_id_type'] = 'date' e['npo_id'] = episode_id e['npo_premium'] = premium e['npo_runtime'] = timer.strip('min').strip() e['language'] = series_info['npo_language'] if remove_url and remove_url['data-link']: e['remove_url'] = remove_url['data-link'] if config.get('remove_accepted'): e.on_complete(self.entry_complete, task=task) entries.append(e) return entries
def wrapped_func(*args, **kwargs): # get task from method parameters task = args[1] # detect api version api_ver = 1 if len(args) == 3: api_ver = 2 if api_ver == 1: # get name for a cache from tasks configuration if not self.name in task.config: raise Exception( '@cache config name %s is not configured in task %s' % (self.name, task.name)) hash = config_hash(task.config[self.name]) else: hash = config_hash(args[2]) log.trace('self.name: %s' % self.name) log.trace('hash: %s' % hash) cache_name = self.name + '_' + hash log.debug('cache name: %s (has: %s)' % (cache_name, ', '.join(self.cache.keys()))) if cache_name in self.cache: # return from the cache log.trace('cache hit') entries = [] for entry in self.cache[cache_name]: fresh = copy.deepcopy(entry) entries.append(fresh) if entries: log.verbose('Restored %s entries from cache' % len(entries)) return entries else: if self.persist and not task.manager.options.nocache: # Check database cache db_cache = task.session.query(InputCache).filter(InputCache.name == self.name).\ filter(InputCache.hash == hash).\ filter(InputCache.added > datetime.now() - self.persist).\ first() if db_cache: entries = [Entry(e.entry) for e in db_cache.entries] log.verbose('Restored %s entries from db cache' % len(entries)) # Store to in memory cache self.cache[cache_name] = copy.deepcopy(entries) return entries # Nothing was restored from db or memory cache, run the function log.trace('cache miss') # call input event try: response = func(*args, **kwargs) except PluginError, e: # If there was an error producing entries, but we have valid entries in the db cache, return those. if self.persist and not task.manager.options.nocache: db_cache = task.session.query(InputCache).filter(InputCache.name == self.name).\ filter(InputCache.hash == hash).first() if db_cache and db_cache.entries: log.error( 'There was an error during %s input (%s), using cache instead.' % (self.name, e)) entries = [ Entry(e.entry) for e in db_cache.entries ] log.verbose('Restored %s entries from db cache' % len(entries)) # Store to in memory cache self.cache[cache_name] = copy.deepcopy(entries) return entries # If there was nothing in the db cache, re-raise the error. raise if api_ver == 1: response = task.entries if not isinstance(response, list): log.warning( 'Input %s did not return a list, cannot cache.' % self.name) return response # store results to cache log.debug('storing to cache %s %s entries' % (cache_name, len(response))) try: self.cache[cache_name] = copy.deepcopy(response) except TypeError: # might be caused because of backlog restoring some idiotic stuff, so not neccessarily a bug log.critical( 'Unable to save task content into cache, if problem persists longer than a day please report this as a bug' ) if self.persist: # Store to database log.debug('Storing cache %s to database.' % cache_name) db_cache = task.session.query(InputCache).filter(InputCache.name == self.name).\ filter(InputCache.hash == hash).first() if not db_cache: db_cache = InputCache(name=self.name, hash=hash) db_cache.entries = [ InputCacheEntry(entry=e) for e in response ] db_cache.added = datetime.now() task.session.merge(db_cache) return response
def create_entries(self, page_url, soup, config): queue = [] duplicates = {} duplicate_limit = 4 def title_exists(title): """Helper method. Return True if title is already added to entries""" for entry in queue: if entry['title'] == title: return True for link in soup.find_all('a'): # not a valid link if not link.has_attr('href'): continue # no content in the link if not link.contents and not config.get('allow_empty_links', False): continue url = link['href'] # fix broken urls if url.startswith('//'): url = 'http:' + url elif not url.startswith('http://') or not url.startswith( 'https://'): url = parse.urljoin(page_url, url) log_link = url log_link = log_link.replace('\n', '') log_link = log_link.replace('\r', '') # get only links matching regexp regexps = config.get('links_re', None) if regexps: accept = False for regexp in regexps: if re.search(regexp, url): accept = True if not accept: log.debug('url does not match any "links_re": %s' % url) continue title_from = config.get('title_from', 'auto') if title_from == 'url': title = self._title_from_url(url) log.debug('title from url: %s' % title) elif title_from == 'title': if not link.has_attr('title'): log.warning( 'Link `%s` doesn\'t have title attribute, ignored.' % log_link) continue title = link['title'] log.debug('title from title: %s' % title) elif title_from == 'auto': title = self._title_from_link(link, log_link) if title is None: continue # automatic mode, check if title is unique # if there are too many duplicate titles, switch to title_from: url if title_exists(title): # ignore index links as a counter if 'index' in title and len(title) < 10: log.debug('ignored index title %s' % title) continue duplicates.setdefault(title, 0) duplicates[title] += 1 if duplicates[title] > duplicate_limit: # if from url seems to be bad choice use title from_url = self._title_from_url(url) switch_to = 'url' for ext in ('.html', '.php'): if from_url.endswith(ext): switch_to = 'title' log.info( 'Link names seem to be useless, auto-configuring \'title_from: %s\'. ' 'This may not work well, you might need to configure it yourself.' % switch_to) config['title_from'] = switch_to # start from the beginning ... return self.create_entries(page_url, soup, config) elif title_from == 'link' or title_from == 'contents': # link from link name title = self._title_from_link(link, log_link) if title is None: continue log.debug('title from link: %s' % title) else: raise plugin.PluginError('Unknown title_from value %s' % title_from) if not title: log.warning('title could not be determined for link %s' % log_link) continue # strip unicode white spaces title = title.replace(u'\u200B', u'').strip() # in case the title contains xxxxxxx.torrent - foooo.torrent clean it a bit (get up to first .torrent) # TODO: hack if title.lower().find('.torrent') > 0: title = title[:title.lower().find('.torrent')] if title_exists(title): # title link should be unique, add CRC32 to end if it's not hash = zlib.crc32(url.encode("utf-8")) crc32 = '%08X' % (hash & 0xFFFFFFFF) title = '%s [%s]' % (title, crc32) # truly duplicate, title + url crc already exists in queue if title_exists(title): continue log.debug('uniqued title to %s' % title) entry = Entry() entry['url'] = url entry['title'] = title if 'username' in config and 'password' in config: entry['download_auth'] = (config['username'], config['password']) queue.append(entry) # add from queue to task return queue
def search(self, task, entry, config=None): """ Search for name from iptorrents """ categories = config.get('category', 'All') # Make sure categories is a list if not isinstance(categories, list): categories = [categories] # If there are any text categories, turn them into their id number categories = [ c if isinstance(c, int) else CATEGORIES[c] for c in categories ] category_params = {str(c): '' for c in categories if str(c)} entries = set() for search_string in entry.get('search_strings', [entry['title']]): search_params = { key: value for (key, value) in category_params.items() } query = normalize_unicode(search_string) search_params.update({'q': query, 'qf': ''}) logger.debug('searching with params: {}', search_params) if config.get('free'): req = requests.get(FREE_SEARCH_URL, params=search_params, cookies={ 'uid': str(config['uid']), 'pass': config['password'] }) else: req = requests.get(SEARCH_URL, params=search_params, cookies={ 'uid': str(config['uid']), 'pass': config['password'] }) logger.debug('full search URL: {}', req.url) if '/u/' + str(config['uid']) not in req.text: raise plugin.PluginError( "Invalid cookies (user not logged in)...") soup = get_soup(req.content, parser="html.parser") torrents = soup.find('table', {'id': 'torrents'}) results = torrents.findAll('tr') for torrent in results: if torrent.th and 'ac' in torrent.th.get('class'): # Header column continue if torrent.find('td', {'colspan': '99'}): logger.debug('No results found for search {}', search_string) break entry = Entry() link = torrent.find('a', href=re.compile('download'))['href'] entry[ 'url'] = f"{BASE_URL}{link}?torrent_pass={config.get('rss_key')}" entry['title'] = torrent.find('a', href=re.compile('details')).text seeders = torrent.findNext('td', { 'class': 'ac t_seeders' }).text leechers = torrent.findNext('td', { 'class': 'ac t_leechers' }).text entry['torrent_seeds'] = int(seeders) entry['torrent_leeches'] = int(leechers) entry['torrent_availability'] = torrent_availability( entry['torrent_seeds'], entry['torrent_leeches']) size = torrent.findNext( text=re.compile(r'^([\.\d]+) ([GMK]?)B$')) size = re.search(r'^([\.\d]+) ([GMK]?)B$', size) entry['content_size'] = parse_filesize(size.group(0)) logger.debug('Found entry {}', entry) entries.add(entry) return entries
def search(self, task, entry, config=None): if not config: return [] elif config == 'fail': raise plugin.PluginError('search plugin failure') return [Entry(entry)]
def inject_series(self, execute_task, release_name): execute_task('inject_series', options={ 'inject': [Entry(title=release_name, url='')], 'disable_tracking': True })
def on_task_input(self, task, config): if config.get('account') and not config.get('username'): config['username'] = '******' session = get_session(account=config.get('account')) endpoint = ['users', config['username']] if config['list'] in ['collection', 'watchlist', 'watched']: endpoint += (config['list'], config['type']) else: endpoint += ('lists', make_list_slug(config['list']), 'items') log.verbose('Retrieving `%s` list `%s`' % (config['type'], config['list'])) try: result = session.get(get_api_url(endpoint)) try: data = result.json() except ValueError: log.debug('Could not decode json from response: %s', result.text) raise plugin.PluginError('Error getting list from trakt.') except RequestException as e: raise plugin.PluginError( 'Could not retrieve list from trakt (%s)' % e.args[0]) if not data: log.warning('No data returned from trakt for %s list %s.' % (config['type'], config['list'])) return entries = [] list_type = (config['type']).rstrip('s') for item in data: # Collection and watched lists don't return 'type' along with the items (right now) if 'type' in item and item['type'] != list_type: log.debug( 'Skipping %s because it is not a %s' % (item[item['type']].get('title', 'unknown'), list_type)) continue if not item[list_type]['title']: # There seems to be some bad shows sometimes in lists with no titles. Skip them. log.warning( 'Item in trakt list does not appear to have a title, skipping.' ) continue entry = Entry() if list_type == 'episode': entry[ 'url'] = 'http://trakt.tv/shows/%s/seasons/%s/episodes/%s' % ( item['show']['ids']['slug'], item['episode']['season'], item['episode']['number']) else: entry['url'] = 'http://trakt.tv/%s/%s' % ( list_type, item[list_type]['ids'].get('slug')) entry.update_using_map(field_maps[list_type], item) if entry.isvalid(): if config.get('strip_dates'): # Remove year from end of name if present entry['title'] = re.sub(r'\s+\(\d{4}\)$', '', entry['title']) entries.append(entry) else: log.debug('Invalid entry created? %s' % entry) return entries
def on_task_input(self, task, config): config = self.prepare_config(config) if not config['enabled']: return if not self.client: self.client = self.create_rpc_client(config) entries = [] # Hack/Workaround for http://flexget.com/ticket/2002 # TODO: Proper fix if 'username' in config and 'password' in config: self.client.http_handler.set_authentication( self.client.url, config['username'], config['password'] ) session = self.client.get_session() for torrent in self.client.get_torrents(): seed_ratio_ok, idle_limit_ok = self.check_seed_limits(torrent, session) if config['only_complete'] and not ( seed_ratio_ok and idle_limit_ok and torrent.progress == 100 ): continue entry = Entry( title=torrent.name, url='', torrent_info_hash=torrent.hashString, content_size=torrent.totalSize / (1024 * 1024), ) # Location of torrent is only valid if transmission is on same machine as flexget if config['host'] in ('localhost', '127.0.0.1'): entry['location'] = torrent.torrentFile entry['url'] = 'file://' + torrent.torrentFile for attr in [ 'id', 'comment', 'downloadDir', 'isFinished', 'isPrivate', 'ratio', 'status', 'date_active', 'date_added', 'date_done', 'date_started', 'priority', 'progress', 'secondsDownloading', 'secondsSeeding', ]: try: entry['transmission_' + attr] = getattr(torrent, attr) except Exception: log.debug( 'error when requesting transmissionrpc attribute %s', attr, exc_info=True ) entry['transmission_trackers'] = [t['announce'] for t in torrent.trackers] entry['transmission_seed_ratio_ok'] = seed_ratio_ok entry['transmission_idle_limit_ok'] = idle_limit_ok st_error_to_desc = { 0: 'OK', 1: 'tracker_warning', 2: 'tracker_error', 3: 'local_error' } entry['transmission_error_state'] = st_error_to_desc[torrent.error] # Built in done_date doesn't work when user adds an already completed file to transmission if torrent.progress == 100: entry['transmission_date_done'] = datetime.fromtimestamp( max(torrent.addedDate, torrent.doneDate) ) entries.append(entry) return entries
def on_task_input(self, task, config): subtitle_list = SubtitleList(config) recursion_depth = config['recursion_depth'] # A hack to not output certain files without deleting them from the list temp_discarded_items = set() for item in subtitle_list: if not config['force_file_existence'] and not os.path.exists( item['location']): log.error('File %s does not exist. Skipping.', item['location']) temp_discarded_items.add(item) continue if not os.path.exists(item['location']): log.error('File %s does not exist. Removing from list.', item['location']) subtitle_list.discard(item) continue if self._expired(item, config): log.info( 'File %s has been in the list for %s. Removing from list.', item['location'], item['remove_after'] or config['remove_after'], ) subtitle_list.discard(item) continue languages = set(item['subtitle_languages']) or set( config.get('languages', [])) num_potential_files = 0 num_added_files = 0 if os.path.isdir(item['location']): # recursion depth 1 is no recursion max_depth = ( len(normalize_path(item['location']).split(os.sep)) + recursion_depth - 1) for root_dir, _, files in os.walk(item['location']): current_depth = len(root_dir.split(os.sep)) if current_depth > max_depth: break for file in files: if os.path.splitext(file)[1] not in VIDEO_EXTENSIONS: log.debug('File %s is not a video file. Skipping', file) continue num_potential_files += 1 file_path = normalize_path(os.path.join( root_dir, file)) if not config[ 'check_subtitles'] or not self.all_subtitles_exist( file_path, languages): subtitle_list.config['languages'] = languages subtitle_list.add( Entry( title=os.path.splitext( os.path.basename(file_path))[0], url='file://' + file_path, location=file_path, )) num_added_files += 1 # delete the original dir if it contains any video files if num_added_files or num_potential_files: log.debug( 'Added %s file(s) from %s to subtitle list %s', num_added_files, item['location'], config['list'], ) subtitle_list.discard(item) else: log.debug('No files found in %s. Skipping.', item['location']) temp_discarded_items.add(item) elif config['check_subtitles'] and self.all_subtitles_exist( item['location'], languages): subtitle_list.discard(item) return list(set(subtitle_list) - temp_discarded_items)
def to_entry(self): entry = Entry() entry['title'] = entry['regexp'] = self.regexp entry['url'] = 'mock://localhost/regexp_list/%d' % self.id return entry
def search(self, task, entry, config): """ Search for entries on RarBG """ categories = config.get('category', 'all') # Ensure categories a list if not isinstance(categories, list): categories = [categories] # Convert named category to its respective category id number categories = [ c if isinstance(c, int) else CATEGORIES[c] for c in categories ] category_url_fragment = ';'.join(str(c) for c in categories) entries = set() params = { 'mode': 'search', 'ranked': int(config['ranked']), 'min_seeders': config['min_seeders'], 'min_leechers': config['min_leechers'], 'sort': config['sorted_by'], 'category': category_url_fragment, 'format': 'json_extended', 'app_id': 'flexget', } for search_string in entry.get('search_strings', [entry['title']]): params.pop('search_string', None) params.pop('search_imdb', None) params.pop('search_tvdb', None) if entry.get('movie_name') and entry.get('imdb_id'): params['search_imdb'] = entry.get('imdb_id') else: query = normalize_scene(search_string) query_url_fragment = query.encode('utf8') params['search_string'] = query_url_fragment if config['use_tvdb']: plugin.get('thetvdb_lookup', self).lazy_series_lookup(entry, 'en') params['search_tvdb'] = entry.get('tvdb_id') logger.debug('Using tvdb id {}', entry.get('tvdb_id')) response = self.get(params=params) if not response: continue # error code 10 and 20 just mean no results were found if response.get('error_code') in [10, 20]: searched_string = (params.get('search_string') or 'imdb={0}'.format( params.get('search_imdb')) or 'tvdb={0}'.format(params.get('tvdb_id'))) logger.debug( 'No results found for {}. Message from rarbg: {}', searched_string, response.get('error'), ) continue elif response.get('error'): logger.error('Error code {}: {}', response.get('error_code'), response.get('error')) continue else: for result in response.get('torrent_results'): e = Entry() e['title'] = result.get('title') e['url'] = result.get('download') e['torrent_seeds'] = int(result.get('seeders')) e['torrent_leeches'] = int(result.get('leechers')) e['content_size'] = int(result.get('size')) / 1024 / 1024 episode_info = result.get('episode_info') if episode_info: e['imdb_id'] = episode_info.get('imdb') e['tvdb_id'] = episode_info.get('tvdb') e['tvrage_id'] = episode_info.get('tvrage') entries.add(e) return entries
def on_task_input(self, task, config): config = self.prepare_config(config) urlconfig = {} urlappend = "?" entries = [] if config['unwatched_only'] and config[ 'section'] != 'recentlyViewedShows' and config[ 'section'] != 'all': urlconfig['unwatched'] = '1' if config['username'] and config['password']: accesstoken = self.plex_get_accesstoken(config) log.debug("Got accesstoken: %s" % accesstoken) urlconfig['X-Plex-Token'] = accesstoken for key in urlconfig: urlappend += '%s=%s&' % (key, urlconfig[key]) if not self.plex_section_is_int(config['section']): try: path = "/library/sections/" r = requests.get( "http://%s:%d%s%s" % (config['plexserver'], config['port'], path, urlappend)) except requests.RequestException as e: raise plugin.PluginError('Error retrieving source: %s' % e) dom = parseString(r.text.encode("utf-8")) for node in dom.getElementsByTagName('Directory'): if node.getAttribute('title') == config['section']: config['section'] = int(node.getAttribute('key')) if not self.plex_section_is_int(config['section']): raise plugin.PluginError('Could not find section \'%s\'' % config['section']) log.debug("Fetching http://%s:%d/library/sections/%s/%s%s" % (config['server'], config['port'], config['section'], config['selection'], urlappend)) try: path = "/library/sections/%s/%s" % (config['section'], config['selection']) r = requests.get( "http://%s:%d%s%s" % (config['plexserver'], config['port'], path, urlappend)) except requests.RequestException as e: raise plugin.PluginError( 'There is no section with number %d. (%s)' % (config['section'], e)) dom = parseString(r.text.encode("utf-8")) plexsectionname = dom.getElementsByTagName( 'MediaContainer')[0].getAttribute('title1') viewgroup = dom.getElementsByTagName('MediaContainer')[0].getAttribute( 'viewGroup') log.debug("Plex section \"%s\" is a \"%s\" section" % (plexsectionname, viewgroup)) if viewgroup != "movie" and viewgroup != "show" and viewgroup != "episode": raise plugin.PluginError( "Section is neither a movie nor tv show section!") domroot = "Directory" titletag = "title" if viewgroup == "episode": domroot = "Video" titletag = "grandparentTitle" thumbtag = "thumb" arttag = "art" seasoncovertag = "parentThumb" covertag = "grandparentThumb" elif viewgroup == "movie": domroot = "Video" titletag = "title" arttag = "art" seasoncovertag = "thumb" covertag = "thumb" if config['fetch'] == "thumb": raise plugin.PluginError( "Movie sections does not have any thumbnails to download!") for node in dom.getElementsByTagName(domroot): e = Entry() e['plex_server'] = config['plexserver'] e['plex_port'] = config['port'] e['plex_section'] = config['section'] e['plex_section_name'] = plexsectionname e['plex_episode_thumb'] = '' title = node.getAttribute(titletag) if config['strip_year']: title = re.sub(r'^(.*)\(\d{4}\)(.*)', r'\1\2', title) if config['strip_parens']: title = re.sub(r'\(.*?\)', r'', title) title = title.strip() if config['strip_non_alpha']: title = re.sub(r'[\(\)]', r'', title) title = re.sub(r'&', r'And', title) title = re.sub(r'[^A-Za-z0-9- \']', r'', title) if config['lowercase_title']: title = title.lower() if viewgroup == "show": e['title'] = title e['url'] = 'NULL' entries.append(e) # show ends here. continue e['plex_art'] = "http://%s:%d%s%s" % ( config['server'], config['port'], node.getAttribute(arttag), urlappend) e['plex_cover'] = "http://%s:%d%s%s" % ( config['server'], config['port'], node.getAttribute(covertag), urlappend) e['plex_season_cover'] = "http://%s:%d%s%s" % ( config['server'], config['port'], node.getAttribute(seasoncovertag), urlappend) if viewgroup == "episode": e['plex_thumb'] = "http://%s:%d%s%s" % ( config['server'], config['port'], node.getAttribute('thumb'), urlappend) season = int(node.getAttribute('parentIndex')) if node.getAttribute('parentIndex') == node.getAttribute( 'year'): season = node.getAttribute('originallyAvailableAt') filenamemap = "%s_%s%s_%s_%s_%s.%s" episode = "" elif node.getAttribute('index'): episode = int(node.getAttribute('index')) filenamemap = "%s_%02dx%02d_%s_%s_%s.%s" else: log.debug( "Could not get episode number for '%s' (Hint, ratingKey: %s)" % (title, node.getAttribute('ratingKey'))) break elif viewgroup == "movie": filenamemap = "%s_%s_%s_%s.%s" e['plex_duration'] = node.getAttribute('duration') e['plex_summary'] = node.getAttribute('summary') count = node.getAttribute('viewCount') offset = node.getAttribute('viewOffset') if count: e['plex_status'] = "seen" elif offset: e['plex_status'] = "inprogress" else: e['plex_status'] = "unwatched" for media in node.getElementsByTagName('Media'): vcodec = media.getAttribute('videoCodec') acodec = media.getAttribute('audioCodec') if config['fetch'] == "file" or not config['fetch']: container = media.getAttribute('container') else: container = "jpg" resolution = media.getAttribute('videoResolution') + "p" for part in media.getElementsByTagName('Part'): if config['fetch'] == "file" or not config['fetch']: key = part.getAttribute('key') elif config['fetch'] == "art": key = node.getAttribute(arttag) elif config['fetch'] == "cover": key = node.getAttribute(arttag) elif config['fetch'] == "season_cover": key = node.getAttribute(seasoncovertag) elif config['fetch'] == "thumb": key = node.getAttribute(thumbtag) # key = part.getAttribute('key') duration = part.getAttribute('duration') if viewgroup == "show": e['plex_title'] = episodetitle elif viewgroup == "movie": e['plex_title'] = title if config['original_filename']: filename, fileext = os.path.splitext( basename(part.getAttribute('file'))) if config['fetch'] != 'file': filename += ".jpg" else: filename = "%s.%s" % (filename, fileext) else: if viewgroup == "episode": filename = filenamemap % (title.replace( " ", "."), season, episode, resolution, vcodec, acodec, container) title = filename elif viewgroup == "movie": filename = filenamemap % (title.replace( " ", "."), resolution, vcodec, acodec, container) e['plex_url'] = "http://%s:%d%s%s" % ( config['server'], config['port'], key, urlappend) e['plex_path'] = key e['url'] = "http://%s:%d%s%s" % ( config['server'], config['port'], key, urlappend) e['plex_duration'] = duration e['filename'] = filename e['title'] = title if key == "": log.debug("Could not find anything in PMS to download. Next!") else: entries.append(e) return entries
def items(self): if self._items is None: log.debug('fetching items from IMDB') try: r = self.session.get( 'http://www.imdb.com/list/export?list_id=%s&author_id=%s' % (self.list_id, self.user_id), cookies=self.cookies) except RequestException as e: raise PluginError(e.args[0]) lines = r.iter_lines(decode_unicode=True) # Throw away first line with headers next(lines) self._items = [] for row in csv_reader(lines): log.debug('parsing line from csv: %s', ', '.join(row)) if not len(row) == 16: log.debug('no movie row detected, skipping. %s', ', '.join(row)) continue entry = Entry({ 'title': '%s (%s)' % (row[5], row[11]) if row[11] != '????' else '%s' % row[5], 'url': row[15], 'imdb_id': row[1], 'imdb_url': row[15], 'imdb_list_position': int(row[0]), 'imdb_list_created': datetime.strptime(row[2], '%a %b %d %H:%M:%S %Y') if row[2] else None, 'imdb_list_modified': datetime.strptime(row[3], '%a %b %d %H:%M:%S %Y') if row[3] else None, 'imdb_list_description': row[4], 'imdb_name': row[5], 'imdb_year': int(row[11]) if row[11] != '????' else None, 'imdb_score': float(row[9]) if row[9] else None, 'imdb_user_score': float(row[8]) if row[8] else None, 'imdb_votes': int(row[13]) if row[13] else None, 'imdb_genres': [genre.strip() for genre in row[12].split(',')] }) item_type = row[6].lower() name = row[5] year = int(row[11]) if row[11] != '????' else None if item_type in MOVIE_TYPES: entry['movie_name'] = name entry['movie_year'] = year elif item_type in SERIES_TYPES: entry['series_name'] = name entry['series_year'] = year elif item_type in OTHER_TYPES: entry['title'] = name else: log.verbose( 'Unknown IMDB type entry received: %s. Skipping', item_type) continue self._items.append(entry) return self._items
def search(self, task, entry, config): """CPASBIEN search plugin Config example: tv_search_cpasbien: discover: what: - trakt_list: username: xxxxxxx api_key: xxxxxxx series: watchlist from: - cpasbien: category: "series-vostfr" interval: 1 day ignore_estimations: yes Category is ONE of: all films series musique films-french 1080p 720p series-francaise films-dvdrip films-vostfr series-vostfr ebook """ base_url = 'http://www.cpasbien.io' entries = set() for search_string in entry.get('search_strings', [entry['title']]): search_string = search_string.replace(' ', '-').lower() search_string = search_string.replace('(', '') search_string = search_string.replace(')', '') query = normalize_unicode(search_string) query_url_fragment = urllib.quote_plus(query.encode('utf-8')) # http://www.cpasbien.pe/recherche/ncis.html if config['category'] == 'all': str_url = (base_url, 'recherche', query_url_fragment) url = '/'.join(str_url) else: category_url_fragment = '%s' % config['category'] str_url = (base_url, 'recherche', category_url_fragment, query_url_fragment) url = '/'.join(str_url) log.debug('search url: %s' % url + '.html') # GET URL f = task.requests.get(url + '.html').content soup = get_soup(f) if soup.findAll(text=re.compile(' 0 torrents')): log.debug('search returned no results') else: nextpage = 0 while (nextpage >= 0): if (nextpage > 0): newurl = url + '/page-' + str(nextpage) log.debug('-----> NEXT PAGE : %s' % newurl) f1 = task.requests.get(newurl).content soup = get_soup(f1) for result in soup.findAll( 'div', attrs={'class': re.compile('ligne')}): entry = Entry() link = result.find( 'a', attrs={'href': re.compile('dl-torrent')}) entry['title'] = link.contents[0] # REWRITE URL page_link = link.get('href') link_rewrite = page_link.split('/') # get last value in array remove .html and replace by .torrent endlink = link_rewrite[-1] str_url = (base_url, '/telechargement/', endlink[:-5], '.torrent') entry['url'] = ''.join(str_url) log.debug('Title: %s | DL LINK: %s' % (entry['title'], entry['url'])) entry['torrent_seeds'] = (int( result.find('span', attrs={ 'class': re.compile('seed') }).text)) entry['torrent_leeches'] = (int( result.find('div', attrs={ 'class': re.compile('down') }).text)) sizefull = (result.find('div', attrs={ 'class': re.compile('poid') }).text) size = sizefull[:-3] unit = sizefull[-2:] if unit == 'GB': entry['content_size'] = int(float(size) * 1024) elif unit == 'MB': entry['content_size'] = int(float(size)) elif unit == 'KB': entry['content_size'] = int(float(size) / 1024) if (entry['torrent_seeds'] > 0): entries.add(entry) else: log.debug('0 SEED, not adding entry') if soup.find(text=re.compile('Suiv')): nextpage += 1 else: nextpage = -1 return entries
def process(self): imdb_lookup = plugin.get_plugin_by_name('imdb_lookup').instance self.changes.sort() udata = load_uoccin_data(self.folder) for line in self.changes: tmp = line.split('|') typ = tmp[1] tid = tmp[2] fld = tmp[3] val = tmp[4] self.log.verbose( 'processing: type=%s, target=%s, field=%s, value=%s' % (typ, tid, fld, val)) if typ == 'movie': # default mov = udata['movies'].setdefault( tid, { 'name': 'N/A', 'watchlist': False, 'collected': False, 'watched': False }) # movie title is unknown at this time fake = Entry() fake['url'] = 'http://www.imdb.com/title/' + tid fake['imdb_id'] = tid try: imdb_lookup.lookup(fake) mov['name'] = fake.get('imdb_name') except plugin.PluginError: self.log.warning( 'Unable to lookup movie %s from imdb, using raw name.' % tid) # setting if fld == 'watchlist': mov['watchlist'] = val == 'true' elif fld == 'collected': mov['collected'] = val == 'true' elif fld == 'watched': mov['watched'] = val == 'true' elif fld == 'tags': mov['tags'] = re.split(',\s*', val) elif fld == 'subtitles': mov['subtitles'] = re.split(',\s*', val) elif fld == 'rating': mov['rating'] = int(val) # cleaning if not (mov['watchlist'] or mov['collected'] or mov['watched']): self.log.verbose('deleting unused section: movies\%s' % tid) udata['movies'].pop(tid) elif typ == 'series': tmp = tid.split('.') sid = tmp[0] sno = tmp[1] if len(tmp) > 2 else None eno = tmp[2] if len(tmp) > 2 else None # default ser = udata['series'].setdefault( sid, { 'name': 'N/A', 'watchlist': False, 'collected': {}, 'watched': {} }) # series name is unknown at this time try: series = lookup_series(tvdb_id=sid) ser['name'] = series.name except LookupError: self.log.warning( 'Unable to lookup series %s from tvdb, using raw name.' % sid) # setting if fld == 'watchlist': ser['watchlist'] = val == 'true' elif fld == 'tags': ser['tags'] = re.split(',\s*', val) elif fld == 'rating': ser['rating'] = int(val) elif sno is None or eno is None: self.log.warning( 'invalid line "%s": season and episode numbers are required' % line) elif fld == 'collected': season = ser['collected'].setdefault(sno, {}) if val == 'true': season.setdefault(eno, []) else: if eno in season: season.pop(eno) if not season: self.log.verbose( 'deleting unused section: series\%s\collected\%s' % (sid, sno)) ser['collected'].pop(sno) elif fld == 'subtitles': ser['collected'].setdefault(sno, {})[eno] = re.split( ',\s*', val) elif fld == 'watched': season = ser['watched'].setdefault(sno, []) if val == 'true': season = ser['watched'][sno] = list( set(season) | set([int(eno)])) elif int(eno) in season: season.remove(int(eno)) season.sort() if not season: self.log.debug( 'deleting unused section: series\%s\watched\%s' % (sid, sno)) ser['watched'].pop(sno) # cleaning if not (ser['watchlist'] or ser['collected'] or ser['watched']): self.log.debug('deleting unused section: series\%s' % sid) udata['series'].pop(sid) else: self.log.warning('invalid element type "%s"' % typ) # save the updated uoccin.json ufile = os.path.join(self.folder, 'uoccin.json') try: text = json.dumps(udata, sort_keys=True, indent=4, separators=(',', ': ')) with open(ufile, 'w') as f: f.write(text) except Exception as err: self.log.debug('error writing %s: %s' % (ufile, err)) raise plugin.PluginError('error writing %s: %s' % (ufile, err))
def search(self, task, entry, config=None): config = self.prepare_config(config) if not session.cookies: log.debug('Logging in to %s...' % URL) params = { 'username': config['username'], 'password': config['password'], 'keeplogged': '1', 'login': '******' } session.post(URL + 'login.php', data=params) cat = ''.join([ '&' + ('filter_cat[%s]' % id) + '=1' for id in config['category'] ]) rls = 'release_type=' + config['type'] url_params = rls + cat multip = config['gravity_multiplier'] entries = set() for search_string in entry.get('search_strings', [entry['title']]): srch = normalize_unicode(clean_title(search_string)) srch = '&searchstr=' + quote(srch.encode('utf8')) url = URL + 'torrents.php?' + url_params + srch log.debug('Fetching URL for `%s`: %s' % (search_string, url)) page = session.get(url).content soup = get_soup(page) for result in soup.findAll('tr', attrs={'class': 'torrent'}): entry = Entry() entry['title'] = result.find('span', attrs={ 'class': 'torrent_name_link' }).text entry['url'] = URL + result.find( 'a', href=re.compile( 'torrents\.php\?action=download')).get('href') entry['torrent_seeds'], entry['torrent_leeches'] = [ r.text for r in result.findAll('td')[-2:] ] entry['search_sort'] = torrent_availability( entry['torrent_seeds'], entry['torrent_leeches']) * multip size = result.findAll('td')[-4].text size = re.search('(\d+(?:[.,]\d+)*)\s?([KMG]B)', size) if size: if size.group(2) == 'GB': entry['content_size'] = int( float(size.group(1).replace(',', '')) * 1000**3 / 1024**2) elif size.group(2) == 'MB': entry['content_size'] = int( float(size.group(1).replace(',', '')) * 1000**2 / 1024**2) elif size.group(2) == 'KB': entry['content_size'] = int( float(size.group(1).replace(',', '')) * 1000 / 1024**2) else: entry['content_size'] = int( float(size.group(1).replace(',', '')) / 1024**2) entries.add(entry) return entries