def notify(self, message = '', data = None, listener = None): if not data: data = {} api_data = { 'user': self.conf('user_key'), 'token': self.conf('api_token'), 'message': toUnicode(message), 'priority': self.conf('priority'), 'sound': self.conf('sound'), } if data and getIdentifier(data): api_data.update({ 'url': toUnicode('http://www.imdb.com/title/%s/' % getIdentifier(data)), 'url_title': toUnicode('%s on IMDb' % getTitle(data)), }) try: data = self.urlopen('%s/%s' % (self.api_url, '1/messages.json'), headers = {'Content-type': 'application/x-www-form-urlencoded'}, data = api_data) log.info2('Pushover responded with: %s', data) return True except: return False
def searchAll(self, manual=False): if self.in_progress: log.info("Search already in progress") fireEvent( "notify.frontend", type="movie.searcher.already_started", data=True, message="Full search already in progress", ) return self.in_progress = True fireEvent("notify.frontend", type="movie.searcher.started", data=True, message="Full search started") medias = [x["_id"] for x in fireEvent("media.with_status", "active", with_doc=False, single=True)] random.shuffle(medias) total = len(medias) self.in_progress = {"total": total, "to_go": total} try: search_protocols = fireEvent("searcher.protocols", single=True) for media_id in medias: media = fireEvent("media.get", media_id, single=True) try: self.single(media, search_protocols, manual=manual) except IndexError: log.error( "Forcing library update for %s, if you see this often, please report: %s", (getIdentifier(media), traceback.format_exc()), ) fireEvent("movie.update_info", media_id) except: log.error("Search failed for %s: %s", (getIdentifier(media), traceback.format_exc())) self.in_progress["to_go"] -= 1 # Break if CP wants to shut down if self.shuttingDown(): break except SearchSetupError: pass self.in_progress = False
def notify(self, message = '', data = None, listener = None): if not data: data = {} # Get configuration data token = self.conf('bot_token') usr_id = self.conf('receiver_user_id') # Add IMDB url to message: if data: imdb_id = getIdentifier(data) if imdb_id: url = 'http://www.imdb.com/title/{0}/'.format(imdb_id) message = '{0}\n{1}'.format(message, url) # Cosntruct message payload = {'chat_id': usr_id, 'text': message, 'parse_mode': 'Markdown'} # Send message user Telegram's Bot API response = requests.post(self.TELEGRAM_API % (token, "sendMessage"), data=payload) # Error logging sent_successfuly = True if not response.status_code == 200: log.error('Could not send notification to TelegramBot (token=%s). Response: [%s]', (token, response.text)) sent_successfuly = False return sent_successfuly
def updateReleaseDate(self, media_id): """ Update release_date (eta) info only @param media_id: document id @return: dict, with dates dvd, theater, bluray, expires """ try: db = get_db() media = db.get('id', media_id) if not media.get('info'): media = self.update(media_id) dates = media.get('info', {}).get('release_date') else: dates = media.get('info').get('release_date') if dates and (dates.get('expires', 0) < time.time() or dates.get('expires', 0) > time.time() + (604800 * 4)) or not dates: dates = fireEvent('movie.info.release_date', identifier = getIdentifier(media), merge = True) media['info'].update({'release_date': dates}) db.update(media) return dates except: log.error('Failed updating release dates: %s', traceback.format_exc()) return {}
def search(self, group): movie_name = getTitle(group) url = self.urls['api'] % self.movieUrlName(movie_name) try: data = self.getCache('hdtrailers.%s' % getIdentifier(group), url, show_error = False) except HTTPError: log.debug('No page found for: %s', movie_name) data = None result_data = {'480p': [], '720p': [], '1080p': []} if not data: return result_data did_alternative = False for provider in self.providers: results = self.findByProvider(data, provider) # Find alternative if results.get('404') and not did_alternative: results = self.findViaAlternative(group) did_alternative = True result_data = mergeDicts(result_data, results) return result_data
def _search(self, movie, quality, results): search_url = self.urls['search'] % (self.getDomain(), getIdentifier(movie), quality['identifier']) data = self.getJsonData(search_url) if data and data.get('MovieList'): try: for result in data.get('MovieList'): try: title = result['TorrentUrl'].split('/')[-1][:-8].replace('_', '.').strip('._') title = title.replace('.-.', '-') title = title.replace('..', '.') except: continue results.append({ 'id': result['MovieID'], 'name': title, 'url': result['TorrentMagnetUrl'], 'detail_url': self.urls['detail'] % (self.getDomain(), result['MovieID']), 'size': self.parseSize(result['Size']), 'seeders': tryInt(result['TorrentSeeds']), 'leechers': tryInt(result['TorrentPeers']) }) except: log.error('Failed getting results from %s: %s', (self.getName(), traceback.format_exc()))
def _search(self, movie, quality, results): domain = self.getDomain() if not domain: return search_url = self.urls['search'] % (domain, getIdentifier(movie), quality['identifier']) data = self.getJsonData(search_url) if data and data.get('MovieList'): try: for result in data.get('MovieList'): if result['Quality'] and result['Quality'] not in result['MovieTitle']: title = result['MovieTitle'] + ' BrRip ' + result['Quality'] else: title = result['MovieTitle'] + ' BrRip' results.append({ 'id': result['MovieID'], 'name': title, 'url': result['TorrentMagnetUrl'], 'detail_url': self.urls['detail'] % (domain, result['MovieID']), 'size': self.parseSize(result['Size']), 'seeders': tryInt(result['TorrentSeeds']), 'leechers': tryInt(result['TorrentPeers']), }) except: log.error('Failed getting results from %s: %s', (self.getName(), traceback.format_exc()))
def suggestView(self, limit = 6, **kwargs): movies = splitString(kwargs.get('movies', '')) ignored = splitString(kwargs.get('ignored', '')) seen = splitString(kwargs.get('seen', '')) cached_suggestion = self.getCache('suggestion_cached') if cached_suggestion: suggestions = cached_suggestion else: if not movies or len(movies) == 0: active_movies = fireEvent('media.with_status', ['active', 'done'], single = True) movies = [getIdentifier(x) for x in active_movies] if not ignored or len(ignored) == 0: ignored = splitString(Env.prop('suggest_ignore', default = '')) if not seen or len(seen) == 0: movies.extend(splitString(Env.prop('suggest_seen', default = ''))) suggestions = fireEvent('movie.suggest', movies = movies, ignore = ignored, single = True) self.setCache('suggestion_cached', suggestions, timeout = 6048000) # Cache for 10 weeks return { 'success': True, 'count': len(suggestions), 'suggestions': suggestions[:int(limit)] }
def notify(self, message = '', data = None, listener = None): if not data: data = {} if listener == 'test': post_data = { 'username': self.conf('automation_username'), 'password': self.conf('automation_password'), } result = self.call((self.urls['test'] % self.conf('automation_api_key')), post_data) return result else: post_data = { 'username': self.conf('automation_username'), 'password': self.conf('automation_password'), 'movies': [{ 'imdb_id': getIdentifier(data), 'title': getTitle(data), 'year': data['info']['year'] }] if data else [] } result = self.call((self.urls['library'] % self.conf('automation_api_key')), post_data) if self.conf('remove_watchlist_enabled'): result = result and self.call((self.urls['unwatchlist'] % self.conf('automation_api_key')), post_data) return result
def buildUrl(self, media, host): arguments = tryUrlencode({ 'user': host['name'], 'passkey': host['pass_key'], 'imdbid': getIdentifier(media), }) return '%s?%s' % (host['host'], arguments)
def updateSuggestionCache(self, ignore_imdb = None, limit = 6, ignored = None, seen = None): # Combine with previous suggestion_cache cached_suggestion = self.getCache('suggestion_cached') or [] new_suggestions = [] ignored = [] if not ignored else ignored seen = [] if not seen else seen if ignore_imdb: suggested_imdbs = [] for cs in cached_suggestion: if cs.get('imdb') != ignore_imdb and cs.get('imdb') not in suggested_imdbs: suggested_imdbs.append(cs.get('imdb')) new_suggestions.append(cs) # Get new results and add them if len(new_suggestions) - 1 < limit: db = get_db() active_movies = fireEvent('media.with_status', ['active', 'done'], single = True) movies = [getIdentifier(x) for x in active_movies] movies.extend(seen) ignored.extend([x.get('imdb') for x in cached_suggestion]) suggestions = fireEvent('movie.suggest', movies = movies, ignore = removeDuplicate(ignored), single = True) if suggestions: new_suggestions.extend(suggestions) self.setCache('suggestion_cached', new_suggestions, timeout = 3024000) return new_suggestions
def updateReleaseDate(self, media_id): """ Update release_date (eta) info only @param media_id: document id @return: dict, with dates dvd, theater, bluray, expires """ try: db = get_db() media = db.get("id", media_id) if not media.get("info"): media = self.update(media_id) dates = media.get("info", {}).get("release_date") else: dates = media.get("info").get("release_date") if ( dates and (dates.get("expires", 0) < time.time() or dates.get("expires", 0) > time.time() + (604800 * 4)) or not dates ): dates = fireEvent("movie.info.release_date", identifier=getIdentifier(media), merge=True) media["info"].update({"release_date": dates}) db.update(media) return dates except: log.error("Failed updating release dates: %s", traceback.format_exc()) return {}
def _search(self, movie, quality, results): title = getIdentifier(movie) data = self._post_query(title, self.getNorbitsQuality(quality.get('custom').get('quality'))) if data: log.info('We got data: %s' % data) try: for result in data: log.info('We got result: %s' % result) download_url = self.getDownloadUrl(result['id']) details_url = self.urls['detail'] % result['id'] log.info('Download url: %s' % download_url) log.info('Details url: %s' % details_url) append_data = { 'id': result['id'], 'name': result['name'], 'detail_url': details_url, 'size': tryInt(int(result['size']) / 1024 / 1024), 'seeders': tryInt(result['seeders']), 'leechers': tryInt(result['leechers']), 'url': download_url } log.info('Appending data: %s' % json.dumps(append_data)) results.append(append_data) except: log.error('Failed getting resutls from %s: %s' % (self.getName(), traceback.format_exc())) finally: log.info('Final results: %s' % results) return results
def _search(self, movie, quality, results): domain = self.getDomain() if not domain: return search_url = self.urls['search'] % (domain, getIdentifier(movie)) data = self.getJsonData(search_url) or {} data = data.get('data') if isinstance(data, dict) and data.get('movies'): try: for result in data.get('movies'): for release in result.get('torrents', []): if release['quality'] and release['quality'] not in result['title_long']: title = result['title_long'] + ' BRRip ' + release['quality'] else: title = result['title_long'] + ' BRRip' results.append({ 'id': release['hash'], 'name': title, 'url': release['url'], 'detail_url': result['url'], 'size': self.parseSize(release['size']), 'seeders': tryInt(release['seeds']), 'leechers': tryInt(release['peers']), }) except: log.error('Failed getting results from %s: %s', (self.getName(), traceback.format_exc()))
def findViaAlternative(self, group): results = {'480p': [], '720p': [], '1080p': []} movie_name = getTitle(group) url = "%s?%s" % (self.urls['backup'], tryUrlencode({'s':movie_name})) try: data = self.getCache('hdtrailers.alt.%s' % getIdentifier(group), url, show_error = False) except HTTPError: log.debug('No alternative page found for: %s', movie_name) data = None if not data: return results try: html = BeautifulSoup(data, parse_only = self.only_tables_tags) result_table = html.find_all('h2', text = re.compile(movie_name)) for h2 in result_table: if 'trailer' in h2.lower(): parent = h2.parent.parent.parent trailerLinks = parent.find_all('a', text = re.compile('480p|720p|1080p')) try: for trailer in trailerLinks: results[trailer].insert(0, trailer.parent['href']) except: pass except AttributeError: log.debug('No trailers found in via alternative.') return results
def buildUrl(self, media, host): arguments = tryUrlencode({ 'user': host['name'], 'passkey': host['pass_key'], 'imdbid': getIdentifier(media), 'search' : getTitle(media) + ' ' + str(media['info']['year']), }) return '%s?%s' % (host['host'], arguments)
def buildUrl(self, media, api_key): query = tryUrlencode({ 't': 'movie', 'imdbid': getIdentifier(media).replace('tt', ''), 'apikey': api_key, 'extended': 1 }) return query
def _search(self, media, quality, results): data = self.getHTMLData(self.urls['search'] % (self.getDomain(), 'm', getIdentifier(media).replace('tt', ''))) if data: cat_ids = self.getCatId(quality) table_order = ['name', 'size', None, 'age', 'seeds', 'leechers'] try: html = BeautifulSoup(data) resultdiv = html.find('div', attrs = {'class': 'tabs'}) for result in resultdiv.find_all('div', recursive = False): if result.get('id').lower().strip('tab-') not in cat_ids: continue try: for temp in result.find_all('tr'): if temp['class'] is 'firstr' or not temp.get('id'): continue new = {} nr = 0 for td in temp.find_all('td'): column_name = table_order[nr] if column_name: if column_name == 'name': link = td.find('div', {'class': 'torrentname'}).find_all('a')[2] new['id'] = temp.get('id')[-7:] new['name'] = link.text new['url'] = td.find('a', 'imagnet')['href'] new['detail_url'] = self.urls['detail'] % (self.getDomain(), link['href'][1:]) new['verified'] = True if td.find('a', 'iverify') else False new['score'] = 100 if new['verified'] else 0 elif column_name is 'size': new['size'] = self.parseSize(td.text) elif column_name is 'age': new['age'] = self.ageToDays(td.text) elif column_name is 'seeds': new['seeders'] = tryInt(td.text) elif column_name is 'leechers': new['leechers'] = tryInt(td.text) nr += 1 # Only store verified torrents if self.conf('only_verified') and not new['verified']: continue results.append(new) except: log.error('Failed parsing KickAssTorrents: %s', traceback.format_exc()) except AttributeError: log.debug('No search results found.')
def _search(self, movie, quality, results): data = self.getHTMLData(self.urls['search'] % (self.conf('passkey'), getIdentifier(movie), self.conf('only_internal'))) if data: try: soup = BeautifulSoup(data) if soup.find('error'): log.info(soup.find('error').get_text()) return authkey = soup.find('authkey').get_text() entries = soup.find_all('torrent') for entry in entries: torrentscore = 0 torrent_id = entry.find('id').get_text() name = entry.find('name').get_text() year = entry.find('year').get_text() releasegroup = entry.find('releasegroup').get_text() resolution = entry.find('resolution').get_text() encoding = entry.find('encoding').get_text() freeleech = entry.find('freeleech').get_text() media = entry.find('media').get_text() audioformat = entry.find('audioformat').get_text() # skip audio channel only releases if resolution == '': continue torrent_desc = '%s.%s.%s.%s-%s' % (resolution, media, audioformat, encoding, releasegroup) if self.conf('prefer_internal') and freeleech in ['0.25', '0.50']: torrentscore += 200 if encoding == 'x264' and self.conf('favor') in ['encode', 'both']: torrentscore += 200 elif re.search('Remux', encoding) and self.conf('favor') in ['remux', 'both']: torrentscore += 200 name = re.sub(r'\W', '.', name) name = re.sub(r'\.+', '.', name) results.append({ 'id': torrent_id, 'name': '%s.%s.%s' % (name, year, torrent_desc), 'url': self.urls['download'] % (torrent_id, authkey, self.conf('passkey')), 'detail_url': self.urls['detail'] % torrent_id, 'size': tryInt(entry.find('size').get_text()) / 1048576, 'seeders': tryInt(entry.find('seeders').get_text()), 'leechers': tryInt(entry.find('leechers').get_text()), 'score': torrentscore }) except: log.error('Failed getting results from %s: %s', (self.getName(), traceback.format_exc()))
def notify(self, message = '', data = None, listener = None): if not data: data = {} script_data = { 'message': toUnicode(message) } if getIdentifier(data): script_data.update({ 'imdb_id': getIdentifier(data) }) try: subprocess.call([self.conf('path'), message]) return True except: log.error('Script notification failed: %s', traceback.format_exc()) return False
def suggestView(self, limit = 6, **kwargs): if self.isDisabled(): return { 'success': True, 'movies': [] } movies = splitString(kwargs.get('movies', '')) ignored = splitString(kwargs.get('ignored', '')) seen = splitString(kwargs.get('seen', '')) cached_suggestion = self.getCache('suggestion_cached') if cached_suggestion: suggestions = cached_suggestion else: if not movies or len(movies) == 0: active_movies = fireEvent('media.with_status', ['active', 'done'], types = 'movie', single = True) movies = [getIdentifier(x) for x in active_movies] if not ignored or len(ignored) == 0: ignored = splitString(Env.prop('suggest_ignore', default = '')) if not seen or len(seen) == 0: movies.extend(splitString(Env.prop('suggest_seen', default = ''))) suggestions = fireEvent('movie.suggest', movies = movies, ignore = ignored, single = True) self.setCache('suggestion_cached', suggestions, timeout = 6048000) # Cache for 10 weeks medias = [] for suggestion in suggestions[:int(limit)]: # Cache poster posters = suggestion.get('images', {}).get('poster', []) poster = [x for x in posters if 'tmdb' in x] posters = poster if len(poster) > 0 else posters cached_poster = fireEvent('file.download', url = posters[0], single = True) if len(posters) > 0 else False files = {'image_poster': [cached_poster] } if cached_poster else {} medias.append({ 'status': 'suggested', 'title': getTitle(suggestion), 'type': 'movie', 'info': suggestion, 'files': files, 'identifiers': { 'imdb': suggestion.get('imdb') } }) return { 'success': True, 'movies': medias }
def _search(self, movie, quality, results): imdbId = getIdentifier (movie).replace ("t", "") url = self.urls['search'] % (imdbId)#, cats[0]) data = self.getHTMLData(url) if data: # Remove HDSpace NEW list split_data = data.partition('<form name="tcategories" action="index.php" method="post">') data = split_data[2] html = BeautifulSoup(data) try: #Now attempt to get any others result_table = html.find('table', attrs = {'class' : 'lista'}) if not result_table: return entries = result_table.find_all('tr') log.info("entries length: %s", len(entries)) if not entries: return for result in entries: block2 = result.find_all('td', attrs={'class' : 'header'}) # Ignore header if block2: continue cells = result.find_all('td') log.info("cells length: %s", len(cells)) extend = 0 detail = cells[1 + extend].find('a')['href'] torrent_id = detail.replace('index.php?page=torrent-details&id=', '') try: torrent_age = datetime.now() - datetime.strptime(cells[4 + extend].get_text().encode('ascii','ignore'), '%B %d, %Y,%H:%M:%S') except: torrent_age = timedelta(1) results.append({ 'id': torrent_id, 'name': cells[9 + extend].find('a')['title'].strip('History - ').replace('Blu-ray', 'bd50'), 'url': self.urls['home'] % cells[3 + extend].find('a')['href'], 'detail_url': self.urls['home'] % cells[1 + extend].find('a')['href'], 'size': self.parseSize(cells[5 + extend].get_text()), 'age': torrent_age.days, 'seeders': tryInt(cells[7 + extend].find('a').get_text()), 'leechers': tryInt(cells[8 + extend].find('a').get_text()), 'get_more_info': self.getMoreInfo, }) except: log.error('Failed getting results from %s: %s', (self.getName(), traceback.format_exc()))
def notify(self, message="", data=None, listener=None): if not data: data = {} http_handler = HTTPSConnection("api.pushover.net:443") api_data = { "user": self.conf("user_key"), "token": self.conf("api_token"), "message": toUnicode(message), "priority": self.conf("priority"), "sound": self.conf("sound"), } if data and getIdentifier(data): api_data.update( { "url": toUnicode("http://www.imdb.com/title/%s/" % getIdentifier(data)), "url_title": toUnicode("%s on IMDb" % getTitle(data)), } ) http_handler.request( "POST", "/1/messages.json", headers={"Content-type": "application/x-www-form-urlencoded"}, body=tryUrlencode(api_data), ) response = http_handler.getresponse() request_status = response.status if request_status == 200: log.info("Pushover notifications sent.") return True elif request_status == 401: log.error("Pushover auth failed: %s", response.reason) return False else: log.error("Pushover notification failed: %s", request_status) return False
def buildUrl(self, media, quality): query = tryUrlencode({ 'q': getIdentifier(media), 'm': 'n', 'max': 400, 'adv_age': Env.setting('retention', 'nzb'), 'adv_sort': 'date', 'adv_col': 'on', 'adv_nfo': 'on', 'xminsize': quality.get('size_min'), 'xmaxsize': quality.get('size_max'), }) return query
def buildUrl(self, media, host): query = tryUrlencode({ 't': 'movie', 'imdbid': getIdentifier(media).replace('tt', ''), 'apikey': host['api_key'], 'extended': 1 }) if len(host.get('custom_tag', '')) > 0: query = '%s&%s' % (query, host.get('custom_tag')) return query
def notify(self, message = '', data = None, listener = None): if not data: data = {} post_data = { 'message': toUnicode(message) } if getIdentifier(data): post_data.update({ 'imdb_id': getIdentifier(data) }) headers = { 'Content-type': 'application/x-www-form-urlencoded' } try: self.urlopen(self.conf('url'), headers = headers, data = post_data, show_error = False) return True except: log.error('Webhook notification failed: %s', traceback.format_exc()) return False
def cpTag(self, media, unique_tag = False): tag = '' if Env.setting('enabled', 'renamer') or unique_tag: identifier = getIdentifier(media) or '' unique_tag = ', ' + randomString() if unique_tag else '' tag = '.cp(' tag += identifier tag += ', ' if unique_tag and identifier else '' tag += randomString() if unique_tag else '' tag += ')' return tag if len(tag) > 7 else ''
def _search(self, movie, quality, results): data = self.getHTMLData(self.urls['search'] % (self.conf('passkey'), getIdentifier(movie), self.conf('only_internal'))) if data: try: soup = BeautifulSoup(data) if soup.find('error'): log.error(soup.find('error').get_text()) return authkey = soup.find('authkey').get_text() entries = soup.find_all('torrent') for entry in entries: torrentscore = 0 torrent_id = entry.find('id').get_text() name = entry.find('name').get_text() year = entry.find('year').get_text() releasegroup = entry.find('releasegroup').get_text() resolution = entry.find('resolution').get_text() encoding = entry.find('encoding').get_text() freeleech = entry.find('freeleech').get_text() torrent_desc = '/ %s / %s / %s ' % (releasegroup, resolution, encoding) if freeleech == '0.25' and self.conf('prefer_internal'): torrent_desc += '/ Internal' torrentscore += 200 if encoding == 'x264' and self.conf('favor') in ['encode', 'both']: torrentscore += 300 if re.search('Remux', encoding) and self.conf('favor') in ['remux', 'both']: torrentscore += 200 results.append({ 'id': torrent_id, 'name': re.sub('[^A-Za-z0-9\-_ \(\).]+', '', '%s (%s) %s' % (name, year, torrent_desc)), 'url': self.urls['download'] % (torrent_id, authkey, self.conf('passkey')), 'detail_url': self.urls['detail'] % torrent_id, 'size': tryInt(entry.find('size').get_text()) / 1048576, 'seeders': tryInt(entry.find('seeders').get_text()), 'leechers': tryInt(entry.find('leechers').get_text()), 'score': torrentscore }) except: log.error('Failed getting results from %s: %s', (self.getName(), traceback.format_exc()))
def notify(self, message = '', data = None, listener = None): if not data: data = {} http_handler = HTTPSConnection("api.pushover.net:443") api_data = { 'user': self.conf('user_key'), 'token': self.conf('api_token'), 'message': toUnicode(message), 'priority': self.conf('priority'), 'sound': self.conf('sound'), } if data and getIdentifier(data): api_data.update({ 'url': toUnicode('http://www.imdb.com/title/%s/' % getIdentifier(data)), 'url_title': toUnicode('%s on IMDb' % getTitle(data)), }) http_handler.request('POST', '/1/messages.json', headers = {'Content-type': 'application/x-www-form-urlencoded'}, body = tryUrlencode(api_data) ) response = http_handler.getresponse() request_status = response.status if request_status == 200: log.info('Pushover notifications sent.') return True elif request_status == 401: log.error('Pushover auth failed: %s', response.reason) return False else: log.error('Pushover notification failed: %s', request_status) return False
def searchAll(self, manual = False): if self.in_progress: log.info('Search already in progress') fireEvent('notify.frontend', type = 'movie.searcher.already_started', data = True, message = 'Full search already in progress') return self.in_progress = True fireEvent('notify.frontend', type = 'movie.searcher.started', data = True, message = 'Full search started') medias = [x['_id'] for x in fireEvent('media.with_status', 'active', types = 'movie', with_doc = False, single = True)] random.shuffle(medias) total = len(medias) self.in_progress = { 'total': total, 'to_go': total, } try: search_protocols = fireEvent('searcher.protocols', single = True) for media_id in medias: media = fireEvent('media.get', media_id, single = True) if not media: continue try: self.single(media, search_protocols, manual = manual) except IndexError: log.error('Forcing library update for %s, if you see this often, please report: %s', (getIdentifier(media), traceback.format_exc())) fireEvent('movie.update', media_id) except: log.error('Search failed for %s: %s', (getIdentifier(media), traceback.format_exc())) self.in_progress['to_go'] -= 1 # Break if CP wants to shut down if self.shuttingDown(): break except SearchSetupError: pass self.in_progress = False
def cpTag(self, media): if Env.setting('enabled', 'renamer'): identifier = getIdentifier(media) return '.cp(' + identifier + ')' if identifier else '' return ''
def _search(self, media, quality, results): movie_title = getTitle(media) quality_id = quality['identifier'] params = mergeDicts( self.quality_search_params[quality_id].copy(), { 'order_by': 'relevance', 'order_way': 'descending', 'searchstr': getIdentifier(media) }) url = '%s?json=noredirect&%s' % (self.urls['torrent'], tryUrlencode(params)) res = self.getJsonData(url) try: if not 'Movies' in res: return authkey = res['AuthKey'] passkey = res['PassKey'] for ptpmovie in res['Movies']: if not 'Torrents' in ptpmovie: log.debug('Movie %s (%s) has NO torrents', (ptpmovie['Title'], ptpmovie['Year'])) continue log.debug('Movie %s (%s) has %d torrents', (ptpmovie['Title'], ptpmovie['Year'], len(ptpmovie['Torrents']))) for torrent in ptpmovie['Torrents']: torrent_id = tryInt(torrent['Id']) torrentdesc = '%s %s %s' % (torrent['Resolution'], torrent['Source'], torrent['Codec']) torrentscore = 0 if 'GoldenPopcorn' in torrent and torrent['GoldenPopcorn']: torrentdesc += ' HQ' if self.conf('prefer_golden'): torrentscore += 5000 if 'FreeleechType' in torrent: torrentdesc += ' Freeleech' if self.conf('prefer_freeleech'): torrentscore += 7000 if 'Scene' in torrent and torrent['Scene']: torrentdesc += ' Scene' if self.conf('prefer_scene'): torrentscore += 2000 if 'RemasterTitle' in torrent and torrent['RemasterTitle']: torrentdesc += self.htmlToASCII( ' %s' % torrent['RemasterTitle']) torrentdesc += ' (%s)' % quality_id torrent_name = re.sub( '[^A-Za-z0-9\-_ \(\).]+', '', '%s (%s) - %s' % (movie_title, ptpmovie['Year'], torrentdesc)) def extra_check(item): return self.torrentMeetsQualitySpec(item, quality_id) results.append({ 'id': torrent_id, 'name': torrent_name, 'Source': torrent['Source'], 'Checked': 'true' if torrent['Checked'] else 'false', 'Resolution': torrent['Resolution'], 'url': '%s?action=download&id=%d&authkey=%s&torrent_pass=%s' % (self.urls['torrent'], torrent_id, authkey, passkey), 'detail_url': self.urls['detail'] % torrent_id, 'date': tryInt( time.mktime( parse(torrent['UploadTime']).timetuple())), 'size': tryInt(torrent['Size']) / (1024 * 1024), 'seeders': tryInt(torrent['Seeders']), 'leechers': tryInt(torrent['Leechers']), 'score': torrentscore, 'extra_check': extra_check, }) except: log.error('Failed getting results from %s: %s', (self.getName(), traceback.format_exc()))
def _search(self, media, quality, results): data = self.getHTMLData( self.urls['search'] % (self.getDomain(), 'm', getIdentifier(media).replace('tt', ''))) if data: cat_ids = self.getCatId(quality) table_order = ['name', 'size', None, 'age', 'seeds', 'leechers'] try: html = BeautifulSoup(data) resultdiv = html.find('div', attrs={'class': 'tabs'}) for result in resultdiv.find_all('div', recursive=False): if result.get('id').lower().strip('tab-') not in cat_ids: continue try: for temp in result.find_all('tr'): if temp['class'] is 'firstr' or not temp.get('id'): continue new = {} nr = 0 for td in temp.find_all('td'): column_name = table_order[nr] if column_name: if column_name == 'name': link = td.find('div', { 'class': 'torrentname' }).find_all('a')[2] new['id'] = temp.get('id')[-7:] new['name'] = link.text new['url'] = td.find('a', 'imagnet')['href'] new['detail_url'] = self.urls[ 'detail'] % (self.getDomain(), link['href'][1:]) new['verified'] = True if td.find( 'a', 'iverify') else False new['score'] = 100 if new[ 'verified'] else 0 elif column_name is 'size': new['size'] = self.parseSize(td.text) elif column_name is 'age': new['age'] = self.ageToDays(td.text) elif column_name is 'seeds': new['seeders'] = tryInt(td.text) elif column_name is 'leechers': new['leechers'] = tryInt(td.text) nr += 1 # Only store verified torrents if self.conf( 'only_verified') and not new['verified']: continue results.append(new) except: log.error('Failed parsing KickAssTorrents: %s', traceback.format_exc()) except AttributeError: log.debug('No search results found.')
def updateInfo(self, media_id=None, identifier=None, default_title=None, extended=False): """ Update movie information inside media['doc']['info'] @param media_id: document id @param default_title: default title, if empty, use first one or existing one @param extended: update with extended info (parses more info, actors, images from some info providers) @return: dict, with media """ if self.shuttingDown(): return try: db = get_db() if media_id: media = db.get('id', media_id) else: media = db.get('media', 'imdb-%s' % identifier, with_doc=True)['doc'] info = fireEvent('movie.info', merge=True, extended=extended, identifier=getIdentifier(media)) # Don't need those here try: del info['in_wanted'] except: pass try: del info['in_library'] except: pass if not info or len(info) == 0: log.error('Could not update, no movie info to work with: %s', identifier) return False # Update basic info media['info'] = info titles = info.get('titles', []) log.debug('Adding titles: %s', titles) # Define default title if default_title: def_title = None if default_title: counter = 0 for title in titles: if title.lower() == toUnicode( default_title.lower()) or ( toUnicode(default_title) == six.u('') and toUnicode(titles[0]) == title): def_title = toUnicode(title) break counter += 1 if not def_title: def_title = toUnicode(titles[0]) media['title'] = def_title # Files images = info.get('images', []) media['files'] = media.get('files', {}) for image_type in ['poster']: # Remove non-existing files file_type = 'image_%s' % image_type existing_files = list(set(media['files'].get(file_type, []))) for ef in media['files'].get(file_type, []): if not os.path.isfile(ef): existing_files.remove(ef) # Replace new files list media['files'][file_type] = existing_files if len(existing_files) == 0: del media['files'][file_type] # Loop over type for image in images.get(image_type, []): if not isinstance(image, (str, unicode)): continue if file_type not in media['files'] or len( media['files'].get(file_type, [])) == 0: file_path = fireEvent('file.download', url=image, single=True) if file_path: media['files'][file_type] = [file_path] break else: break db.update(media) return media except: log.error('Failed update media: %s', traceback.format_exc()) return {}
def correctRelease(self, nzb=None, media=None, quality=None, **kwargs): if media.get('type') != 'movie': return media_title = fireEvent('searcher.get_search_title', media, single=True) imdb_results = kwargs.get('imdb_results', False) retention = Env.setting('retention', section='nzb') if nzb.get('seeders') is None and 0 < retention < nzb.get('age', 0): log.info2( 'Wrong: Outside retention, age is %s, needs %s or lower: %s', (nzb['age'], retention, nzb['name'])) return False # Check for required and ignored words if not fireEvent( 'searcher.correct_words', nzb['name'], media, single=True): return False preferred_quality = quality if quality else fireEvent( 'quality.single', identifier=quality['identifier'], single=True) # Contains lower quality string contains_other = fireEvent('searcher.contains_other_quality', nzb, movie_year=media['info']['year'], preferred_quality=preferred_quality, single=True) if contains_other != False: log.info2( 'Wrong: %s, looking for %s, found %s', (nzb['name'], quality['label'], [x for x in contains_other] if contains_other else 'no quality')) return False # Contains lower quality string if not fireEvent('searcher.correct_3d', nzb, preferred_quality=preferred_quality, single=True): log.info2( 'Wrong: %s, %slooking for %s in 3D', (nzb['name'], ('' if preferred_quality['custom'].get('3d') else 'NOT '), quality['label'])) return False # File to small if nzb['size'] and tryInt(preferred_quality['size_min']) > tryInt( nzb['size']): log.info2( 'Wrong: "%s" is too small to be %s. %sMB instead of the minimal of %sMB.', (nzb['name'], preferred_quality['label'], nzb['size'], preferred_quality['size_min'])) return False # File to large if nzb['size'] and tryInt(preferred_quality['size_max']) < tryInt( nzb['size']): log.info2( 'Wrong: "%s" is too large to be %s. %sMB instead of the maximum of %sMB.', (nzb['name'], preferred_quality['label'], nzb['size'], preferred_quality['size_max'])) return False # Provider specific functions get_more = nzb.get('get_more_info') if get_more: get_more(nzb) extra_check = nzb.get('extra_check') if extra_check and not extra_check(nzb): return False if imdb_results: return True # Check if nzb contains imdb link if getImdb(nzb.get('description', '')) == getIdentifier(media): return True for raw_title in media['info']['titles']: for movie_title in possibleTitles(raw_title): movie_words = re.split('\W+', simplifyString(movie_title)) if fireEvent('searcher.correct_name', nzb['name'], movie_title, single=True): # if no IMDB link, at least check year range 1 if len(movie_words) > 2 and fireEvent( 'searcher.correct_year', nzb['name'], media['info']['year'], 1, single=True): return True # if no IMDB link, at least check year if len(movie_words) <= 2 and fireEvent( 'searcher.correct_year', nzb['name'], media['info']['year'], 0, single=True): return True log.info("Wrong: %s, undetermined naming. Looking for '%s (%s)'", (nzb['name'], media_title, media['info']['year'])) return False
def _search(self, movie, quality, results): url = self.urls['search'] % (getIdentifier(movie))#, cats[0]) data = self.getHTMLData(url) if data: # Remove HDTorrents NEW list split_data = data.partition('<!-- Show New Torrents After Last Visit -->\n\n\n\n') data = split_data[2] html = BeautifulSoup(data) try: #Get first entry in table entries = html.find_all('td', attrs={'align' : 'center'}) if len(entries) < 21: return base = 21 extend = 0 try: torrent_id = entries[base].find('div')['id'] except: extend = 2 torrent_id = entries[base + extend].find('div')['id'] torrent_age = datetime.now() - datetime.strptime(entries[15 + extend].get_text()[:8] + ' ' + entries[15 + extend].get_text()[-10::], '%H:%M:%S %d/%m/%Y') results.append({ 'id': torrent_id, 'name': entries[20 + extend].find('a')['title'].strip('History - ').replace('Blu-ray', 'bd50'), 'url': self.urls['home'] % entries[13 + extend].find('a')['href'], 'detail_url': self.urls['detail'] % torrent_id, 'size': self.parseSize(entries[16 + extend].get_text()), 'age': torrent_age.days, 'seeders': tryInt(entries[18 + extend].get_text()), 'leechers': tryInt(entries[19 + extend].get_text()), 'get_more_info': self.getMoreInfo, }) #Now attempt to get any others result_table = html.find('table', attrs = {'class' : 'mainblockcontenttt'}) if not result_table: return entries = result_table.find_all('td', attrs={'align' : 'center', 'class' : 'listas'}) if not entries: return for result in entries: block2 = result.find_parent('tr').find_next_sibling('tr') if not block2: continue cells = block2.find_all('td') try: extend = 0 detail = cells[1 + extend].find('a')['href'] except: extend = 1 detail = cells[1 + extend].find('a')['href'] torrent_id = detail.replace('details.php?id=', '') torrent_age = datetime.now() - datetime.strptime(cells[5 + extend].get_text(), '%H:%M:%S %d/%m/%Y') results.append({ 'id': torrent_id, 'name': cells[1 + extend].find('b').get_text().strip('\t ').replace('Blu-ray', 'bd50'), 'url': self.urls['home'] % cells[3 + extend].find('a')['href'], 'detail_url': self.urls['home'] % cells[1 + extend].find('a')['href'], 'size': self.parseSize(cells[6 + extend].get_text()), 'age': torrent_age.days, 'seeders': tryInt(cells[8 + extend].get_text()), 'leechers': tryInt(cells[9 + extend].get_text()), 'get_more_info': self.getMoreInfo, }) except: log.error('Failed getting results from %s: %s', (self.getName(), traceback.format_exc()))
def update(self, media_id=None, identifier=None, default_title=None, extended=False): """ Update movie information inside media['doc']['info'] @param media_id: document id @param default_title: default title, if empty, use first one or existing one @param extended: update with extended info (parses more info, actors, images from some info providers) @return: dict, with media """ if self.shuttingDown(): return lock_key = 'media.get.%s' % media_id if media_id else identifier self.acquireLock(lock_key) media = {} try: db = get_db() if media_id: media = db.get('id', media_id) else: media = db.get('media', 'imdb-%s' % identifier, with_doc=True)['doc'] info = fireEvent('movie.info', merge=True, extended=extended, identifier=getIdentifier(media)) # Don't need those here try: del info['in_wanted'] except: pass try: del info['in_library'] except: pass if not info or len(info) == 0: log.error('Could not update, no movie info to work with: %s', identifier) return False # Update basic info media['info'] = info titles = info.get('titles', []) log.debug('Adding titles: %s', titles) # Define default title if default_title: def_title = None if default_title: counter = 0 for title in titles: if title.lower() == toUnicode( default_title.lower()) or ( toUnicode(default_title) == six.u('') and toUnicode(titles[0]) == title): def_title = toUnicode(title) break counter += 1 if not def_title: def_title = toUnicode(titles[0]) media['title'] = def_title # Files image_urls = info.get('images', []) self.getPoster(media, image_urls) db.update(media) except: log.error('Failed update media: %s', traceback.format_exc()) self.releaseLock(lock_key) return media
def updateLibrary(self, full=True): last_update = float(Env.prop('manage.last_update', default=0)) if self.in_progress: log.info('Already updating library: %s', self.in_progress) return elif self.isDisabled() or (last_update > time.time() - 20): return self.in_progress = {} fireEvent('notify.frontend', type='manage.updating', data=True) try: directories = self.directories() directories.sort() added_identifiers = [] # Add some progress for directory in directories: self.in_progress[os.path.normpath(directory)] = { 'started': False, 'eta': -1, 'total': None, 'to_go': None, } for directory in directories: folder = os.path.normpath(directory) self.in_progress[os.path.normpath( directory)]['started'] = tryInt(time.time()) if not os.path.isdir(folder): if len(directory) > 0: log.error('Directory doesn\'t exist: %s', folder) continue log.info('Updating manage library: %s', folder) fireEvent('notify.frontend', type='manage.update', data=True, message='Scanning for movies in "%s"' % folder) onFound = self.createAddToLibrary(folder, added_identifiers) fireEvent('scanner.scan', folder=folder, simple=True, newer_than=last_update if not full else 0, on_found=onFound, single=True) # Break if CP wants to shut down if self.shuttingDown(): break # If cleanup option is enabled, remove offline files from database if self.conf('cleanup') and full and not self.shuttingDown(): # Get movies with done status total_movies, done_movies = fireEvent('media.list', types='movie', status='done', release_status='done', status_or=True, single=True) for done_movie in done_movies: if getIdentifier(done_movie) not in added_identifiers: fireEvent('media.delete', media_id=done_movie['_id'], delete_from='all') else: releases = done_movie.get('releases', []) for release in releases: if release.get('files'): brk = False for file_type in release.get('files', {}): for release_file in release['files'][ file_type]: # Remove release not available anymore if not os.path.isfile( sp(release_file)): fireEvent('release.clean', release['_id']) brk = True break if brk: break # Check if there are duplicate releases (different quality) use the last one, delete the rest if len(releases) > 1: used_files = {} for release in releases: for file_type in release.get('files', {}): for release_file in release['files'][ file_type]: already_used = used_files.get( release_file) if already_used: # delete current one if already_used.get( 'last_edit', 0) < release.get( 'last_edit', 0): fireEvent('release.delete', release['_id'], single=True) # delete previous one else: fireEvent('release.delete', already_used['_id'], single=True) break else: used_files[release_file] = release del used_files Env.prop('manage.last_update', time.time()) except: log.error('Failed updating library: %s', (traceback.format_exc())) while self.in_progress and len( self.in_progress) > 0 and not self.shuttingDown(): delete_me = {} for folder in self.in_progress: if self.in_progress[folder]['to_go'] <= 0: delete_me[folder] = True for delete in delete_me: del self.in_progress[delete] time.sleep(1) fireEvent('notify.frontend', type='manage.updating', data=False) self.in_progress = False
def _search(self, movie, quality, results): data = self.getHTMLData(self.urls['search'] % (self.conf('passkey'), getIdentifier(movie), self.conf('only_internal'))) if data: try: soup = BeautifulSoup(data) if soup.find('error'): log.error(soup.find('error').get_text()) return authkey = soup.find('authkey').get_text() entries = soup.find_all('torrent') for entry in entries: torrentscore = 0 torrent_id = entry.find('id').get_text() name = entry.find('name').get_text() year = entry.find('year').get_text() releasegroup = entry.find('releasegroup').get_text() resolution = entry.find('resolution').get_text() encoding = entry.find('encoding').get_text() freeleech = entry.find('freeleech').get_text() torrent_desc = '/ %s / %s / %s ' % (releasegroup, resolution, encoding) if freeleech == '0.25' and self.conf('prefer_internal'): torrent_desc += '/ Internal' torrentscore += 200 if encoding == 'x264' and self.conf('favor') in [ 'encode', 'both' ]: torrentscore += 300 if re.search('Remux', encoding) and self.conf('favor') in [ 'remux', 'both' ]: torrentscore += 200 results.append({ 'id': torrent_id, 'name': re.sub('[^A-Za-z0-9\-_ \(\).]+', '', '%s (%s) %s' % (name, year, torrent_desc)), 'url': self.urls['download'] % (torrent_id, authkey, self.conf('passkey')), 'detail_url': self.urls['detail'] % torrent_id, 'size': self.parseSize(entry.find('size').get_text()), 'seeders': tryInt(entry.find('seeders').get_text()), 'leechers': tryInt(entry.find('leechers').get_text()), 'score': torrentscore }) except: log.error('Failed getting results from %s: %s', (self.getName(), traceback.format_exc()))
def scan(self, folder=None, files=None, release_download=None, simple=False, newer_than=0, return_ignored=True, check_file_date=True, on_found=None): folder = sp(folder) if not folder or not os.path.isdir(folder): log.error('Folder doesn\'t exists: %s', folder) return {} # Get movie "master" files movie_files = {} leftovers = [] # Scan all files of the folder if no files are set if not files: try: files = [] for root, dirs, walk_files in os.walk(folder, followlinks=True): files.extend([ sp(os.path.join(sp(root), ss(filename))) for filename in walk_files ]) # Break if CP wants to shut down if self.shuttingDown(): break except: log.error('Failed getting files from %s: %s', (folder, traceback.format_exc())) log.debug('Found %s files to scan and group in %s', (len(files), folder)) else: check_file_date = False files = [sp(x) for x in files] for file_path in files: if not os.path.exists(file_path): continue # Remove ignored files if self.isSampleFile(file_path): leftovers.append(file_path) continue elif not self.keepFile(file_path): continue is_dvd_file = self.isDVDFile(file_path) if self.filesizeBetween( file_path, self.file_sizes['movie'] ) or is_dvd_file: # Minimal 300MB files or is DVD file # Normal identifier identifier = self.createStringIdentifier( file_path, folder, exclude_filename=is_dvd_file) identifiers = [identifier] # Identifier with quality quality = fireEvent('quality.guess', files=[file_path], size=self.getFileSize(file_path), single=True) if not is_dvd_file else { 'identifier': 'dvdr' } if quality: identifier_with_quality = '%s %s' % ( identifier, quality.get('identifier', '')) identifiers = [identifier_with_quality, identifier] if not movie_files.get(identifier): movie_files[identifier] = { 'unsorted_files': [], 'identifiers': identifiers, 'is_dvd': is_dvd_file, } movie_files[identifier]['unsorted_files'].append(file_path) else: leftovers.append(file_path) # Break if CP wants to shut down if self.shuttingDown(): break # Cleanup del files # Sort reverse, this prevents "Iron man 2" from getting grouped with "Iron man" as the "Iron Man 2" # files will be grouped first. leftovers = set(sorted(leftovers, reverse=True)) # Group files minus extension ignored_identifiers = [] for identifier, group in movie_files.items(): if identifier not in group['identifiers'] and len(identifier) > 0: group['identifiers'].append(identifier) log.debug('Grouping files: %s', identifier) has_ignored = 0 for file_path in list(group['unsorted_files']): ext = getExt(file_path) wo_ext = file_path[:-(len(ext) + 1)] found_files = set([i for i in leftovers if wo_ext in i]) group['unsorted_files'].extend(found_files) leftovers = leftovers - found_files has_ignored += 1 if ext in self.ignored_extensions else 0 if has_ignored == 0: for file_path in list(group['unsorted_files']): ext = getExt(file_path) has_ignored += 1 if ext in self.ignored_extensions else 0 if has_ignored > 0: ignored_identifiers.append(identifier) # Break if CP wants to shut down if self.shuttingDown(): break # Create identifiers for all leftover files path_identifiers = {} for file_path in leftovers: identifier = self.createStringIdentifier(file_path, folder) if not path_identifiers.get(identifier): path_identifiers[identifier] = [] path_identifiers[identifier].append(file_path) # Group the files based on the identifier delete_identifiers = [] for identifier, found_files in path_identifiers.items(): log.debug('Grouping files on identifier: %s', identifier) group = movie_files.get(identifier) if group: group['unsorted_files'].extend(found_files) delete_identifiers.append(identifier) # Remove the found files from the leftover stack leftovers = leftovers - set(found_files) # Break if CP wants to shut down if self.shuttingDown(): break # Cleaning up used for identifier in delete_identifiers: if path_identifiers.get(identifier): del path_identifiers[identifier] del delete_identifiers # Group based on folder delete_identifiers = [] for identifier, found_files in path_identifiers.items(): log.debug('Grouping files on foldername: %s', identifier) for ff in found_files: new_identifier = self.createStringIdentifier( os.path.dirname(ff), folder) group = movie_files.get(new_identifier) if group: group['unsorted_files'].extend([ff]) delete_identifiers.append(identifier) # Remove the found files from the leftover stack leftovers -= leftovers - set([ff]) # Break if CP wants to shut down if self.shuttingDown(): break # leftovers should be empty if leftovers: log.debug('Some files are still left over: %s', leftovers) # Cleaning up used for identifier in delete_identifiers: if path_identifiers.get(identifier): del path_identifiers[identifier] del delete_identifiers # Make sure we remove older / still extracting files valid_files = {} while True and not self.shuttingDown(): try: identifier, group = movie_files.popitem() except: break # Check if movie is fresh and maybe still unpacking, ignore files newer than 1 minute if check_file_date: files_too_new, time_string = self.checkFilesChanged( group['unsorted_files']) if files_too_new: log.info( 'Files seem to be still unpacking or just unpacked (created on %s), ignoring for now: %s', (time_string, identifier)) # Delete the unsorted list del group['unsorted_files'] continue # Only process movies newer than x if newer_than and newer_than > 0: has_new_files = False for cur_file in group['unsorted_files']: file_time = self.getFileTimes(cur_file) if file_time[0] > newer_than or file_time[1] > newer_than: has_new_files = True break if not has_new_files: log.debug( 'None of the files have changed since %s for %s, skipping.', (time.ctime(newer_than), identifier)) # Delete the unsorted list del group['unsorted_files'] continue valid_files[identifier] = group del movie_files total_found = len(valid_files) # Make sure only one movie was found if a download ID is provided if release_download and total_found == 0: log.info( 'Download ID provided (%s), but no groups found! Make sure the download contains valid media files (fully extracted).', release_download.get('imdb_id')) elif release_download and total_found > 1: log.info( 'Download ID provided (%s), but more than one group found (%s). Ignoring Download ID...', (release_download.get('imdb_id'), len(valid_files))) release_download = None # Determine file types processed_movies = {} while True and not self.shuttingDown(): try: identifier, group = valid_files.popitem() except: break if return_ignored is False and identifier in ignored_identifiers: log.debug('Ignore file found, ignoring release: %s', identifier) total_found -= 1 continue # Group extra (and easy) files first group['files'] = { 'movie_extra': self.getMovieExtras(group['unsorted_files']), 'subtitle': self.getSubtitles(group['unsorted_files']), 'subtitle_extra': self.getSubtitlesExtras(group['unsorted_files']), 'nfo': self.getNfo(group['unsorted_files']), 'trailer': self.getTrailers(group['unsorted_files']), 'leftover': set(group['unsorted_files']), } # Media files if group['is_dvd']: group['files']['movie'] = self.getDVDFiles( group['unsorted_files']) else: group['files']['movie'] = self.getMediaFiles( group['unsorted_files']) if len(group['files']['movie']) == 0: log.error('Couldn\'t find any movie files for %s', identifier) total_found -= 1 continue log.debug('Getting metadata for %s', identifier) group['meta_data'] = self.getMetaData( group, folder=folder, release_download=release_download) # Subtitle meta group['subtitle_language'] = self.getSubtitleLanguage( group) if not simple else {} # Get parent dir from movie files for movie_file in group['files']['movie']: group['parentdir'] = os.path.dirname(movie_file) group['dirname'] = None folder_names = group['parentdir'].replace(folder, '').split( os.path.sep) folder_names.reverse() # Try and get a proper dirname, so no "A", "Movie", "Download" etc for folder_name in folder_names: if folder_name.lower( ) not in self.ignore_names and len(folder_name) > 2: group['dirname'] = folder_name break break # Leftover "sorted" files for file_type in group['files']: if not file_type is 'leftover': group['files']['leftover'] -= set( group['files'][file_type]) group['files'][file_type] = list(group['files'][file_type]) group['files']['leftover'] = list(group['files']['leftover']) # Delete the unsorted list del group['unsorted_files'] # Determine movie group['media'] = self.determineMedia( group, release_download=release_download) if not group['media']: log.error('Unable to determine media: %s', group['identifiers']) else: group['identifier'] = getIdentifier( group['media']) or group['media']['info'].get('imdb') processed_movies[identifier] = group # Notify parent & progress on something found if on_found: on_found(group, total_found, len(valid_files)) # Wait for all the async events calm down a bit while threading.activeCount() > 100 and not self.shuttingDown(): log.debug('Too many threads active, waiting a few seconds') time.sleep(10) if len(processed_movies) > 0: log.info('Found %s movies in the folder %s', (len(processed_movies), folder)) else: log.debug('Found no movies in the folder %s', folder) return processed_movies
def _searchOnTitle(self, title, movie, quality, results): torrentlist = [] if self.conf('only_freeleech'): onlyFreelech = True else: onlyFreelech = False if self.conf('only_verified'): onlyVerified = True else: onlyVerified = False if not '/logout.php' in self.urlopen(self.urls['login'], data = self.getLoginParams()).lower(): log.info('problems logging into tehconnection.eu') return [] data = self.getHTMLData(self.urls['search'] % tryUrlencode(getIdentifier(movie))) if data: try: resultsTable = BeautifulSoup(data).find('table', attrs = {'id' : 'browse_torrent_table'}) if resultsTable is None: log.info('movie not found on TehConnection') return [] pagelinkdata = resultsTable.find("a", { "title" : "View Torrent" }) torrentpage = (pagelinkdata.attrs['href']).strip() indivTorrData = self.getHTMLData(self.urls['baseurl'] + (torrentpage)) soup = BeautifulSoup(indivTorrData) items = soup.findAll("div", { "class" : "torrent_widget box pad" }) for item in items: torrentData = TorrentDetails(0, 0, '', '', 0, '', '', False, False) detailstats = item.find("div", { "class" : "details_stats" }) #seeders seed = detailstats.find("img", { "title" : "Seeders" }).parent torrentData.seeders = ((seed.text).strip()) #leechers leech = detailstats.find("img", { "title" : "Leechers" }).parent torrentData.leechers = ((leech.text).strip()) #permalink perma = detailstats.find("a", { "title" : "Permalink" }) torrentData.permalink = self.urls['baseurl'] + perma.attrs['href'] #download link downlo = detailstats.find("a", { "title" : "Download" }) torrentData.downlink = self.urls['baseurl'] + downlo.attrs['href'] #Torrent ID m = re.search(r'\d+$', torrentData.permalink) torrentData.torrentID = (int(m.group()) if m else None) #TorrentName namedata = item.find("div", { "id" : "desc_%s" % torrentData.torrentID }) torrentData.torrentName = ((namedata.text).splitlines()[1]).strip() #FileSize sizedata = item.find("div", { "class" : "details_title" }) sizefile = ((sizedata.text).splitlines()[3]).replace("(","").replace(")","").strip() torrentData.filesize = sizefile #FreeLeech freeleechdata = item.find("span", { "class" : "freeleech" }) if freeleechdata is None: torrentData.freeleech = False else: torrentData.freeleech = True #QualityEncode qualityenc = detailstats.find("img", { "class" : "approved" }) if qualityenc is None: torrentData.qualityEncode = False else: torrentData.torrentName += " HQ" torrentData.qualityEncode = True #Test if the Freelech or Verified boxes have been checked & add depending if (onlyFreelech == False) or (onlyFreelech == True and torrentData.freeleech == True): #Only Freelech is switched off OR only Freelech is ON and the torrent is a freelech, so safe to add to results if (onlyVerified == False) or (onlyVerified == True and torrentData.qualityEncode == True): #Only Verified is switched off OR only Verified is ON and the torrent is verified, so safe to add to results torrentlist.append(torrentData) for torrentFind in torrentlist: log.info('TehConnection found ' + torrentFind.torrentName) results.append({ 'leechers': torrentFind.leechers, 'seeders': torrentFind.seeders, 'name': torrentFind.torrentName, 'url': torrentFind.downlink, 'detail_url': torrentFind.permalink, 'id': torrentFind.torrentID, 'size': self.parseSize(torrentFind.filesize) }) except: log.error('Failed getting results from %s: %s', (self.getName(), traceback.format_exc()))
def _search(self, movie, quality, results): imdbId = getIdentifier(movie).replace("t", "") url = self.urls['search'] % (imdbId) #, cats[0]) data = self.getHTMLData(url) if data: # Remove HDSpace NEW list split_data = data.partition( '<form name="tcategories" action="index.php" method="post">') data = split_data[2] html = BeautifulSoup(data) try: #Now attempt to get any others result_table = html.find('table', attrs={'class': 'lista'}) if not result_table: return entries = result_table.find_all('tr') log.info("entries length: %s", len(entries)) if not entries: return for result in entries: block2 = result.find_all('td', attrs={'class': 'header'}) # Ignore header if block2: continue cells = result.find_all('td') log.info("cells length: %s", len(cells)) extend = 0 detail = cells[1 + extend].find('a')['href'] torrent_id = detail.replace( 'index.php?page=torrent-details&id=', '') try: torrent_age = datetime.now() - datetime.strptime( cells[4 + extend].get_text().encode( 'ascii', 'ignore'), '%B %d, %Y,%H:%M:%S') except: torrent_age = timedelta(1) results.append({ 'id': torrent_id, 'name': cells[9 + extend].find('a')['title'].strip( 'History - ').replace('Blu-ray', 'bd50'), 'url': self.urls['home'] % cells[3 + extend].find('a')['href'], 'detail_url': self.urls['home'] % cells[1 + extend].find('a')['href'], 'size': self.parseSize(cells[5 + extend].get_text()), 'age': torrent_age.days, 'seeders': tryInt(cells[7 + extend].find('a').get_text()), 'leechers': tryInt(cells[8 + extend].find('a').get_text()), 'get_more_info': self.getMoreInfo, }) except: log.error('Failed getting results from %s: %s', (self.getName(), traceback.format_exc()))