def update_series(cls, series_id):
"""
Refetches series, and refetches its seasons IF it has changes
"""
series_id = int(series_id)
series = Series.get_by_id(series_id)
seasons_changed = TMDB.seasons_changed_in_series(series_id)
for season_number in seasons_changed:
# refetch seasons that have changed
new_season = Season.from_json(
TMDB.season(series_id, season_number))
# store the season
series.set_season(new_season)
# refetch series
new_series = Series.from_json(TMDB.series(series_id))
# store seasons
new_series.seasons = series.seasons
new_series.put()
logging.info("Updated series: %s %s" % (series_id, series.name))
return new_series
def load_series(series_id):
"""
Loads series, season, episode into db.
"""
#check if already in database
series = Series.get_by_id(series_id)
if series is not None:
logging.info("Tried to re-add series %d" % series_id)
return False
# fetch from TMDB
series_json = TMDB.series(series_id)
if not series_json:
return False;
series = Series.from_json(series_json)
for i in range(1, series.number_of_seasons() + 1):
season_json = TMDB.season(series_id, season_number=i)
if not season_json:
continue
season = Season.from_json(season_json)
series.set_season(season)
series.put()
return True
def sync(self):
"""
Reloads shows to db that were changed in the last 24 hours
"""
# Get changed ids from tmdb
changed_ids = TMDB.tv_changed_ids()
# track # of changes
updated_count = AppStat.get_by_id("daily_sync_count")
if not updated_count:
updated_count = AppStat(id="daily_sync_count",
description="Number of seasons updated last night")
updated_count.value = 0
# if a show in db changed, update
new_series_list = list()
for changed_id in changed_ids:
series = Series.get_by_id(changed_id)
if series:
# enqueue update series
self.add_update_series(changed_id)
updated_count.value += 1
updated_count.put()
return
import unittest
from tmdb import TMDB
"""
Run Tests
python test_tmdb.py
python test_tmdb.py TMDBTestCase.test_search
python test_tmdb.py TMDBTestCase.test_info
python test_tmdb.py TMDBTestCase.test_recommendations
"""
tmdb = TMDB()
ANIME_ID_1 = 12971 # DBZ
ANIME_ID_2 = 295830 # AOT
MOVIE_ID_1 = 120 # LOTR Fellowship
MOVIE_ID_2 = 447404 # Detective Pikachu
SHOW_ID_1 = 95834 # Legend of Fei
SHOW_ID_2 = 1399 # GoT
class TMDBTestCase(unittest.TestCase):
def display_list(self, lst):
"""
Prints the results in user friendly view
Args: list of movies and shows
"""
for i in range(len(lst)):
def refetch(cls):
data = TMDB.configuration()
config = cls(id=cls.STRING_ID, json=data)
config.put()
return config
def main(argv):
optional_args = '[-y <year>] [-d] [--allow-chronologies] [--high-quality] [-s <season>] [-e <episode>] [--debug]'
try:
opts, args = getopt.getopt(argv, "hdn:y:s:e:", [
"name=", "year=", "download", "allow-chronologies", "high-quality",
"season=", "episode=", "debug"
])
except getopt.GetoptError:
print "find_torrents.py -n '<name of movie>' " + optional_args
sys.exit(2)
year = ''
download = False
chronologies = False
high_quality = False
season = None
episode = None
debug = False
for opt, arg in opts:
if opt == '-h':
print "find_torrents.py -n '<name of movie>' " + optional_args
sys.exit()
elif opt in ("-n", "--name"):
name = arg
elif opt in ("-d", "--download"):
download = True
elif opt in ("-y", "--year"):
year = arg
elif opt == '--allow-chronologies':
chronologies = True
elif opt == '--high-quality':
high_quality = True
elif opt in ("-s", "--season"):
season = '{:02d}'.format(int(arg))
season = int(arg)
elif opt in ("-e", "--episode"):
episode = '{:02d}'.format(int(arg))
episode = int(arg)
elif opt == '--debug':
debug = True
if season or episode:
video_type = 'television'
else:
video_type = 'movie'
logger = logging.getLogger(__name__)
handler = logging.FileHandler('find_torrent.log')
handler.setLevel(logging.DEBUG)
formatter = logging.Formatter(
'%(asctime)s %(name)-12s %(levelname)-8s %(message)s')
handler.setFormatter(formatter)
logger.addHandler(handler)
if debug:
logger.setLevel(logging.DEBUG)
else:
logger.setLevel(logging.INFO)
logger.debug("arguments: %s" % ' '.join(argv))
transmission = Transmission()
transmission_creds = transmission.user + ':' + transmission.password
tmdb = TMDB()
tmdb_url = 'https://api.themoviedb.org/3/search/movie'
title_variations = []
# logger.debug("name.split() = %s" % name.split())
for delimiter in [' ', '.', '-', '_']:
variation = delimiter.join(name.split())
logger.debug(variation)
title_variations.append(variation)
query = re.sub(r'[\s-]', '+', name)
if year:
logger.debug("year: %s" % year)
if video_type == 'movie':
if year == '':
logger.debug('searching for year on tmdb')
r = requests.get(tmdb_url + '?api_key=' + tmdb.key + '&query=' +
name)
tmdb_results = r.json()
logger.debug("len(tmdb_results): %s" % len(tmdb_results))
for result in tmdb_results['results']:
logger.debug(result)
if result['title'] == name:
logger.debug("result['title']: %s" % result['title'])
m = re.match(r'(\d{4})-\d{2}-\d{2}',
result['release_date'])
year = m.group(1)
logger.info(
"this film appears to have been released in %s" % year)
break
query = "%s+%s" % (query, year)
if year:
logger.debug("year: %s" % year)
elif video_type == 'television':
if season:
if episode:
catalog = "S%sE%s" % ('{:02d}'.format(season),
'{:02d}'.format(episode))
else:
catalog = "Season+%s" % str(season)
logger.debug("catalog: %s" % catalog)
query = "%s+%s" % (query, catalog)
logger.debug("query: %s" % query)
great_words = title_variations
if year != '':
great_words.append(year)
good_words = [
'BrRip', 'BDRip', 'BRRip', 'BluRay', 'Bluray', 'x264', 'H.264'
]
bad_words = ['DVDR', 'PAL', 'DvDrip', 'DVDrip', 'DVDscr', '480p']
avoid_words = ['YIFY', 'H.265', 'h265', 'x265', 'HEVC']
if '3D' not in name:
avoid_words.append('3D')
chronology_words = [
'chronology', 'collection', 'sequel', '1, 2', '1&2', '1 & 2', 'series',
'duology', 'trilogy', 'triology', 'quadrilogy', 'tetralogy',
'pentalogy', 'hexalogy', 'heptalogy'
]
sequel_words = ['part', 'chapter']
if video_type == 'movie':
sequel_words.append('episode')
numbers = [str(x) for x in range(1, 11)]
numbers = numbers + [
'i', 'ii', 'iii', 'iv', 'v', 'vi', 'vii', 'viii', 'ix', 'x'
]
sequel_phrases = [w + ' ' + x for w in sequel_words for x in numbers]
if video_type == 'movie':
too_small = 1.0
ideal_min = 4.0
ideal_max = 9.0
elif video_type == 'television':
too_small = 0.12
ideal_min = 0.7
ideal_max = 1.5
too_big = 17.0
if high_quality:
hiq_min = ideal_min + ((too_big - ideal_min) / 2)
hiq_max = too_big
else:
hiq_min = ideal_min
hiq_max = ideal_max
torrents = []
# scrape torrent site
search_results = []
r = requests.get("https://kickass.cd/search.php?q=%s" % query)
# r = requests.get("https://kickass2.org/search.php?q=%s" % query)
search_results.append(r.content)
if len(search_results) == 0:
logger.critical('no HTML response from kickass.cd')
sys.exit(1)
else:
logger.info('got HTML response from kickass.cd')
#html_doc = open('webpage2.html', 'r')
# parse scraped HTML
start_list = []
final_list = []
for doc in search_results:
soup = BeautifulSoup(doc, 'lxml')
div = soup.find(id='mainSearchTable')
# div = soup.find(class_='mainpart')
if not div:
sys.exit("couldn't find mainpart div")
logger.debug('found mainSearchTable div')
rows = div.find_all('tr', class_='odd')
if len(rows) == 0:
logger.critical('query returned no results')
sys.exit(1)
else:
logger.info("query returned %s results" % len(rows))
for tr in div.find_all('tr', class_='odd'):
score = 0
rejected = False
link = tr.find('a', {'title': 'Torrent magnet link'})['href']
title = tr.find('a', class_='cellMainLink').get_text()
logger.debug("scraped title: %s" % title)
size = tr.find_all('td')[1].get_text()
size = re.sub('<[^<]+?>', '', size)
if isinstance(size, unicode):
size = size.encode('ascii', 'ignore')
trs = tr.find_all('td')[3]
# logger.debug(tr.find_all('td')[3])
try:
seeders = tr.find_all('td')[3].get_text()
seeders = re.sub('<[^<]+?>', '', seeders)
seeders = int(seeders)
except:
logger.debug(
"couldn't display HTML in seeders column. setting seeders to a default of 1"
)
seeders = 1
size = size.replace(' ', ' ')
if 'MiB' in size or 'MB' in size:
size = re.sub(r'Mi*B', '', size)
size = float(size.split()[0]) / 1024
elif 'GiB' in size or 'GB' in size:
size = re.sub(r'Gi*B', '', size)
size = float(size.split()[0])
else:
rejected = True
size = 0
logger.debug('rejected due to size')
chronology_matches = [
w for w in chronology_words
if w in title.lower() and w not in name.lower()
]
#logger.debug(chronology_matches)
sequel_matches = [
w for w in sequel_phrases
if w in title.lower() and w not in name.lower()
]
#logger.debug(sequel_matches)
if ((any(chronology_matches) or any(sequel_matches)
or re.search(r'\d{4}[-\s]\d{4}', title))
and not chronologies):
rejected = True
logger.debug(
'rejected because it looks like a sequel or collection')
if title not in [t['title'] for t in torrents]:
if seeders == 0:
rejected = True
logger.debug('rejected because there are no seeders')
if size < too_small:
rejected = True
logger.debug("rejected because %.2f GB is too small" %
size)
if size > too_big:
rejected = True
logger.debug("rejected because %.2f GB is too big" % size)
if any(w in title for w in avoid_words):
rejected = True
rejected_words = ' '.join(
list(set(title.split()).intersection(avoid_words)))
logger.debug(
"rejected because torrent name contains the following keywords: %s"
% rejected_words)
if not rejected:
torrent = {
'title': title,
'link': link,
'size': size,
'seeders': seeders,
'score': score,
'res': ''
}
start_list.append(torrent)
logger.info("%s results look relevant" % len(start_list))
# categorize torrents based on resolution
logger.debug("separating torrents by resolution")
hd1080 = []
hd720 = []
other = []
for torrent in start_list:
if any(w in torrent['title'] for w in ['1080p', '1080i']):
hd1080.append(torrent)
torrent['res'] = '1080p'
elif '720p' in torrent['title']:
hd720.append(torrent)
torrent['res'] = '720p'
else:
other.append(torrent)
torrent['res'] = ''
logger.debug("\t1080i/p: %s torrents" % len(hd1080))
logger.debug("\t720p: %s torrents" % len(hd720))
logger.debug("\tother: %s torrents" % len(other))
# score torrents based on keywords, size, and number of seeders
for tlist in [hd1080, hd720, other]:
for torrent in tlist:
logger.debug(torrent['title'])
delta = math.log(torrent['seeders'], 16)
logger.debug("\tnumber of seeders: %+.2f points" % delta)
torrent['score'] = torrent['score'] + delta
for w in great_words:
if w in torrent['title']:
if w == year:
delta = 2
else:
delta = 4
logger.debug("\tmatched great_words: %+.2f points" % delta)
torrent['score'] = torrent['score'] + delta
for w in good_words:
if w in torrent['title']:
delta = 1
logger.debug("\tmatched good_words: %+.2f points" % delta)
torrent['score'] = torrent['score'] + delta
for w in bad_words:
if w in torrent['title']:
delta = -1
logger.debug("\tmatched bad_words: %+.2f points" % delta)
torrent['score'] = torrent['score'] + delta
if season:
delta = 2
if episode:
if catalog in torrent['title']:
logger.debug("\tmatched season/episode: %+.2f points" %
delta)
torrent['score'] = torrent['score'] + delta
else:
if re.search(r"season[\s\.\-_]*0*%d" % season,
torrent['title'], re.I):
logger.debug("\tmatched season: %+.2f points" % delta)
torrent['score'] = torrent['score'] + delta
if torrent['size'] < ideal_min:
delta = 1.0 * (
(math.log(torrent['size'] / math.log(torrent['size'] + 1))
+ 1) +
(math.log(torrent['size'] / math.log(torrent['size'] + 1))
* 15) - 6) / 2
logger.debug("\tsmaller than ideal file size: %+.2f points" %
delta)
torrent['score'] = torrent['score'] + delta
if torrent['size'] > ideal_max:
delta = -1
logger.debug("\tlarger than ideal file size: %+.2f points" %
delta)
torrent['score'] = torrent['score'] + delta
if torrent['size'] > hiq_min or torrent['size'] < hiq_max:
if high_quality:
delta = (torrent['size'] / 2) * math.log(torrent['size'])
#else:
# delta = 1
logger.debug("\tideal file size: %+.2f points" % delta)
torrent['score'] = torrent['score'] + delta
# sort torrents by score
for torrent in sorted(tlist, key=lambda t: t['score'], reverse=True):
final_list.append(torrent)
logger.debug("final list contains %s torrents" % len(final_list))
if video_type == 'television':
logger.debug(
'merging 1080 and 720p content scores because this is a television program'
)
final_list = sorted(final_list, key=lambda t: t['score'], reverse=True)
# return the results
if download:
if len(final_list) < 1:
logger.info("final list was empty")
sys.exit("no results")
logger.info("downloading %s" % final_list[0]['link'])
# print final_list[0]['link']
sp = subprocess.Popen(' '.join([
'transmission-remote', '--auth', transmission_creds, '-a',
final_list[0]['link']
]),
shell=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
sp_data = []
sp_data = sp.communicate()
# print(sp_data)
if sp_data[1]:
if re.search(r"transmission-remote.*Couldn\'t connect to server",
sp_data[1]):
sys.exit("transmission-daemon unreachable")
elif re.search(r"invalid or corrupt torrent file", sp_data[0]):
sys.exit("invalid or corrupt torrent file")
else:
print("found a %s copy of %s" %
(final_list[0]['res'], final_list[0]['title']))
else:
logger.info("not downloading")
if len(final_list) > 0:
logger.info('---------------- RESULTS ----------------')
logger.info("score\tseeders\tsize\t title")
print("score\tseeders\tsize\t title")
for torrent in final_list[:10]:
result = "%.2f\t%s\t%.1f GiB\t %s" % (
torrent['score'], torrent['seeders'], torrent['size'],
torrent['title'])
print(result)
logger.info(result)
logger.info('-----------------------------------------')
else:
logger.warning("no results!")
print "Error: no results"
import json
import multiprocessing as mp
import pathlib as pl
from tmdb import TMDB
from cache import cache_json
with open('/home/wbhorn/Documents/keys/moviedb.key', 'r') as f:
API_KEY = f.read().strip()
DB = TMDB(API_KEY)
def main():
with open('movies.csv') as f:
movie_titles = f.read().split('\n')
threaded(movie_titles)
def threaded(movie_titles):
pool = mp.Pool(5)
movie_results = pool.map(scrape, movie_titles)
pool.map(download_poster, filter(lambda x: x, movie_results))
def single(movie_titles):
movie_results = [scrape(movie) for movie in movie_titles]
for r in filter():
download_poster(r)
# -*- coding: utf-8 -*-
"""
Functions that bot use
All of them return tuple of 2 elements (except send_movie_info, send_movies_by_keyword):
text to send and keyboard
"""
from tmdb import TMDB
from telebot.types import InlineKeyboardMarkup, InlineKeyboardButton, ReplyKeyboardMarkup
import messages
import config
tmdb_obj = TMDB(config.API_KEY)
def send_start_message():
# Sends key "Genres" and start message
search_keyboard = ReplyKeyboardMarkup(one_time_keyboard=True,
resize_keyboard=True)
search_keyboard.row("Жанры")
return messages.start_message, search_keyboard
def send_genres_message():
# Sends genres callback keyboard
markup = InlineKeyboardMarkup()
markup.row_width = 2
genres = tmdb_obj.get_all_genres()
for i in range(0, len(genres) - 1, 2):
markup.add(
InlineKeyboardButton(genres[i]["name"],
#!/usr/bin/env python
"""Application init module"""
from tmdb import TMDB
from media_manager import MediaManager
import fresh_tomatoes
__author__ = "Christiaan Lombard <*****@*****.**>"
# initialize api and media manager
api = TMDB("e8b472cbe1c343ed368509f11912186f")
manager = MediaManager(api)
# get lists of movies
print "Getting a list of popular movies..."
popular_movies = manager.list_popular_movies()
print "Getting a list of favorite movies..."
favorite_movies = manager.list_favorite_movies()
# create HTML and open in browser
fresh_tomatoes.open_movies_page(favorite_movies, popular_movies)
from tmdb import TMDB tmdb = TMDB(api_key='e37969a8464642a43ea8e9094892b371')
from config import DB_URI, TMDB_KEY
from tmdb import TMDB
from pymongo import MongoClient
# DB_URI should be a url to mongodb with ?retryWrites=false
client = MongoClient(DB_URI)
class DataBase:
def __init__(self, client, db_name):
self.db = client[db_name]
print("Initialized database.")
def add_list(self, list):
print("Adding a list to database:", list)
self.db.lists.insert_one(list)
tmdb = TMDB(TMDB_KEY)
tmdb.search_tv('Lucifer')
# TODO:
# When TheMovieDB adds tv-show lists
# Search TV Show -> Get lists -> add tv shows which are in the same list
# That way select the recommended tv show