def test_for_updating_movie_in_db(self, mock_request):
mock_request.return_value = self.response_id550
Movie.objects.create(id=550, title="Fix Me")
Movie.save_movie_in_db(id=550)
self.assertEqual(Movie.objects.count(), 1)
self.assertEqual(Movie.objects.first().title,
self.response_id550["title"])
def test_save_movie_adds_movie_to_genre(self, mock_request):
mock_request.return_value = self.response_id550
mock_request.return_value = self.response_id550
genre = Genre.objects.create(id=18, name="Drama")
Movie.save_movie_in_db(id=550)
self.assertEqual(Movie.objects.get(id=550).genres.count(), 1)
self.assertEqual(
Movie.objects.get(id=550).genres.first().name, "Drama")
def get_movie(self,
id,
min_update_level=MIN_UPDATE_LEVEL,
force_update=False):
try:
movie = Movie.get(id,
min_update_level=min_update_level,
force_update=force_update)
movie_fields = [
field.name for field in Movie._meta.get_fields()
if field.related_model is None
]
data = model_to_dict(movie, fields=movie_fields)
data["genres"] = [{
"id": genre.id,
"name": genre.name
} for genre in movie.genres.all()]
data["w2w_status"] = "OK"
except requests.exceptions.RequestException as e:
data = {
"w2w_status": "ERROR",
"msg": "Connection error with TMDB."
}
except Movie.DoesNotExist:
data = {
"w2w_status": "ERROR",
"msg": "Movie with given id does not exist."
}
return data
def search_movies(self, query, page=1):
try:
data = tmdb_request(method="POST",
path="search/movie",
params={
"page": page,
"query": query
})
data["movies"] = data.pop("results", [])
data["w2w_status"] = "OK"
for movie in data["movies"]:
try:
Movie.objects.get(id=movie["id"],
update_level__gte=SEARCH_UPDATE_LEVEL)
except:
movie["update_level"] = SEARCH_UPDATE_LEVEL
Movie.save_movie_in_db(id=movie["id"], data=movie)
except requests.exceptions.RequestException:
data = {"w2w_status": "ERROR"}
return data
def get_movie(r):
return Movie(
tmdb_id=r.get('id'),
title=r.get('title'),
release_date=r.get('release_date'),
popularity=r.get('popularity'),
overview=r.get('overview') or '',
original_title=r.get('original_title'),
original_language=r.get('original_language'),
vote_count=r.get('vote_count'),
vote_average=r.get('vote_average'),
backdrop_path=r.get('backdrop_path') or '',
poster_path=r.get('poster_path') or '',
genres=r.get('genre_ids'),
is_adult=r.get('adult'),
is_video=r.get('video')
)
def test_returns_status_error_when_movie_exception(self, mock_request):
mock_request.return_value = self.mock_response
mock_request.side_effect = Movie.DoesNotExist()
data = Client().get_movie(id=550)
self.assertIn("w2w_status", data)
self.assertEqual(data["w2w_status"], "ERROR")
def test_propagates_request_exception(self, mock_tmdb):
mock_tmdb.side_effect = RequestException()
with self.assertRaises(RequestException):
movie = Movie.get(id=1, force_update=True)
def test_calls_tmdb_request_when_force_update(self, mock_tmdb):
Movie.objects.create(id=1, title="JAGO 2000", update_level=100)
mock_tmdb.return_value = {"id": 1, "title": "JAGO 2000"}
movie = Movie.get(id=1, force_update=True)
mock_tmdb.assert_called_with(method="GET", path="movie/1")
def test_calls_tmdb_request_if_two_low_update_level(self, mock_tmdb):
Movie.objects.create(id=1, title="JAGO 2000", update_level=0)
mock_tmdb.return_value = {"id": 1, "title": "JAGO 2000"}
movie = Movie.get(id=1,
min_update_level=MoviePopularQuery.update_level)
mock_tmdb.assert_called_with(method="GET", path="movie/1")
def test_raises_error_movie_does_not_exist_in_tmdb(self, mock_tmdb):
mock_tmdb.return_value = {"status_code": 34}
with self.assertRaises(Movie.DoesNotExist):
movie = Movie.get(id=1)
def test_saves_new_movie_in_db(self, mock_tmdb):
mock_tmdb.return_value = {"id": 1, "title": "JAGO 2000"}
movie = Movie.get(id=1)
self.assertEqual(Movie.objects.count(), 1)
self.assertEqual(movie.id, 1)
def test_raise_error_when_offline(self, mock_tmdb):
mock_tmdb.return_value = {"id": 1, "title": "JAGO 2000"}
with self.assertRaises(Movie.DoesNotExist):
movie = Movie.get(id=1, offline=True)
def test_calls_tmdb_request_if_movie_does_not_exist(self, mock_tmdb):
mock_tmdb.return_value = {"id": 1, "title": "JAGO 2000"}
movie = Movie.get(id=1)
mock_tmdb.assert_called_with(method="GET", path="movie/1")
def test_get_returns_correct_movie_object(self):
Movie.objects.create(id=1)
movie = Movie.get(id=1)
self.assertEqual(movie.id, 1)
def test_querying_db_for_wanted_movie(self, mock_get):
movie = Movie.get(id=1)
self.assertTrue(mock_get.called)
def handle(self, *args, **options):
path = options.get("path")[0]
batch = options.get("batch")
load_movies = options.get("movies")
load_ratings = options.get("ratings")
ratings_file = options.get("ratings_file")
# Load data from the files
self.stdout.write("Processing links.csv ...")
links = dict()
with open(os.path.join(path, "links.csv"), newline="") as csvfile:
reader = csv.DictReader(csvfile)
for row in reader:
try:
movie_id = int(row["movieId"])
tmdb_id = int(row["tmdbId"])
links[movie_id] = tmdb_id
except ValueError:
pass
if load_movies:
self.stdout.write("Processing movies.csv ...")
movies = list()
with open(os.path.join(path, "movies.csv"), newline="") as csvfile:
reader = csv.DictReader(csvfile)
for row in reader:
try:
movie_id = links[int(row["movieId"])]
title = row["title"]
movies.append({"id": movie_id, "title": title})
except KeyError:
pass
# Create coressponding movies in database
self.stdout.write("Creating movies in database ...")
movies2create = list()
movies_id = set()
for movie in movies:
if movie["id"] in movies_id:
continue
movies_id.add(movie["id"])
try:
Movie.objects.get(id=movie["id"])
except Movie.DoesNotExist:
movies2create.append(
Movie(id=movie["id"], title=movie["title"]))
if len(movies2create) > batch:
print(" - insarting ...")
Movie.objects.bulk_create(movies2create)
movies2create = list()
if len(movies2create) > 0:
Movie.objects.bulk_create(movies2create)
if load_ratings:
User = get_user_model()
chunk_index = 0
for chunk in pd.read_csv(os.path.join(path, ratings_file),
chunksize=100000):
self.stdout.write("Processing ratings.csv (chunk: %d) ..." %
chunk_index)
users = dict()
df = pd.DataFrame(chunk)
for index, row in df.iterrows():
try:
movie_id = links[int(row["movieId"])]
movie = {
"id": movie_id,
"rating": float(row["rating"]),
"timestamp": row["timestamp"]
}
user_id = row["userId"]
if user_id not in users:
users[user_id] = list()
users[user_id].append(movie)
except KeyError:
pass
# Create artificial users and their favourite movies list
self.stdout.write("Populating database with users ...")
users2create = list()
new_users = list()
for user in users:
try:
user_db = User.objects.get(
email="*****@*****.**" % user)
except User.DoesNotExist:
user_db = User(email="*****@*****.**" %
user,
is_real=False,
password="******")
new_users.append(user)
users2create.append(user_db)
if len(users2create) > batch:
print(" - insarting ...")
User.objects.bulk_create(users2create)
users2create = list()
if len(users2create) > 0:
User.objects.bulk_create(users2create)
print("%d users saved." % len(new_users))
self.stdout.write("Creating users' preferences lists ...")
prefs2create = list()
for user in new_users:
user_db = User.objects.get(
email="*****@*****.**" % user)
pref_db = PrefList(user=user_db)
prefs2create.append(pref_db)
if len(prefs2create) > batch:
print(" - insarting ...")
PrefList.objects.bulk_create(prefs2create)
prefs2create = list()
if len(prefs2create) > 0:
PrefList.objects.bulk_create(prefs2create)
print("%d preferences lists saved." % PrefList.objects.count())
self.stdout.write("Populating database with preferences ...")
prefs2create = list()
for user in users:
user_db = User.objects.get(
email="*****@*****.**" % user)
preflist_db = user_db.pref
for pref in users[user]:
timestamp = datetime.datetime.fromtimestamp(
int(pref["timestamp"]))
timestamp = timezone.make_aware(
timestamp, timezone.get_current_timezone())
movie = Movie.objects.get(id=pref["id"])
pref_db = MoviePref(preflist=preflist_db,
movie=movie,
rating=self.adjust_rating(
int(pref["rating"])),
timestamp=timestamp)
prefs2create.append(pref_db)
if len(prefs2create) > batch:
print(" - insarting ...")
MoviePref.objects.bulk_create(prefs2create)
prefs2create = list()
if len(prefs2create) > 0:
MoviePref.objects.bulk_create(prefs2create)
print("%d preferences saved." % MoviePref.objects.count())
chunk_index += 1