def user_recommendations(user, films, with_rated=False, order_by=None):
film_ids = [f.id for f in films]
if user and user.is_authenticated():
profile = user.get_profile()
basket = BasketItem.user_basket(user)
rated_ids = set(Rating.get_user_ratings(user).keys())
if profile.recommendations_status in (profile.NORMAL_RECOMMENDATIONS,
profile.FAST_RECOMMENDATIONS):
recommendations = recommendations_engine.compute_guess_score_for_films(
user, film_ids)
else:
recommendations = {}
for f in films:
r = recommendations.get(f.id)
b = basket.get(f.id)
f._rated = f.id in rated_ids
f._guess_rating = r or 0
f._on_wishlist = b and b[0] and (
b[0] != BasketItem.NOT_INTERESTED) or False
f._on_shitlist = b and (b[0] == BasketItem.NOT_INTERESTED) or False
else:
for f in films:
f._rated = False
f._guess_rating = 0
f._on_wishlist = False
f._on_shitlist = False
test_with_rated = lambda f: with_rated or not f._rated
films = list(f for f in films if not f._on_shitlist and test_with_rated(f))
comparator = create_film_comparator(order_by)
return sorted(films, cmp=comparator)
def save(self, **kw):
ratings = Rating.get_user_ratings(self.user)
number_of_ratings = len(ratings)
if number_of_ratings:
average_rating = decimal.Decimal(sum(ratings.values())) / number_of_ratings
else:
averate_rating = None
ratings_based_prediction = None
traits_based_prediction = None
film_id = self.cleaned_data['film_id']
vote = self.cleaned_data['vote'] or None
defaults = {
'vote': vote,
'average_rating': average_rating,
'number_of_ratings': number_of_ratings,
'ratings_based_prediction': ratings_based_prediction,
'traits_based_prediction': traits_based_prediction,
}
defaults.update(kw)
vote, created = RecommendationVote.objects.get_or_create(user=self.user, film_id=film_id, defaults=defaults)
if not created:
vars(vote).update(defaults)
vote.save()
return vote
def user_recommendations(user, films, with_rated=False, order_by=None):
film_ids = [f.id for f in films]
if user and user.is_authenticated():
profile = user.get_profile()
basket = BasketItem.user_basket(user)
rated_ids = set(Rating.get_user_ratings(user).keys())
if profile.recommendations_status in (profile.NORMAL_RECOMMENDATIONS, profile.FAST_RECOMMENDATIONS):
recommendations = recommendations_engine.compute_guess_score_for_films(user, film_ids)
else:
recommendations = {}
for f in films:
r = recommendations.get(f.id)
b = basket.get(f.id)
f._rated = f.id in rated_ids
f._guess_rating = r or 0
f._on_wishlist = b and b[0] and (b[0] != BasketItem.NOT_INTERESTED) or False
f._on_shitlist = b and (b[0] == BasketItem.NOT_INTERESTED) or False
else:
for f in films:
f._rated = False
f._guess_rating = 0
f._on_wishlist = False
f._on_shitlist = False
test_with_rated = lambda f: with_rated or not f._rated
films = list(f for f in films if not f._on_shitlist and test_with_rated(f))
comparator = create_film_comparator(order_by)
return sorted(films, cmp=comparator)
def get_excluded_films(self):
ids = set()
ids.update(self.get_seen_films())
if self.user.id:
ids.update(Rating.get_user_ratings(self.user).keys())
ids.update(Film._marked_film_ids(self.user))
return ids
def count_guessed_rate(user, film):
ratings = Rating.get_user_ratings(user)
logger.info("counting guessed rate for film %d", film.id)
if len(ratings) < MIN_USER_RATES:
return None
average_rate, average_weight = AVERAGE_RATE, AVERAGE_WEIGHT
film_weight = 0.0
film_rate = film.average_score() or 0.0
film_rate = float(film_rate)
if film_rate:
film_weight = FILM_WEIGHT * pow(
float(film.number_of_votes() / FILM_RELIABILITY_BAR), 0.3)
user_rate = sum(ratings.values()) / len(ratings)
user_weight = USER_WEIGHT * (len(ratings) / USER_RELIABILITY_BAR) ** 0.3
# film_ids = ratings.keys()
# scores = dict(FilmComparator.objects.filter(
# main_film=film, compared_film__id__in=film_ids).\
# values_list('compared_film', 'score'))
key = cache.Key("compared_films", film.id)
scores = cache.get(key)
if scores is None:
query = FilmComparator.objects.filter(main_film=film).values_list(
'compared_film', 'score')
scores = dict((f, float(s)) for (f, s) in query)
cache.set(key, scores)
sum_weights = 0.0
sum_rate = 0.0
for film_id, rating in ratings.items():
if film_id in scores:
weight = float(scores[film_id])
sum_weights += weight
sum_rate += float(rating) * weight
sum_rate += film_weight * film_rate + user_weight * user_rate
sum_rate += average_weight * average_rate
sum_weights += film_weight + user_weight + average_weight
recommendation = None
if sum_weights > 0.0 and sum_rate >= 0.0:
score = Decimal(str(sum_rate / sum_weights))
if score <= 10:
recommendation, created = Recommendation.objects.get_or_create(
user=user,
film=film,
defaults=dict(guess_rating_alg2=score),
)
if not created:
recommendation.guess_rating_alg2 = score
recommendation.save()
# transaction.commit()
return recommendation
def get_best_psi_films_queryset(self,
user,
tags=None,
year_from=None,
year_to=None,
related_director=None,
related_actor=None,
popularity=None,
include_features=(),
exclude_features=(),
netflix=None,
exclude_production_country=None):
from film20.recommendations import engine
is_filtered = tags or year_from or year_to or related_director or related_actor or popularity or netflix
if exclude_production_country:
kw = dict(exclude_production_country=exclude_production_country)
else:
kw = {}
all_scores = engine.compute_guess_score_for_all_films(user, **kw)
all_scores = dict((key, val) for key, val in all_scores.items() if val)
ratings = Rating.get_user_ratings(user)
shitlisted = set(k for k, v in BasketItem.user_basket(user).items()
if v[0] == 9)
film_ids = set(all_scores.keys())
film_ids.difference_update(ratings.keys())
film_ids.difference_update(shitlisted)
if is_filtered:
filtered = self._filter_films(tags, year_from, year_to,
related_director, related_actor,
popularity, netflix)
filtered = filtered.filter(id__in=film_ids)
film_ids = filtered.values_list('id', flat=True)
if include_features:
from film20.film_features.models import FilmFeature
featured_ids = FilmFeature.matching_film_ids(
include_features, exclude_features)
film_ids = [id for id in film_ids if id in featured_ids]
film_ids = sorted(film_ids, key=lambda f: -all_scores[f])
from film20.utils.misc import ListWrapper
class FilmWrapper(ListWrapper):
def wrap(self, items):
films = list(Film.objects.filter(id__in=items))
films.sort(key=lambda f: -all_scores[f.id])
for film in films:
film.guess_rating = all_scores.get(film.id)
yield film
return FilmWrapper(film_ids)
def count_guessed_rate(user, film):
ratings = Rating.get_user_ratings(user)
logger.info("counting guessed rate for film %d", film.id)
if len(ratings) < MIN_USER_RATES:
return None
average_rate, average_weight = AVERAGE_RATE, AVERAGE_WEIGHT
film_weight = 0.0
film_rate = film.average_score() or 0.0
film_rate = float(film_rate)
if film_rate:
film_weight = FILM_WEIGHT * pow(float(film.number_of_votes() / FILM_RELIABILITY_BAR), 0.3)
user_rate = sum(ratings.values()) / len(ratings)
user_weight = USER_WEIGHT * (len(ratings) / USER_RELIABILITY_BAR) ** 0.3
# film_ids = ratings.keys()
# scores = dict(FilmComparator.objects.filter(
# main_film=film, compared_film__id__in=film_ids).\
# values_list('compared_film', 'score'))
key = cache.Key("compared_films", film.id)
scores = cache.get(key)
if scores is None:
query = FilmComparator.objects.filter(main_film=film).values_list("compared_film", "score")
scores = dict((f, float(s)) for (f, s) in query)
cache.set(key, scores)
sum_weights = 0.0
sum_rate = 0.0
for film_id, rating in ratings.items():
if film_id in scores:
weight = float(scores[film_id])
sum_weights += weight
sum_rate += float(rating) * weight
sum_rate += film_weight * film_rate + user_weight * user_rate
sum_rate += average_weight * average_rate
sum_weights += film_weight + user_weight + average_weight
recommendation = None
if sum_weights > 0.0 and sum_rate >= 0.0:
score = Decimal(str(sum_rate / sum_weights))
if score <= 10:
recommendation, created = Recommendation.objects.get_or_create(
user=user, film=film, defaults=dict(guess_rating_alg2=score)
)
if not created:
recommendation.guess_rating_alg2 = score
recommendation.save()
# transaction.commit()
return recommendation
def next_film_to_vote( self, user, exclude=[], all_movies=False, tag=None ):
voted = [ ff.film.pk for ff in FilmFeatureVote.objects.filter( user=user ) ]
if all_movies:
films = Film.objects.all()
else:
rated = Rating.get_user_ratings( user )
films = Film.objects.filter( pk__in=rated )
if tag:
films = films.tagged( tag )
films = films.exclude( pk__in=voted + exclude ).order_by( '?' )
return films[0] if len( films ) else None
def next_to_vote( self, user ):
items = self.get_query_set().filter( user=user, status=FilmFeatureComparisionVote.STATUS_UNKNOWN )
if len( items ):
return items[0]
rated_films = Film.objects.filter( pk__in=Rating.get_user_ratings( user ) )
similar_films = SimilarFilm.objects.filter( film_a__in=rated_films, film_b__in=rated_films ).values_list( 'film_a', 'film_b' ).order_by( '?' )
films_with_features = itertools.combinations( set( FilmFeature.objects.filter( film__in=rated_films ).values_list( 'film', flat=True ).order_by( '?' ) ), 2 )
films = [similar_films, films_with_features]
random.shuffle( films )
for f in films:
next_to_rate = self._find_next_to_rate( user, f )
if next_to_rate:
return next_to_rate
def get_best_psi_films_queryset(self, user, tags=None, year_from=None, year_to = None, related_director = None, related_actor = None, popularity = None, include_features=(), exclude_features=(), netflix=None, exclude_production_country=None):
from film20.recommendations import engine
is_filtered = tags or year_from or year_to or related_director or related_actor or popularity or netflix
if exclude_production_country:
kw = dict(exclude_production_country=exclude_production_country)
else:
kw = {}
all_scores = engine.compute_guess_score_for_all_films(user, **kw)
all_scores = dict((key, val) for key, val in all_scores.items() if val)
ratings = Rating.get_user_ratings(user)
shitlisted = set(k for k,v in BasketItem.user_basket(user).items() if v[0]==9)
film_ids = set(all_scores.keys())
film_ids.difference_update(ratings.keys())
film_ids.difference_update(shitlisted)
if is_filtered:
filtered = self._filter_films(tags, year_from, year_to, related_director, related_actor, popularity, netflix)
filtered = filtered.filter(id__in=film_ids)
film_ids = filtered.values_list('id', flat=True)
if include_features:
from film20.film_features.models import FilmFeature
featured_ids = FilmFeature.matching_film_ids(include_features, exclude_features)
film_ids = [id for id in film_ids if id in featured_ids]
film_ids = sorted(film_ids, key=lambda f: -all_scores[f])
from film20.utils.misc import ListWrapper
class FilmWrapper(ListWrapper):
def wrap(self, items):
films = list(Film.objects.filter(id__in=items))
films.sort(key=lambda f:-all_scores[f.id])
for film in films:
film.guess_rating = all_scores.get(film.id)
yield film
return FilmWrapper(film_ids)
def __init__(self, data, request, film, initial=None):
initial = initial or {}
self.film = film
self.user = request.unique_user
initial['film_id'] = film.id
if not 'rating_1' in initial:
ratings = Rating.get_user_ratings(self.user)
r = ratings.get(self.film.id)
if r:
initial['rating_1'] = r
if self.user.is_authenticated():
# collections only for registered users
basket = BasketItem.user_basket(self.user)
item = basket.get(film.id)
if item:
initial['dying_for'] = item[0] == BasketItem.DYING_FOR
initial['not_interested'] = item[0] == BasketItem.NOT_INTERESTED
initial['owned'] = item[1] == BasketItem.OWNED
prefix = "flm-%d" % film.id
super(forms.Form, self).__init__(data, initial=initial, prefix=prefix)
def __init__(self, data, user=None, film=None, mark_as_seen=False, initial=None):
initial = initial or {}
self.film = film
self.user = user
if user and user.is_authenticated():
if not 'rating_1' in initial:
ratings = Rating.get_user_ratings(user)
r = ratings.get(self.film.id)
if r:
initial['rating_1'] = r
basket = BasketItem.user_basket(user)
item = basket.get(film.id)
if item:
initial['dying_for'] = item[0] == BasketItem.DYING_FOR
initial['not_interested'] = item[0] == BasketItem.NOT_INTERESTED
initial['owned'] = item[1] == BasketItem.OWNED
if mark_as_seen:
film.mark_as_seen(user)
prefix = "flm-%d" % film.id
super(forms.Form, self).__init__(data, initial=initial, prefix=prefix)
def __init__(self, data, request, film, initial=None):
initial = initial or {}
self.film = film
self.user = request.unique_user
initial['film_id'] = film.id
if not 'rating_1' in initial:
ratings = Rating.get_user_ratings(self.user)
r = ratings.get(self.film.id)
if r:
initial['rating_1'] = r
if self.user.is_authenticated():
# collections only for registered users
basket = BasketItem.user_basket(self.user)
item = basket.get(film.id)
if item:
initial['dying_for'] = item[0] == BasketItem.DYING_FOR
initial['not_interested'] = item[
0] == BasketItem.NOT_INTERESTED
initial['owned'] = item[1] == BasketItem.OWNED
prefix = "flm-%d" % film.id
super(forms.Form, self).__init__(data, initial=initial, prefix=prefix)
def get_user_ratings(user_id):
return Rating.get_user_ratings(user_id)
def save(self, *args, **kw):
email_confirmed = False
username = self.cleaned_data["username"]
email = self.cleaned_data.get("email") or ""
password = self.cleaned_data.get("password1")
tmp_username = self._tmp_username()
if not tmp_username and self.request and self.request.unique_user.id:
tmp_username = self.request.unique_user.username
if tmp_username:
match = re.match("(\w{3})-[0-9a-f]{20,26}$", tmp_username)
match = match or re.match("(fb)-\d+", tmp_username)
assert match
# rename anonymouse user
user = User.objects.get(username=tmp_username, is_active=False)
user.is_active = True
user.username = username
user.email = email
if password:
user.set_password(password)
user.save()
profile = user.get_profile()
profile.user = user
profile.registration_source = match.group(1)
profile.save()
# update username in activities
for activity in UserActivity.objects.filter(user=user):
activity.username = user.username
activity.save()
if (
settings.RECOMMENDATIONS_ENGINE == "film20.new_recommendations.recommendations_engine"
and Rating.get_user_ratings(user) > settings.RECOMMENDATIONS_MIN_VOTES_USER
):
# recompute recommendations
from film20.recommendations import engine
engine.compute_user_features(user, True)
ec = self.get_email_confirmation()
if ec and ec.email_address.email == email:
EmailConfirmation.objects.confirm_email(self.request.GET.get("confirmation_key"))
email_confirmed = True
else:
# User.objects.create_user(username, email, **extra)
user, created = User.objects.get_or_create(username=username, defaults={"email": email})
if created:
if password:
user.set_password(password)
user.save()
else:
# user already exists, probably double form submission
return user
if email and not email_confirmed:
add_email(user, email)
defaults = {}
if self.request and self.request.geo:
latitude = self.request.geo.get("latitude")
longitude = self.request.geo.get("longitude")
country_code = self.request.geo.get("country_code")
timezone = self.request.geo.get("timezone")
if latitude and longitude:
defaults["latitude"] = str(latitude)
defaults["longitude"] = str(longitude)
defaults["country"] = country_code
defaults["timezone_id"] = timezone
profile, created = Profile.objects.get_or_create(user=user, defaults=defaults)
if not created:
profile.__dict__.update(defaults)
profile.save()
from film20.notification import models as notification
from film20.notification.models import NoticeType
inform_friends = self.cleaned_data.get("inform_friends")
if not inform_friends:
# disable all USER_ACTIVITY notifications
notice_types = NoticeType.objects.filter(type=NoticeType.TYPE_USER_ACTIVITY)
for notice_type in notice_types:
for medium in notification.NOTICE_MEDIA:
if medium.supports(notice_type):
medium.update_notification_setting(user, notice_type, False)
if self.request.is_mobile:
names = dict((v, k) for k, v in settings.USER_AGENTS)
platform_name = names.get(self.request.platform, "")
link = reverse("mobile_app")
picture = None
else:
platform_name = ""
link = reverse("main_page")
picture = "/static/layout/logo.png"
notification.send(
[user],
"useractivity_just_joined",
{
"user": user,
"is_mobile": self.request.is_mobile,
"platform": self.request.platform,
"platform_name": platform_name,
"link": link,
"picture": picture,
},
delay=30,
)
return user
def create_comparator_for_two_users(curuser, compared_user, new_user=False):
# data for creating a RatingComparator object
oryg_films_compared = 0
films_compared = 0
sum_difference = 0
comparator = None
## TODO: this has to be optimized. This data can be extracted in one query,
## having compared_user and user rating for each film in one row
# for new users get all their ratings
if new_user:
# new comparator
comparator = RatingComparator()
comparator.main_user = curuser
comparator.compared_user = compared_user
curuser_ratings = Rating.objects.filter(
user=compared_user.id,
type=Rating.TYPE_FILM,
rating__isnull=False
)
# for old users (that we have comparators with) get only
else:
# get old comparator
try:
comparator = RatingComparator.objects.get(main_user=curuser.id, compared_user=compared_user.id)
films_compared = comparator.common_films
oryg_films_compared = comparator.common_films
sum_difference = comparator.sum_difference
ddebug('Fetched: ' + unicode(comparator))
# this means not set
if sum_difference == -1:
sum_difference = oryg_films_compared * comparator.score
ddebug('Re-computed sum difference: ' + unicode(sum_difference))
except RatingComparator.DoesNotExist:
debug("FATAL ERROR!!! CANNOT GET COMPARATOR FOR USERS: " + unicode(curuser.username) + " and " + unicode(compared_user.username))
debug("Creating new one... but this is WRONG!!!")
comparator = RatingComparator()
comparator.main_user = curuser
comparator.compared_user = compared_user
ratings1 = Rating.get_user_ratings(curuser)
ratings2 = Rating.get_user_ratings(compared_user)
common_films = set(ratings1.keys()).intersection(ratings2.keys())
sum_difference = sum(abs(ratings1[id]-ratings2[id]) for id in common_films)
films_compared = len(common_films)
if True:
ddebug('Films compared: ' + unicode(films_compared))
ddebug('Accum. difference: ' + unicode(sum_difference))
if oryg_films_compared != films_compared:
if films_compared > MIN_FILMS_COMPARED:
the_score = Decimal(sum_difference)/Decimal(films_compared)
comparator.score = the_score
comparator.common_films = films_compared
comparator.sum_difference = sum_difference
# save every other time to make sure that we always update both comparators (user1 vs user2 and user2 vs user1)
# THIS IS NOT NECESSARY ANYMORE (we only need to do it once)
# if comparator.previous_save_date:
# comparator.previous_save_date = None
# else:
# comparator.previous_save_date = comparator.updated_at
comparator.previous_save_date = None
comparator.save()
# create/update the other object
try:
other_comparator = RatingComparator.objects.get(main_user=compared_user.id, compared_user=curuser.id)
ddebug('Fetched the other comparator: ' + unicode(comparator))
except RatingComparator.DoesNotExist:
ddebug("Creating the other comparator")
other_comparator = RatingComparator()
other_comparator.main_user = compared_user
other_comparator.compared_user = curuser
other_comparator.score = comparator.score
other_comparator.common_films = comparator.common_films
other_comparator.sum_difference = comparator.sum_difference
other_comparator.previous_save_date = comparator.previous_save_date
other_comparator.save()
debug("Updated ratings between: " + unicode(curuser) + ", " + unicode(compared_user) + "[" + unicode(comparator.score) + "]")
else:
ddebug("Too few common films ("+unicode(films_compared)+") between users: " + unicode(curuser) + ", " + unicode(compared_user))
else:
ddebug("No new films to be compared between users: " + unicode(curuser) + ", " + unicode(compared_user))
