def test_map_numbers(self):
input_num = 500
in_min = 1
in_max = 500
out_min = 10
out_max = 20
results = recommend_utils.map_numbers(input_num, in_min, in_max, out_min, out_max)
self.assertEqual(results, 20)
def test_map_numbers_1(self):
input_num = 2
in_min = 1
in_max = 500
out_min = 10
out_max = 20
results = recommend_utils.map_numbers(input_num, in_min, in_max, out_min, out_max)
self.assertEqual(results, 10.02004008016032)
def test_map_numbers_1(self):
input_num = 2
in_min = 1
in_max = 500
out_min = 10
out_max = 20
results = recommend_utils.map_numbers(input_num, in_min, in_max,
out_min, out_max)
self.assertEqual(results, 10.02004008016032)
def test_map_numbers(self):
input_num = 500
in_min = 1
in_max = 500
out_min = 10
out_max = 20
results = recommend_utils.map_numbers(input_num, in_min, in_max,
out_min, out_max)
self.assertEqual(results, 20)
def recommendation_logic(self):
"""
Recommendation logic is where the use of Collaborative vs Content differ. Both use the User Profile to get
infromation on the user, but with Content, it takes the content from user profiles on apps based on criteria
that is implemented in the calling method and gets a recommended list of apps to return to the user as
recommendations.
The list is then normalized and added to the recommendations database.
"""
all_profiles = models.Profile.objects.all()
all_profiles_count = len(all_profiles)
performed_search_request = False
new_user_return_list = []
current_profile_count = 0
for profile in all_profiles:
start_ms = time.time() * 1000.0
current_profile_count = current_profile_count + 1
profile_id = profile.id
es_query_result = self.es_content_based_recommendation(
profile_id, self.result_size)
# Check if results returned are returned or if it is empty (New User):
if es_query_result['hits']['total'] == 0:
if not performed_search_request:
new_user_return_list = self.new_user_return_list(
int(self.result_size / 2))
performed_search_request = True
recommended_items = new_user_return_list['hits']['hits']
max_score_es_content = new_user_return_list['hits'][
'max_score'] + self.content_norm_factor * new_user_return_list[
'hits']['max_score']
else:
recommended_items = es_query_result['hits']['hits']
max_score_es_content = es_query_result['hits'][
'max_score'] + self.content_norm_factor * es_query_result[
'hits']['max_score']
for indexitem in recommended_items:
score = recommend_utils.map_numbers(indexitem['_score'], 0,
max_score_es_content,
self.min_new_score,
self.max_new_score)
itemtoadd = indexitem['_source']['id']
self.profile_result_set.add_listing_to_user_profile(
profile_id, itemtoadd, score, False)
end_ms = time.time() * 1000.0
logger.debug('Calculated Profile {}/{}, took {} ms'.format(
current_profile_count, all_profiles_count,
round(end_ms - start_ms, 3)))
def recommendation_logic(self):
"""
Recommendation logic is where the use of Collaborative vs Content differ. Both use the User Profile to get
infromation on the user, but with Content, it takes the content from user profiles on apps based on criteria
that is implemented in the calling method and gets a recommended list of apps to return to the user as
recommendations.
The list is then normalized and added to the recommendations database.
"""
logger.info('= Elasticsearch Content Base Recommendation Engine= ')
all_profiles = models.Profile.objects.all()
all_profiles_count = all_profiles.count()
performed_search_request = False
new_user_return_list = []
current_profile_count = 0
for profile in all_profiles:
current_profile_count = current_profile_count + 1
profile_id = profile.id
es_query_result = self.es_content_based_recommendation(
profile_id, self.result_size)
# Check if results returned are returned or if it is empty (New User):
if es_query_result['hits']['total'] == 0:
if not performed_search_request:
new_user_return_list = self.new_user_return_list(
int(self.result_size / 2))
performed_search_request = True
recommended_items = new_user_return_list['hits']['hits']
max_score_es_content = new_user_return_list['hits'][
'max_score'] + self.content_norm_factor * new_user_return_list[
'hits']['max_score']
else:
recommended_items = es_query_result['hits']['hits']
max_score_es_content = es_query_result['hits'][
'max_score'] + self.content_norm_factor * es_query_result[
'hits']['max_score']
for indexitem in recommended_items:
score = recommend_utils.map_numbers(indexitem['_score'], 0,
max_score_es_content,
self.min_new_score,
self.max_new_score)
itemtoadd = indexitem['_source']['id']
self.add_listing_to_user_profile(profile_id, itemtoadd, score,
False)
logger.info(
"= ES CONTENT RECOMMENDER Engine Completed Results for {}/{} ="
.format(current_profile_count, all_profiles_count))
logger.info("= ES CONTENT RECOMMENDATION Results Completed =")
def recommendation_logic(self):
"""
Recommendation logic is where the use of Collaborative vs Content differ. Both use the User Profile to get
infromation on the user, but with Collaborative, it then matches against other users that have similar profiles to
return a recommendation.
Recommendation logic
- Cycle through each profile:
- Call ESRecommendUtils method to get User based Recommendations
- Take max score from results for profile and rebase all results while adding them to the recommendation list
- For each recommendation add it to the list while rescalling the score based on the max score returned
"""
# Retreive all of the profiles from database:
all_profiles = models.Profile.objects.all()
all_profiles_count = len(all_profiles)
current_profile_count = 0
for profile in all_profiles:
start_ms = time.time() * 1000.0
current_profile_count = current_profile_count + 1
profile_id = profile.id
recommended_items = self.es_user_based_recommendation(profile_id)
# recommended_item = [{'key_as_string': '154', 'doc_count': 3, 'score': 0.025898896404414385, 'bg_count': 3, 'key': 154},
# {'key_as_string': '1', 'doc_count': 3, 'score': 0.025898896404414385, 'bg_count': 3, 'key': 1},
# {'key_as_string': '173', 'doc_count': 2, 'score': 0.017265930936276253, 'bg_count': 2, 'key': 173}, ...]
# If a recommendaiton list is returned then get the max score,
# otherwise it is a new user or there is no profile to base recommendations on:
if recommended_items:
max_score_es_user = recommended_items[0]['score']
for indexitem in recommended_items:
score = recommend_utils.map_numbers(indexitem['score'], 0,
max_score_es_user,
self.min_new_score,
self.max_new_score)
self.profile_result_set.add_listing_to_user_profile(
profile_id, indexitem['key'], score, False)
end_ms = time.time() * 1000.0
logger.debug('Calculated Profile {}/{}, took {} ms'.format(
current_profile_count, all_profiles_count,
round(end_ms - start_ms, 3)))
def recommendation_logic(self):
"""
Recommendation logic is where the use of Collaborative vs Content differ. Both use the User Profile to get
infromation on the user, but with Collaborative, it then matches against other users that have similar profiles to
return a recommendation.
Recommendation logic
- Cycle through each profile:
- Call ESRecommendUtils method to get User based Recommendations
- Take max score from results for profile and rebase all results while adding them to the recommendation list
- For each recommendation add it to the list while rescalling the score based on the max score returned
"""
logger.info('= Elasticsearch User Base Recommendation Engine =')
# Retreive all of the profiles from database:
all_profiles = models.Profile.objects.all()
all_profiles_count = all_profiles.count()
current_profile_count = 0
for profile in all_profiles:
current_profile_count = current_profile_count + 1
profile_id = profile.id
recommended_items = self.es_user_based_recommendation(profile_id)
# If a recommendaiton list is returned then get the max score,
# otherwise it is a new user or there is no profile to base recommendations on:
if recommended_items:
max_score_es_user = recommended_items[0]['score']
for indexitem in recommended_items:
score = recommend_utils.map_numbers(indexitem['score'], 0,
max_score_es_user,
self.min_new_score,
self.max_new_score)
self.add_listing_to_user_profile(profile_id, indexitem['key'],
score, False)
logger.info(
"= ES USER RECOMMENDER Engine Completed Results for {}/{} =".
format(current_profile_count, all_profiles_count))
logger.info("= ES USER RECOMMENDATION Results Completed =")
def recommendation_logic(self):
"""
Recommendation logic is where the use of Collaborative vs Content differ. Both use the User Profile to get
infromation on the user, but with Content, it takes the content from user profiles on apps based on criteria
that is implemented in the calling method and gets a recommended list of apps to return to the user as
recommendations.
The list is then normalized and added to the recommendations database.
"""
all_profiles = models.Profile.objects.all()
all_profiles_count = len(all_profiles)
performed_search_request = False
new_user_return_list = []
current_profile_count = 0
for profile in all_profiles:
start_ms = time.time() * 1000.0
current_profile_count = current_profile_count + 1
profile_id = profile.id
es_query_result = self.es_content_based_recommendation(profile_id, self.result_size)
# Check if results returned are returned or if it is empty (New User):
if es_query_result['hits']['total'] == 0:
if not performed_search_request:
new_user_return_list = self.new_user_return_list(int(self.result_size / 2))
performed_search_request = True
recommended_items = new_user_return_list['hits']['hits']
max_score_es_content = new_user_return_list['hits']['max_score'] + self.content_norm_factor * new_user_return_list['hits']['max_score']
else:
recommended_items = es_query_result['hits']['hits']
max_score_es_content = es_query_result['hits']['max_score'] + self.content_norm_factor * es_query_result['hits']['max_score']
for indexitem in recommended_items:
score = recommend_utils.map_numbers(indexitem['_score'], 0, max_score_es_content, self.min_new_score, self.max_new_score)
itemtoadd = indexitem['_source']['id']
self.profile_result_set.add_listing_to_user_profile(profile_id, itemtoadd, score, False)
end_ms = time.time() * 1000.0
logger.debug('Calculated Profile {}/{}, took {} ms'.format(current_profile_count, all_profiles_count, round(end_ms - start_ms, 3)))
def recommendation_logic(self):
"""
Recommendation logic is where the use of Collaborative vs Content differ. Both use the User Profile to get
infromation on the user, but with Collaborative, it then matches against other users that have similar profiles to
return a recommendation.
Recommendation logic
- Cycle through each profile:
- Call ESRecommendUtils method to get User based Recommendations
- Take max score from results for profile and rebase all results while adding them to the recommendation list
- For each recommendation add it to the list while rescalling the score based on the max score returned
"""
# Retreive all of the profiles from database:
all_profiles = models.Profile.objects.all()
all_profiles_count = len(all_profiles)
current_profile_count = 0
for profile in all_profiles:
start_ms = time.time() * 1000.0
current_profile_count = current_profile_count + 1
profile_id = profile.id
recommended_items = self.es_user_based_recommendation(profile_id)
# recommended_item = [{'key_as_string': '154', 'doc_count': 3, 'score': 0.025898896404414385, 'bg_count': 3, 'key': 154},
# {'key_as_string': '1', 'doc_count': 3, 'score': 0.025898896404414385, 'bg_count': 3, 'key': 1},
# {'key_as_string': '173', 'doc_count': 2, 'score': 0.017265930936276253, 'bg_count': 2, 'key': 173}, ...]
# If a recommendaiton list is returned then get the max score,
# otherwise it is a new user or there is no profile to base recommendations on:
if recommended_items:
max_score_es_user = recommended_items[0]['score']
for indexitem in recommended_items:
score = recommend_utils.map_numbers(indexitem['score'], 0, max_score_es_user, self.min_new_score, self.max_new_score)
self.profile_result_set.add_listing_to_user_profile(profile_id, indexitem['key'], score, False)
end_ms = time.time() * 1000.0
logger.debug('Calculated Profile {}/{}, took {} ms'.format(current_profile_count, all_profiles_count, round(end_ms - start_ms, 3)))
def recommendation_logic(self):
"""
Sample Recommendations for all users
"""
all_profiles = models.Profile.objects.all()
all_profiles_count = len(all_profiles)
current_profile_count = 0
for profile in all_profiles:
start_ms = time.time() * 1000.0
current_profile_count = current_profile_count + 1
profile_id = profile.id
profile_username = profile.user.username
# Get Featured Listings
featured_listings = models.Listing.objects.for_user_organization_minus_security_markings(
profile_username).order_by('-approved_date').filter(
is_featured=True,
approval_status=models.Listing.APPROVED,
is_enabled=True,
is_deleted=False)
for current_listing in featured_listings:
self.profile_result_set.add_listing_to_user_profile(profile_id, current_listing.id, 3.0, True)
# Get Recent Listings
recent_listings = models.Listing.objects.for_user_organization_minus_security_markings(
profile_username).order_by(
'-approved_date').filter(
is_featured=False,
approval_status=models.Listing.APPROVED,
is_enabled=True,
is_deleted=False)
for current_listing in recent_listings:
self.profile_result_set.add_listing_to_user_profile(profile_id, current_listing.id, 2.0, True)
# Get most popular listings via a weighted average
most_popular_listings = models.Listing.objects.for_user_organization_minus_security_markings(
profile_username).filter(
approval_status=models.Listing.APPROVED,
is_enabled=True,
is_deleted=False).order_by('-avg_rate', '-total_reviews')
for current_listing in most_popular_listings:
if current_listing.avg_rate != 0:
self.profile_result_set.add_listing_to_user_profile(profile_id, current_listing.id, current_listing.avg_rate, True)
# Get most popular bookmarked apps for all users
# Would it be faster it this code was outside the loop for profiles?
library_entries = models.ApplicationLibraryEntry.objects.for_user_organization_minus_security_markings(profile_username)
library_entries = library_entries.filter(listing__is_enabled=True)
library_entries = library_entries.filter(listing__is_deleted=False)
library_entries = library_entries.filter(listing__approval_status=models.Listing.APPROVED)
library_entries_group_by_count = library_entries.values('listing_id').annotate(count=Count('listing_id')).order_by('-count')
# [{'listing_id': 1, 'count': 1}, {'listing_id': 2, 'count': 1}]
# Calculation of Min and Max new scores dynamically. This will increase the values that are lower
# to a range within 2 and 5, but will not cause values higher than new_min and new_max to become even
# larger.
old_min = 1
old_max = 1
new_min = 2
new_max = 5
for entry in library_entries_group_by_count:
count = entry['count']
if count == 0:
continue
if count > old_max:
old_max = count
if count < old_min:
old_min = count
for entry in library_entries_group_by_count:
listing_id = entry['listing_id']
count = entry['count']
calculation = recommend_utils.map_numbers(count, old_min, old_max, new_min, new_max)
self.profile_result_set.add_listing_to_user_profile(profile_id, listing_id, calculation, True)
end_ms = time.time() * 1000.0
logger.debug('Calculated Profile {}/{}, took {} ms'.format(current_profile_count, all_profiles_count, round(end_ms - start_ms, 3)))
def recommendation_logic(self):
"""
Sample Recommendations for all users
"""
all_profiles = models.Profile.objects.all()
all_profiles_count = len(all_profiles)
current_profile_count = 0
for profile in all_profiles:
start_ms = time.time() * 1000.0
current_profile_count = current_profile_count + 1
profile_id = profile.id
profile_username = profile.user.username
# Get Featured Listings
featured_listings = models.Listing.objects.for_user_organization_minus_security_markings(
profile_username).order_by('-approved_date').filter(
is_featured=True,
approval_status=models.Listing.APPROVED,
is_enabled=True,
is_deleted=False)
for current_listing in featured_listings:
self.profile_result_set.add_listing_to_user_profile(
profile_id, current_listing.id, 3.0, True)
# Get Recent Listings
recent_listings = models.Listing.objects.for_user_organization_minus_security_markings(
profile_username).order_by('-approved_date').filter(
is_featured=False,
approval_status=models.Listing.APPROVED,
is_enabled=True,
is_deleted=False)
for current_listing in recent_listings:
self.profile_result_set.add_listing_to_user_profile(
profile_id, current_listing.id, 2.0, True)
# Get most popular listings via a weighted average
most_popular_listings = models.Listing.objects.for_user_organization_minus_security_markings(
profile_username).filter(
approval_status=models.Listing.APPROVED,
is_enabled=True,
is_deleted=False).order_by('-avg_rate', '-total_reviews')
for current_listing in most_popular_listings:
if current_listing.avg_rate != 0:
self.profile_result_set.add_listing_to_user_profile(
profile_id, current_listing.id,
current_listing.avg_rate, True)
# Get most popular bookmarked apps for all users
# Would it be faster it this code was outside the loop for profiles?
library_entries = models.ApplicationLibraryEntry.objects.for_user_organization_minus_security_markings(
profile_username)
library_entries = library_entries.filter(listing__is_enabled=True)
library_entries = library_entries.filter(listing__is_deleted=False)
library_entries = library_entries.filter(
listing__approval_status=models.Listing.APPROVED)
library_entries_group_by_count = library_entries.values(
'listing_id').annotate(
count=Count('listing_id')).order_by('-count')
# [{'listing_id': 1, 'count': 1}, {'listing_id': 2, 'count': 1}]
# Calculation of Min and Max new scores dynamically. This will increase the values that are lower
# to a range within 2 and 5, but will not cause values higher than new_min and new_max to become even
# larger.
old_min = 1
old_max = 1
new_min = 2
new_max = 5
for entry in library_entries_group_by_count:
count = entry['count']
if count == 0:
continue
if count > old_max:
old_max = count
if count < old_min:
old_min = count
for entry in library_entries_group_by_count:
listing_id = entry['listing_id']
count = entry['count']
calculation = recommend_utils.map_numbers(
count, old_min, old_max, new_min, new_max)
self.profile_result_set.add_listing_to_user_profile(
profile_id, listing_id, calculation, True)
end_ms = time.time() * 1000.0
logger.debug('Calculated Profile {}/{}, took {} ms'.format(
current_profile_count, all_profiles_count,
round(end_ms - start_ms, 3)))