def get(self, request, user_external_id):
"""
Get the users owned items. It receives an extra set of GET parameters. The offset and the items.
The ´´offset´´ represents the amount of items to pass before start sending results.
The ´´items´´ represents the amount of items to show.
:param request: The HTTP request.
:param user_external_id: The user external id that are making the request.
:return: A list of app external ids of the user owned (items that are in database with reference to this
user and the dropped date set to null).
"""
offset = int(request.GET.get("offset", 0)) # Offset of items
number_of_items = int(request.GET.get("items",
0)) # Number of items to present
limit = (offset + number_of_items) or None
user = User.get_user_by_external_id(user_external_id)
try:
items = [{
"external_id": Item.get_item_external_id_by_id(item_id),
"is_dropped": is_dropped
} for item_id, is_dropped in User.get_user_items(user.pk).items()
[offset:limit]]
except KeyError:
return self.format_response(self.NOT_FOUND_ERROR_MESSAGE,
status=NOT_FOUND_ERROR)
data = {"user": user_external_id, "items": items}
return self.format_response(data)
def setup_class(cls, *args, **kwargs):
"""
Put elements in db
"""
for language in LANGUAGES:
Locale.objects.create(**language)
for app in ITEMS:
item = Item.objects.create(id=app["id"],
name=app["name"],
external_id=app["external_id"])
for language in app["languages"]:
l = Locale.objects.get(country_code=language)
ItemLocale.objects.create(locale=l, item=item)
for u in USERS:
user = User.objects.create(id=u["id"],
external_id=u["external_id"])
for i in u["items"]:
Inventory.objects.create(user=user,
item=Item.get_item_by_external_id(i))
for language in u["languages"]:
l = Locale.objects.get(country_code=language)
UserLocale.objects.create(locale=l, user=user)
Item.load_to_cache()
User.load_to_cache()
Locale.load_to_cache()
def test_diversity_on_recommendation_25(self):
"""
[recommendation.api.GetRecommendation] Test diversity for size 25 recommendation (at least 2/3 of user genres)
"""
size = 25
response = \
self.client.get("/api/v2/recommend/%d/"
"00b65a359307654a7deee7c71a7563d2816d6b7e522377a66aaefe8848da5961/" % size)
user_id = User.get_user_id_by_external_id("00b65a359307654a7deee7c71a7563d2816d6b7e522377a66aaefe8848da5961")
user_genres = ItemGenre.genre_in(
Item.get_item_by_id(item_id) for item_id in User.get_user_items(user_id)
)
recommendation_genres = ItemGenre.genre_in(
Item.get_item_by_external_id(item_eid) for item_eid in json.loads(response.content)["recommendations"]
)
less, more = (user_genres, recommendation_genres) if len(user_genres) < len(recommendation_genres) else \
(recommendation_genres, user_genres)
measure = 0
for genre in less:
if genre in more:
measure += 1
assert measure > len(less)*2./3., \
"Not sufficient genres in recommendation" \
"(user: %d, recommendation: %d, similarity: %d)" % (len(user_genres), len(recommendation_genres), measure)
def test_user_genres_in_recommendation_size_25(self):
"""
[recommendation.api.GetRecommendation] At least 19 of the top genres in the size 25 recommendation
"""
get_cache("default").clear()
LogEntry.objects.all().delete()
size = 25
response = \
self.client.get("/api/v2/recommend/%d/"
"00b65a359307654a7deee7c71a7563d2816d6b7e522377a66aaefe8848da5961/" % size)
user_id = User.get_user_id_by_external_id("00b65a359307654a7deee7c71a7563d2816d6b7e522377a66aaefe8848da5961")
user_genres = sorted(ItemGenre.genre_in(
Item.get_item_by_id(item_id) for item_id in User.get_user_items(user_id)
).items(), key=lambda x: x[1], reverse=True)
recommendation_genres = ItemGenre.genre_in(
Item.get_item_by_external_id(item_eid) for item_eid in json.loads(response.content)["recommendations"]
)
measure = []
for no, (genre, _) in enumerate(user_genres[:int(size)], start=1):
if genre not in recommendation_genres:
measure.append(no)
assert len(measure) < 6, "Major genres failing by index: %s." \
"\nUser %s" \
"\nRecommendation %s" % (
measure, user_genres, [ItemGenre.genre_in([Item.get_item_by_external_id(item)])
for item in json.loads(response.content)["recommendations"]])
def get(self, request, user_external_id):
"""
Get the users owned items. It receives an extra set of GET parameters. The offset and the items.
The ´´offset´´ represents the amount of items to pass before start sending results.
The ´´items´´ represents the amount of items to show.
:param request: The HTTP request.
:param user_external_id: The user external id that are making the request.
:return: A list of app external ids of the user owned (items that are in database with reference to this
user and the dropped date set to null).
"""
offset = int(request.GET.get("offset", 0)) # Offset of items
number_of_items = int(request.GET.get("items", 0)) # Number of items to present
limit = (offset+number_of_items) or None
user = User.get_user_by_external_id(user_external_id)
try:
items = [
{
"external_id": Item.get_item_external_id_by_id(item_id),
"is_dropped": is_dropped
} for item_id, is_dropped in User.get_user_items(user.pk).items()[offset:limit]
]
except KeyError:
return self.format_response(self.NOT_FOUND_ERROR_MESSAGE, status=NOT_FOUND_ERROR)
data = {"user": user_external_id, "items": items}
return self.format_response(data)
def load_rest():
User.load_to_cache()
# Load main models
Popularity.load_to_cache()
TensorCoFi.load_to_cache()
if "recommendation.language" in settings.INSTALLED_APPS:
from recommendation.language.models import Region
Region.load_to_cache()
if "recommendation.diversity" in settings.INSTALLED_APPS:
from recommendation.diversity.models import ItemGenre, Genre
Genre.load_to_cache()
ItemGenre.load_to_cache()
def setup_class(cls, *args, **kwargs):
"""
Put elements in db
"""
path = resource_filename(recommendation.__name__, "/")
fill.FillTool({"items": True, "--mozilla": True, "prod": True}).load()
fill.FillTool({"users": True, "--mozilla": True, "<path>": path+"data/user"}).load()
modelcrafter.main("train", "popularity")
modelcrafter.main("train", "tensorcofi")
# Load user and items
Item.load_to_cache()
User.load_to_cache()
# Load main models
Popularity.load_to_cache()
TensorCoFi.load_to_cache()
cls.client = Client()
def test_filter_regions(self):
"""
[recommendation.filter.Region] Test a region filter on recommendation
"""
rfilter = SimpleRegionFilter()
recommendation = [random.random() for _ in range(len(ITEMS))]
app_rec = [
Item.get_item_by_id(aid + 1).external_id for aid, _ in sorted(
enumerate(recommendation), key=lambda x: x[1], reverse=True)
]
for u in USERS:
user = User.get_user_by_external_id(u["external_id"])
with self.assertNumQueries(0):
result = rfilter(user, np.array(recommendation[:]))
new_rec = [
Item.get_item_by_id(aid + 1).external_id for aid, _ in sorted(
enumerate(result), key=lambda x: x[1], reverse=True)
]
n = len(u["last_apps_region"])
if n == 0:
assert new_rec == app_rec, \
"Recommendation is not the same when not apps to filter(%s != %s)" % (new_rec, app_rec)
else:
for item in u["last_apps_region"]:
assert item in new_rec[0-n:], \
"Item %s for user %s is not in the last in recommendation %s. Items filtered %s." % \
(item, user, new_rec, list(u["last_apps_region"]))
def test_get_item_by_external_id(self):
"""
[recommendation.models.User] Test queries by external id made by getting user and check integrity of that user
"""
with self.assertNumQueries(0):
for u in USERS:
user = User.get_user_by_external_id(u["external_id"])
assert isinstance(user, User), "Cached user is not instance of User."
def test_size_of_logs_in_cache(self):
"""
[recommendation.cache.SimpleLogger] Test size of cache is 10 for all users in system
"""
for user in USERS:
user = User.get_user_by_external_id(user["external_id"])
assert len(LogEntry.get_logs_for(user.pk)) == 10, \
"logs size are bigger than predicted (%s != 10)" % len(LogEntry.get_logs_for(user.pk))
def test_size_of_logs_in_cache(self):
"""
[recommendation.cache.SimpleLogger] Test size of cache is 10 for all users in system
"""
for user in USERS:
user = User.get_user_by_external_id(user["external_id"])
assert len(LogEntry.get_logs_for(user.pk)) == 10, \
"logs size are bigger than predicted (%s != 10)" % len(LogEntry.get_logs_for(user.pk))
def test_user_items(self):
"""
[recommendation.models.User] Test user items
"""
for u in USERS:
user = User.get_user_by_external_id(u["external_id"])
for i in u["items"]:
assert Item.get_item_by_external_id(i).pk in user.all_items, \
"Item %s is not in user %s" % (i, user.external_id)
def setup_class(cls, *args, **kwargs):
"""
Put elements in db
"""
for region in REGIONS:
Region.objects.create(**region)
for app in ITEMS:
item = Item.objects.create(id=app["id"], name=app["name"], external_id=app["external_id"])
for region in app["regions"]:
ItemRegion.objects.create(region=Region.objects.get(slug=region), item=item)
for u in USERS:
user = User.objects.create(id=u["id"], external_id=u["external_id"])
for i in u["items"]:
Inventory.objects.create(user=user, item=Item.get_item_by_external_id(i))
UserRegion.objects.create(region=Region.objects.get(slug=u["region"]), user=user)
Item.load_to_cache()
User.load_to_cache()
Region.load_to_cache()
def test_recommendation_size_after_filter(self):
"""
[recommendation.filter.Region] Test the size of the recommendation after the filter
"""
rfilter = SimpleRegionFilter()
recommendation = [random.random() for _ in range(len(ITEMS))]
for u in USERS:
user = User.get_user_by_external_id(u["external_id"])
with self.assertNumQueries(0):
result = rfilter(user, np.array(recommendation[:]))
new_rec = [aid+1 for aid, _ in sorted(enumerate(result), key=lambda x: x[1], reverse=True)]
assert len(new_rec) == len(ITEMS), "Recommendation size changed (%d != %s)" % (len(new_rec), len(ITEMS))
def setup_class(cls, *args, **kwargs):
"""
Put elements in db
"""
for language in LANGUAGES:
Locale.objects.create(**language)
for app in ITEMS:
item = Item.objects.create(id=app["id"], name=app["name"], external_id=app["external_id"])
for language in app["languages"]:
l = Locale.objects.get(country_code=language)
ItemLocale.objects.create(locale=l, item=item)
for u in USERS:
user = User.objects.create(id=u["id"], external_id=u["external_id"])
for i in u["items"]:
Inventory.objects.create(user=user, item=Item.get_item_by_external_id(i))
for language in u["languages"]:
l = Locale.objects.get(country_code=language)
UserLocale.objects.create(locale=l, user=user)
Item.load_to_cache()
User.load_to_cache()
Locale.load_to_cache()
def test_click_logging(self):
"""
[recommendation.decorator.SimpleLogger] Test if a clicked item is logged
"""
logger = LogEvent(LogEvent.CLICK)
user = User.get_user_by_external_id("joaonrb")
logger(lambda uid, iid: None)(user, Item.get_item_by_external_id("10004"))
time.sleep(1.)
logs = list(LogEntry.objects.filter(user=user, type=logger.CLICK))
assert len(logs) == 1, "Number of register is not correct %s" % logs
assert "10004" == logs[0].item.external_id, \
"The item in log is incorrect(1004 != %s)" % logs[0].item.external_id
def test_acquire_logging(self):
"""
[recommendation.decorator.SimpleLogger] Test if a acquired item is logged
"""
logger = LogEvent(LogEvent.ACQUIRE)
user = User.get_user_by_external_id("joaonrb")
logger(lambda uid, iid: None)(user,
Item.get_item_by_external_id("10004"))
time.sleep(1.)
logs = list(LogEntry.objects.filter(user=user, type=logger.ACQUIRE))
assert len(logs) == 1, "Number of register is not correct %s" % logs
assert "10004" == logs[0].item.external_id, \
"The item in log is incorrect(1004 != %s)" % logs[0].item.external_id
def test_reranker_diversity(self):
"""
[recommendation.diversity.ReRanker] Test a diversity re-ranker on recommendation
"""
diversity = SimpleDiversityReRanker()
recommendation = [Item.get_item_by_external_id(i).pk for i in ("10001", "10002", "10003", "10004", "98766")]
shuffle(recommendation)
for u in USERS:
user = User.get_user_by_external_id(u["external_id"])
with self.assertNumQueries(0):
result = diversity(user=user, recommendation=recommendation[:], size=5)
new_rec = [aid+1 for aid, _ in sorted(enumerate(result), key=lambda x: x[1], reverse=True)]
assert len(result) == len(ITEMS), "Recommendation size changed (%d != %s)" % (len(new_rec), len(ITEMS))
def test_recommendation_update_user(self):
"""
[recommendation.api.GetUserItems] Test update user items
"""
response = self.client.put(
"/api/v2/user-items/00504e6196ab5fa37ae7450dad99d031a80c50ef4b762c15151a2e4e92c64e0b/",
'{"user_items": ["504343", "413346"]}',
content_type="application/json"
)
sleep(0.8)
user = User.get_user_by_external_id("00504e6196ab5fa37ae7450dad99d031a80c50ef4b762c15151a2e4e92c64e0b")
assert response.status_code == 200, "Request failed. Status code %d." % response.status_code
assert len(user.owned_items) == 2, "Owned items should be 3(%d)" % len(user.owned_items)
assert Item.get_item_by_external_id("504343").pk in user.owned_items, "New item not in owned items"
assert Item.get_item_by_external_id("413346").pk in user.owned_items, "New item not in owned items"
def test_recommendation_acquire_new_item(self):
"""
[recommendation.api.GetUserItems] Test acquire new item
"""
user = User.get_user_by_external_id("00504e6196ab5fa37ae7450dad99d031a80c50ef4b762c15151a2e4e92c64e0b")
items = user.owned_items
response = self.client.post(
"/api/v2/user-items/00504e6196ab5fa37ae7450dad99d031a80c50ef4b762c15151a2e4e92c64e0b/",
{"item_to_acquire": "504343"}
)
sleep(0.8)
assert response.status_code == 200, "Request failed. Status code %d." % response.status_code
assert len(user.owned_items) == len(items)+1, "Owned items should be %d(%d)" % (len(items)+1,
len(user.owned_items))
assert Item.get_item_by_external_id("504343").pk in user.owned_items.keys(), "New item not in owned items"
def test_owned_items(self):
"""
[recommendation.models.User] Test owned items
"""
for u in USERS:
user = User.get_user_by_external_id(u["external_id"])
for i in u["items"]:
ivent = Inventory.objects.get(item=user.all_items[Item.get_item_by_external_id(i).pk], user=user)
ivent.is_dropped = True
ivent.save()
assert Item.get_item_by_external_id(i).pk not in user.owned_items, \
"Item %s is in user %s owned items" % (i, user.external_id)
ivent = Inventory.objects.get(item=user.all_items[Item.get_item_by_external_id(i).pk], user=user)
ivent.is_dropped = False
ivent.save()
def test_recommendation_remove_new_item(self):
"""
[recommendation.api.GetUserItems] Test remove old item
"""
response = self.client.delete(
"/api/v2/user-items/006a508fe63e87619db5c3db21da2c536f24e296c29d885e4b48d0b5aa561173/",
"item_to_remove=413346",
content_type="application/x-www-form-urlencoded; charset=UTF-8"
)
sleep(0.8)
assert response.status_code == 200, "Request failed. Status code %d. Message: %s" % \
(response.status_code, json.loads(response.content).get("error", ""))
assert len(User.get_user_by_external_id(
"006a508fe63e87619db5c3db21da2c536f24e296c29d885e4b48d0b5aa561173").owned_items) == 0, \
"Owned items should be 0"
def setup_class(cls, *args, **kwargs):
"""
Put elements in db
"""
for region in REGIONS:
Region.objects.create(**region)
for app in ITEMS:
item = Item.objects.create(id=app["id"],
name=app["name"],
external_id=app["external_id"])
for region in app["regions"]:
ItemRegion.objects.create(
region=Region.objects.get(slug=region), item=item)
for u in USERS:
user = User.objects.create(id=u["id"],
external_id=u["external_id"])
for i in u["items"]:
Inventory.objects.create(user=user,
item=Item.get_item_by_external_id(i))
UserRegion.objects.create(
region=Region.objects.get(slug=u["region"]), user=user)
Item.load_to_cache()
User.load_to_cache()
Region.load_to_cache()
def get_external_id_recommendations(self, user, n=10):
"""
Returns the recommendations with a list of external_is's
:param user: Same parameters that get_app_significance
:param n:
:return: Item external id list
"""
try:
user = User.get_user_by_external_id(user)
except IndexError:
logging.info("User %s not exist. Is going to be created")
user = User.objects.create(external_id=user)
result = self.get_recommendation(user=user, n=n)
return [Item.get_item_external_id_by_id(r) for r in result]
def test_filter_owned(self):
"""
[recommendation.filter.OwnedIems] Test a filter owned items on recommendation
"""
rfilter = FilterOwned()
recommendation = [random.random() for _ in range(len(ITEMS))]
for u in USERS:
user = User.get_user_by_external_id(u["external_id"])
with self.assertNumQueries(0):
result = rfilter(user, np.array(recommendation[:]))
new_rec = [aid+1 for aid, _ in sorted(enumerate(result), key=lambda x: x[1], reverse=True)]
n = len(user.owned_items)
for item in user.owned_items:
assert item in new_rec[0-n:], "Item %d is not in the last in recommendation %s. User owned items %s" % \
(item, new_rec, list(user.owned_items.keys()))
def test_recommendation_size_after_filter(self):
"""
[recommendation.filter.Region] Test the size of the recommendation after the filter
"""
rfilter = SimpleRegionFilter()
recommendation = [random.random() for _ in range(len(ITEMS))]
for u in USERS:
user = User.get_user_by_external_id(u["external_id"])
with self.assertNumQueries(0):
result = rfilter(user, np.array(recommendation[:]))
new_rec = [
aid + 1 for aid, _ in sorted(
enumerate(result), key=lambda x: x[1], reverse=True)
]
assert len(new_rec) == len(
ITEMS), "Recommendation size changed (%d != %s)" % (
len(new_rec), len(ITEMS))
def test_recommendation_logging(self):
"""
[recommendation.decorator.SimpleLogger] Test if a recommendation is logged
"""
logger = LogEvent(LogEvent.RECOMMEND)
user = User.get_user_by_external_id("joaonrb")
recommendation = [Item.get_item_by_external_id(i).pk for i in ("10001", "10002", "10003", "10004", "98766")]
recommendation = logger(lambda user: recommendation)(user=user)
time.sleep(1.)
logs = list(LogEntry.objects.filter(user=user, type=logger.RECOMMEND).order_by("value"))
assert len(logs) == 5, "Number of register is not correct %s" % logs
logs_iter = iter(logs)
for i, item in enumerate(recommendation, start=1):
log = logs_iter.next()
assert item == log.item.pk, "The item in position %d is not the same that in recommendation " \
"(%s != %s)" % (i, item, log.item.external_id)
assert i == log.value, "The item in position %d do not have the right value. (%d)" % (i, log.value)
def test_filter_owned(self):
"""
[recommendation.filter.OwnedIems] Test a filter owned items on recommendation
"""
rfilter = FilterOwned()
recommendation = [random.random() for _ in range(len(ITEMS))]
for u in USERS:
user = User.get_user_by_external_id(u["external_id"])
with self.assertNumQueries(0):
result = rfilter(user, np.array(recommendation[:]))
new_rec = [
aid + 1 for aid, _ in sorted(
enumerate(result), key=lambda x: x[1], reverse=True)
]
n = len(user.owned_items)
for item in user.owned_items:
assert item in new_rec[0-n:], "Item %d is not in the last in recommendation %s. User owned items %s" % \
(item, new_rec, list(user.owned_items.keys()))
def delete(self, request, user_external_id):
"""
removes an old item in the user installed apps. It should have a special POST parameter (besides the csrf token
or other token needed to the connection) that is item_to_acquire.
The ´´item_to_remove´´ is the item external id that is supposed to be removed in the user inventory.
:param request: The HTTP request.
:param user_external_id: The user external id that are making the request.
:return: A success response if the input was successful =p
"""
try:
item_id = request.DATA["item_to_remove"]
except KeyError:
return self.format_response(PARAMETERS_IN_MISS, status=FORMAT_ERROR)
self.remove_item(User.get_user_by_external_id(user_external_id), Item.get_item_by_external_id(item_id))
return self.format_response(SUCCESS_MESSAGE)
def post(self, request, user_external_id):
"""
Inserts a new item in the user owned items. It should have a special POST parameter (besides the csrf token or
other token needed to the connection - to development and presentation purposes the csrf was disabled) that is
item_to_acquire.
The ´´item_to_acquire´´ is the item external id that is supposed to be in the user inventory.
:param request: The HTTP request.
:param user_external_id: The user external id that are making the request.
:return: A success response if the input was successful =p
"""
try:
item_id = request.POST["item_to_acquire"]
except KeyError:
return self.format_response(PARAMETERS_IN_MISS, status=FORMAT_ERROR)
self.insert_acquisition(User.get_user_by_external_id(user_external_id), Item.get_item_by_external_id(item_id))
return self.format_response(SUCCESS_MESSAGE)
def delete(self, request, user_external_id):
"""
removes an old item in the user installed apps. It should have a special POST parameter (besides the csrf token
or other token needed to the connection) that is item_to_acquire.
The ´´item_to_remove´´ is the item external id that is supposed to be removed in the user inventory.
:param request: The HTTP request.
:param user_external_id: The user external id that are making the request.
:return: A success response if the input was successful =p
"""
try:
item_id = request.DATA["item_to_remove"]
except KeyError:
return self.format_response(PARAMETERS_IN_MISS,
status=FORMAT_ERROR)
self.remove_item(User.get_user_by_external_id(user_external_id),
Item.get_item_by_external_id(item_id))
return self.format_response(SUCCESS_MESSAGE)
def post(self, request, user_external_id):
"""
Inserts a new item in the user owned items. It should have a special POST parameter (besides the csrf token or
other token needed to the connection - to development and presentation purposes the csrf was disabled) that is
item_to_acquire.
The ´´item_to_acquire´´ is the item external id that is supposed to be in the user inventory.
:param request: The HTTP request.
:param user_external_id: The user external id that are making the request.
:return: A success response if the input was successful =p
"""
try:
item_id = request.POST["item_to_acquire"]
except KeyError:
return self.format_response(PARAMETERS_IN_MISS,
status=FORMAT_ERROR)
self.insert_acquisition(User.get_user_by_external_id(user_external_id),
Item.get_item_by_external_id(item_id))
return self.format_response(SUCCESS_MESSAGE)
def register():
if current_user.is_authenticated:
flash(f'You are already logged in ', 'info')
return redirect(url_for('main.account'))
form = RegistrationForm()
role = Role.query.filter_by(name="User").first()
arm = Arm.query.filter_by(name="Science").first()
if form.validate_on_submit():
user = User(username=form.username.data,
password=form.password.data,
email=form.email.data,
role=role,
age=19,
arm=arm)
db.session.add(user)
db.session.commit()
flash(f'Your account has been created! You are now able to loggin',
'success')
return redirect(url_for('users.login'))
return render_template('register.html', title='Register', form=form)
def put(self, request, user_external_id):
"""
Replaces all the user items with the list of given items.
It should have a PUT body parameter items_to_acquire (besides the csrf token or
other token needed to the connection - to development and presentation purposes the csrf was disabled) that is.
The ´´items_to_acquire´´ is the list of items external ids that will become a new user inventory.
:param request: The HTTP request.
:param user_external_id: The user external id that are making the request.
:return: A success response if the input was successful =p
"""
try:
items_ids = request.DATA["user_items"]
except KeyError:
return self.format_response(PARAMETERS_IN_MISS, status=FORMAT_ERROR)
try:
user = User.get_user_by_external_id(user_external_id)
except User.DoesNotExist:
user = User.objects.create(external_id=user_external_id)
items = [Item.get_item_by_external_id(iid) for iid in items_ids]
self.update_user_items(user, items)
return self.format_response(SUCCESS_MESSAGE)
def test_filter_regions(self):
"""
[recommendation.filter.Region] Test a region filter on recommendation
"""
rfilter = SimpleRegionFilter()
recommendation = [random.random() for _ in range(len(ITEMS))]
app_rec = [Item.get_item_by_id(aid+1).external_id
for aid, _ in sorted(enumerate(recommendation), key=lambda x: x[1], reverse=True)]
for u in USERS:
user = User.get_user_by_external_id(u["external_id"])
with self.assertNumQueries(0):
result = rfilter(user, np.array(recommendation[:]))
new_rec = [Item.get_item_by_id(aid+1).external_id
for aid, _ in sorted(enumerate(result), key=lambda x: x[1], reverse=True)]
n = len(u["last_apps_region"])
if n == 0:
assert new_rec == app_rec, \
"Recommendation is not the same when not apps to filter(%s != %s)" % (new_rec, app_rec)
else:
for item in u["last_apps_region"]:
assert item in new_rec[0-n:], \
"Item %s for user %s is not in the last in recommendation %s. Items filtered %s." % \
(item, user, new_rec, list(u["last_apps_region"]))
def test_recommendation_logging(self):
"""
[recommendation.decorator.SimpleLogger] Test if a recommendation is logged
"""
logger = LogEvent(LogEvent.RECOMMEND)
user = User.get_user_by_external_id("joaonrb")
recommendation = [
Item.get_item_by_external_id(i).pk
for i in ("10001", "10002", "10003", "10004", "98766")
]
recommendation = logger(lambda user: recommendation)(user=user)
time.sleep(1.)
logs = list(
LogEntry.objects.filter(user=user,
type=logger.RECOMMEND).order_by("value"))
assert len(logs) == 5, "Number of register is not correct %s" % logs
logs_iter = iter(logs)
for i, item in enumerate(recommendation, start=1):
log = logs_iter.next()
assert item == log.item.pk, "The item in position %d is not the same that in recommendation " \
"(%s != %s)" % (i, item, log.item.external_id)
assert i == log.value, "The item in position %d do not have the right value. (%d)" % (
i, log.value)
def put(self, request, user_external_id):
"""
Replaces all the user items with the list of given items.
It should have a PUT body parameter items_to_acquire (besides the csrf token or
other token needed to the connection - to development and presentation purposes the csrf was disabled) that is.
The ´´items_to_acquire´´ is the list of items external ids that will become a new user inventory.
:param request: The HTTP request.
:param user_external_id: The user external id that are making the request.
:return: A success response if the input was successful =p
"""
try:
items_ids = request.DATA["user_items"]
except KeyError:
return self.format_response(PARAMETERS_IN_MISS,
status=FORMAT_ERROR)
try:
user = User.get_user_by_external_id(user_external_id)
except User.DoesNotExist:
user = User.objects.create(external_id=user_external_id)
items = [Item.get_item_by_external_id(iid) for iid in items_ids]
self.update_user_items(user, items)
return self.format_response(SUCCESS_MESSAGE)
def test_check_std_recommendation(self):
"""
[recommendation.core.TensorCoFiController] Test get default recommendation
"""
rec_controller = get_controller()
for u in USERS:
recommendation = rec_controller.get_recommendation(user=User.get_user_by_external_id(u["external_id"]), n=5)
assert len(recommendation) == 5, "Size of recommendation is not right"
#@ut.skipIf("default" not in RECOMMENDATION_SETTINGS, "Default recommendation is not defined")
#def test_check_alternative_recommendation(self):
# """
# [recommendation.core.TensorCoFiController] Test get alternative recommendation
# """
# rec_controller = get_controller()
# pop_result = \
# [aid+1 for aid, _ in sorted(enumerate(Popularity.get_model().recommendation),
# key=lambda x: x[1], reverse=True)]
# for name in ["Gepeto", "Son_Goku", "Peter_Pan", "Tony_Montana", "Lady_Diana"]:
# user = User.objects.create(external_id=name)
# recommendation = rec_controller.get_recommendation(user=user, n=5)
# #assert len(recommendation) == 5, "Size of recommendation is not right"
# assert recommendation == pop_result[:len(recommendation)], \
# "Recommendation is not popularity for user %s (%s != %s)" % (user, recommendation, pop_result)
def test_reranker_diversity_no_redundancy(self):
"""
[recommendation.diversity.ReRanker] Test a diversity re-ranker on recommendation for non redundancy and loss
"""
diversity = SimpleDiversityReRanker()
recommendation = [
Item.get_item_by_external_id(i).pk
for i in ("10001", "10002", "10003", "10004", "98766")
]
shuffle(recommendation)
for u in USERS:
user = User.get_user_by_external_id(u["external_id"])
with self.assertNumQueries(0):
result = diversity(user=user,
recommendation=recommendation[:],
size=5)
new_rec = [
aid + 1 for aid, _ in sorted(
enumerate(result), key=lambda x: x[1], reverse=True)
]
assert len(set(result)) == len(
ITEMS), "Recommendation size changed (%d != %s)" % (
len(new_rec), len(ITEMS))
def fill_db_with_users(self, objects):
"""
Put users in Database
:return:
"""
size = 0
user_items = {}
json_users = []
regions = {}
langs = {}
with click.progressbar(objects, label="Loading users to memory") as bar:
for obj in bar:
if self.user_file_identifier_field in obj:
user_id = str(obj[self.user_field])
for item in obj.get(self.user_items_field, ()):
item_id = str(item[self.user_item_identifier_field])
user_item = user_id, self.user_item_dropped_field in item
try:
user_items[item_id].append(user_item)
except KeyError:
user_items[item_id] = [user_item]
size += 1
json_users.append(user_id)
if "region" in obj:
try:
regions[obj["region"]].append(user_id)
except KeyError:
regions[obj["region"]] = [user_id]
if "lang" in obj:
try:
langs[obj["lang"]].append(user_id)
except KeyError:
langs[obj["lang"]] = [user_id]
logging.debug("Done!")
users = {user.external_id: user for user in User.objects.filter(external_id__in=json_users)}
new_users = {}
for user_eid in json_users:
if user_eid not in users:
new_users[user_eid] = User(external_id=user_eid)
del json_users
#logging.debug("Users ready to be saved")
User.objects.bulk_create(new_users.values())
for user in User.objects.filter(external_id__in=new_users.keys()):
users[user.external_id] = user
assert len(users) == size, \
"Size of loaded objects and users in db is not the same (%d != %d)" % (len(users), size)
if "recommendation.language" in settings.INSTALLED_APPS:
self.fill_user_locale(users, regions, langs)
#logging.debug("%d new users saved with bulk_create" % len(new_users))
#logging.debug("Preparing items")
items = {ieid: iid for ieid, iid in Item.objects.filter(external_id__in=user_items).values_list("external_id",
"id")}
r = range(0, len(user_items), 300)
n = len(r)
user_items = user_items.items()
user_items = [user_items[i:i+300] for i in r]
with click.progressbar(user_items, length=n, label="Load items for user inventory") as bar:
for user_item in bar:
self.fill_inventory(users, items, user_item)
del user_item
performance improvement.
"""
import os
if "DJANGO_SETTINGS_MODULE" not in os.environ:
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "recommendation.default_settings")
from django.core.wsgi import get_wsgi_application
application = get_wsgi_application()
from django.conf import settings
from recommendation.models import Item, User, TensorCoFi, Popularity
# Load user and items
Item.load_to_cache()
User.load_to_cache()
# Load main models
Popularity.load_to_cache()
TensorCoFi.load_to_cache()
if "recommendation.language" in settings.INSTALLED_APPS:
from recommendation.language.models import Locale, Region
Locale.load_to_cache()
Region.load_to_cache()
#if "recommendation.simple_logging" in recommendation.settings.INSTALLED_APPS:
# print("Loading logs to cache...")
def get_user_locales(user_id):
return set([
pk[0] for pk in User.get_user_by_id(
user_id).locales.all().values_list("locale_id")
])
def get_user_locales(user_id):
return set([pk[0] for pk in User.get_user_by_id(user_id).locales.all().values_list("locale_id")])
"""
import os
if "DJANGO_SETTINGS_MODULE" not in os.environ:
os.environ.setdefault("DJANGO_SETTINGS_MODULE",
"recommendation.default_settings")
from django.core.wsgi import get_wsgi_application
application = get_wsgi_application()
from django.conf import settings
from recommendation.models import Item, User, TensorCoFi, Popularity
# Load user and items
Item.load_to_cache()
User.load_to_cache()
# Load main models
Popularity.load_to_cache()
TensorCoFi.load_to_cache()
if "recommendation.language" in settings.INSTALLED_APPS:
from recommendation.language.models import Locale, Region
Locale.load_to_cache()
Region.load_to_cache()
#if "recommendation.simple_logging" in recommendation.settings.INSTALLED_APPS:
# print("Loading logs to cache...")
# from recommendation.simple_logging.models import LogEntry
# LogEntry.load_to_cache()
