def _recommend_rating(self, user, recommender, params, include=None, exclude=None):
user = user.lower()
if user not in recommender.known_users:
raise NotFound(f"user <{user}> could not be found")
params = params or {}
include = (
frozenset(_parse_ints(params.get("include")))
if include is None
else include
)
# we should only need this if params are set, but see #90
games = include | frozenset(
self.filter_queryset(self.get_queryset())
.order_by()
.values_list("bgg_id", flat=True)
)
games &= recommender.rated_games
if not games:
return ()
exclude = self._excluded_games(user, params, include, exclude)
similarity_model = take_first(params.get("model")) == "similarity"
return recommender.recommend(
users=(user,),
games=games,
similarity_model=similarity_model,
exclude=_exclude(user, ids=exclude),
exclude_known=parse_bool(take_first(params.get("exclude_known"))),
exclude_clusters=parse_bool(take_first(params.get("exclude_clusters"))),
star_percentiles=getattr(settings, "STAR_PERCENTILES", None),
)
def recommend_bga(self, request):
"""recommend games with Board Game Atlas data"""
path = getattr(settings, "BGA_RECOMMENDER_PATH", None)
recommender = load_recommender(path, "bga")
if recommender is None:
return self.list(request)
users = list(_extract_params(request, "user", str))
like = list(_extract_params(request, "like", str))
recommendation = (
recommender.recommend_similar(games=like)
if like and not users
else self._recommend_group_rating_bga(
users, recommender, dict(request.query_params)
)
if len(users) > 1
else recommender.recommend(
users=(take_first(users),),
similarity_model=request.query_params.get("model") == "similarity",
star_percentiles=getattr(settings, "STAR_PERCENTILES", None),
)
)
del path, recommender, users, like
page = self.paginate_queryset(recommendation)
return (
self.get_paginated_response(page)
if page is not None
else Response(list(recommendation[:10]))
)
def _recommend_group_rating_bga(self, users, recommender, params):
import turicreate as tc
users = [user for user in users if user in recommender.known_users]
if not users:
raise NotFound("none of the users could be found")
similarity_model = take_first(params.get("model")) == "similarity"
recommendations = (
recommender.recommend(
users=users,
games=recommender.rated_games,
similarity_model=similarity_model,
exclude_known=False,
)
.groupby(
key_column_names="bga_id",
operations={"score": tc.aggregate.MEAN("score")},
)
.sort("score", ascending=False)
)
recommendations["rank"] = range(1, len(recommendations) + 1)
return recommendations
def extract_bga_id(url: Union[str, ParseResult, None]) -> Optional[str]:
""" extract Board Game Atlas ID from URL """
url = parse_url(url, ("boardgameatlas.com", "www.boardgameatlas.com"))
if not url:
return None
match = REGEX_BGA_ID.match(url.path)
if match:
return match.group(1)
ids_str = extract_query_param(url, "ids")
ids = ids_str.split(",") if ids_str else ()
return take_first(map(normalize_space, ids)) or extract_query_param(
url, "game-id")
def _excluded_games(self, user, params, include=None, exclude=None):
params = params or {}
params.setdefault("exclude_known", True)
exclude = frozenset(arg_to_iter(exclude)) | frozenset(
_parse_ints(params.get("exclude"))
)
exclude_known = parse_bool(take_first(params.get("exclude_known")))
exclude_fields = [
field
for field in self.collection_fields
if parse_bool(take_first(params.get(f"exclude_{field}")))
]
exclude_wishlist = parse_int(take_first(params.get("exclude_wishlist")))
exclude_play_count = parse_int(take_first(params.get("exclude_play_count")))
exclude_clusters = parse_bool(take_first(params.get("exclude_clusters")))
try:
queries = [Q(**{field: True}) for field in exclude_fields]
if exclude_known and exclude_clusters:
queries.append(Q(rating__isnull=False))
if exclude_wishlist:
queries.append(Q(wishlist__lte=exclude_wishlist))
if exclude_play_count:
queries.append(Q(play_count__gte=exclude_play_count))
if queries:
query = reduce(or_, queries)
exclude |= frozenset(
User.objects.get(name=user)
.collection_set.order_by()
.filter(query)
.values_list("game_id", flat=True)
)
except Exception:
pass
return tuple(exclude) if not include else tuple(exclude - include)
def _extract_labels(self, response, value):
json_obj = parse_json(response.text) if hasattr(response, "text") else {}
labels = take_first(jmespath.search(f"entities.{value}.labels", json_obj)) or {}
labels = labels.values()
labels = sorted(
labels,
key=lambda label: self.lang_priorities.get(label.get("language"), math.inf),
)
labels = clear_list(label.get("value") for label in labels)
self.labels[value] = labels
self.logger.debug("resolved labels for %s: %s", value, labels)
return labels
def _recommend_group_rating(self, users, recommender, params):
import turicreate as tc
users = (user.lower() for user in users if user)
users = [user for user in users if user in recommender.known_users]
if not users:
raise NotFound("none of the users could be found")
games = (
frozenset(
self.filter_queryset(self.get_queryset())
.order_by()
.values_list("bgg_id", flat=True)
)
& recommender.rated_games
)
if not games:
return ()
similarity_model = take_first(params.get("model")) == "similarity"
recommendations = (
recommender.recommend(
users=users,
games=games,
similarity_model=similarity_model,
# TODO we want to exclude games based on the group's collections, see #228
# exclude=(),
exclude_known=False,
)
.groupby(
key_column_names="bgg_id",
operations={"score": tc.aggregate.MEAN("score")},
)
.sort("score", ascending=False)
)
recommendations["rank"] = range(1, len(recommendations) + 1)
return recommendations
def _create_references(
model,
items,
foreign=None,
recursive=None,
batch_size=None,
dry_run=False,
):
foreign = foreign or {}
foreign = {k: tuple(arg_to_iter(v)) for k, v in foreign.items()}
foreign = {k: v for k, v in foreign.items() if len(v) == 2}
recursive = ({r: r
for r in arg_to_iter(recursive)}
if not isinstance(recursive, dict) else recursive)
if not foreign and not recursive:
LOGGER.warning(
"neither foreign nor recursive references given, got nothing to do..."
)
return
LOGGER.info("creating foreign references: %r", foreign)
LOGGER.info("creating recursive references: %r", recursive)
count = -1
foreign_values = {f[0]: defaultdict(set) for f in foreign.values()}
updates = {}
for count, item in enumerate(items):
update = defaultdict(list)
for field, (fmodel, _) in foreign.items():
for value in filter(
None, map(_parse_value_id, arg_to_iter(item.get(field)))):
id_ = value.get("id")
value = value.get("value")
if id_ and value:
foreign_values[fmodel][id_].add(value)
update[field].append(id_)
for rec_from, rec_to in recursive.items():
rec = {parse_int(r) for r in arg_to_iter(item.get(rec_from)) if r}
rec = (sorted(
model.objects.filter(pk__in=rec).values_list(
"pk", flat=True).distinct()) if rec else None)
if rec:
update[rec_to] = rec
pkey = parse_int(item.get(model._meta.pk.name))
if pkey and any(update.values()):
updates[pkey] = update
if (count + 1) % 1000 == 0:
LOGGER.info("processed %d items so far", count + 1)
del items, recursive
LOGGER.info("processed %d items in total", count + 1)
for fmodel, value_field in frozenset(foreign.values()):
id_field = fmodel._meta.pk.name
LOGGER.info("found %d items for model %r to create",
len(foreign_values[fmodel]), fmodel)
values = ({
id_field: k,
value_field: take_first(v)
} for k, v in foreign_values[fmodel].items() if k and v)
_create_from_items(
model=fmodel,
items=values,
batch_size=batch_size,
dry_run=dry_run,
)
del foreign, foreign_values
LOGGER.info("found %d items for model %r to update", len(updates), model)
batches = (batchify(updates.items(), batch_size) if batch_size else
(updates.items(), ))
for count, batch in enumerate(batches):
LOGGER.info("processing batch #%d...", count + 1)
if not dry_run:
with atomic():
for pkey, update in batch:
try:
instance = model.objects.get(pk=pkey)
for field, values in update.items():
getattr(instance, field).set(values)
instance.save()
except Exception:
LOGGER.exception(
"an error ocurred when updating <%s> with %r",
pkey,
update,
)
del batches, updates
LOGGER.info("done updating")