def test_data_in_index(self):
"""Verify the data we are indexing."""
p = product(save=True)
q = question(locale='pt-BR', product=p, save=True)
a = answer(question=q, save=True)
self.refresh()
eq_(AnswerMetricsMappingType.search().count(), 1)
data = AnswerMetricsMappingType.search()[0]
eq_(data['locale'], q.locale)
eq_(data['product'], [p.slug])
eq_(data['creator_id'], a.creator_id)
eq_(data['is_solution'], False)
eq_(data['by_asker'], False)
# Mark as solution and verify
q.solution = a
q.save()
self.refresh()
data = AnswerMetricsMappingType.search()[0]
eq_(data['is_solution'], True)
# Make the answer creator to be the question creator and verify.
a.creator = q.creator
a.save()
self.refresh()
data = AnswerMetricsMappingType.search()[0]
eq_(data['by_asker'], True)
def test_data_in_index(self):
"""Verify the data we are indexing."""
p = ProductFactory()
q = QuestionFactory(locale='pt-BR', product=p)
a = AnswerFactory(question=q)
self.refresh()
eq_(AnswerMetricsMappingType.search().count(), 1)
data = AnswerMetricsMappingType.search()[0]
eq_(data['locale'], q.locale)
eq_(data['product'], [p.slug])
eq_(data['creator_id'], a.creator_id)
eq_(data['is_solution'], False)
eq_(data['by_asker'], False)
# Mark as solution and verify
q.solution = a
q.save()
self.refresh()
data = AnswerMetricsMappingType.search()[0]
eq_(data['is_solution'], True)
# Make the answer creator to be the question creator and verify.
a.creator = q.creator
a.save()
self.refresh()
data = AnswerMetricsMappingType.search()[0]
eq_(data['by_asker'], True)
def test_add_and_delete(self):
"""Adding an answer should add it to the index.
Deleting should delete it.
"""
a = AnswerFactory()
self.refresh()
eq_(AnswerMetricsMappingType.search().count(), 1)
a.delete()
self.refresh()
eq_(AnswerMetricsMappingType.search().count(), 0)
def test_add_and_delete(self):
"""Adding an answer should add it to the index.
Deleting should delete it.
"""
a = answer(save=True)
self.refresh()
eq_(AnswerMetricsMappingType.search().count(), 1)
a.delete()
self.refresh()
eq_(AnswerMetricsMappingType.search().count(), 0)
def top_contributors_questions(start=None, end=None, locale=None, product=None,
count=10, page=1, use_cache=True):
"""Get the top Support Forum contributors."""
# Get the user ids and contribution count of the top contributors.
if use_cache:
cache_key = u'{}_{}_{}_{}_{}_{}'.format(start, end, locale, product, count, page)
cache_key = hashlib.sha1(cache_key.encode('utf-8')).hexdigest()
cache_key = 'top_contributors_questions_{}'.format(cache_key)
cached = cache.get(cache_key, None)
if cached:
return cached
query = AnswerMetricsMappingType.search()
# Adding answer to your own question, isn't a contribution.
query = query.filter(by_asker=False)
query = _apply_filters(query, start, end, locale, product)
answers = [q.id for q in query.all()[:HUGE_NUMBER]]
users = (User.objects
.filter(answers__in=answers)
.annotate(query_count=Count('answers'))
.order_by('-query_count'))
counts = _get_creator_counts(users, count, page)
if use_cache:
cache.set(cache_key, counts, 60*15) # 15 minutes
return counts
def top_contributors_questions(start=None,
end=None,
locale=None,
product=None,
count=10,
page=1,
use_cache=True):
"""Get the top Support Forum contributors."""
# Get the user ids and contribution count of the top contributors.
if use_cache:
cache_key = u'{}_{}_{}_{}_{}_{}'.format(start, end, locale, product,
count, page)
cache_key = hashlib.sha1(cache_key.encode('utf-8')).hexdigest()
cache_key = 'top_contributors_questions_{}'.format(cache_key)
cached = cache.get(cache_key, None)
if cached:
return cached
query = AnswerMetricsMappingType.search()
# Adding answer to your own question, isn't a contribution.
query = query.filter(by_asker=False)
query = _apply_filters(query, start, end, locale, product)
answers = [q.id for q in query.all()[:HUGE_NUMBER]]
users = (User.objects.filter(answers__in=answers).annotate(
query_count=Count('answers')).order_by('-query_count'))
counts = _get_creator_counts(users, count, page)
if use_cache:
cache.set(cache_key, counts, 60 * 15) # 15 minutes
return counts
def top_contributors_questions(start=None, end=None, locale=None, product=None, count=10, page=1):
"""Get the top Support Forum contributors."""
# Get the user ids and contribution count of the top contributors.
query = AnswerMetricsMappingType.search().facet("creator_id", filtered=True, size=BIG_NUMBER)
# Adding answer to your own question, isn't a contribution.
query = query.filter(by_asker=False)
query = _apply_filters(query, start, end, locale, product)
return _get_creator_counts(query, count, page)
def top_contributors_questions(start=None,
end=None,
locale=None,
product=None,
count=10):
"""Get the top Support Forum contributors."""
# Get the user ids and contribution count of the top contributors.
query = (AnswerMetricsMappingType.search().facet('creator_id',
filtered=True,
size=count))
query = _apply_filters(query, start, end, locale, product)
return _get_creator_counts(query, count)
def test_data_in_index(self):
"""Verify the data we are indexing."""
p = product(save=True)
q = question(locale='pt-BR', save=True)
q.products.add(p)
a = answer(question=q, save=True)
self.refresh()
eq_(AnswerMetricsMappingType.search().count(), 1)
data = AnswerMetricsMappingType.search().values_dict()[0]
eq_(data['locale'], q.locale)
eq_(data['product'], [p.slug])
eq_(data['creator_id'], a.creator_id)
eq_(data['is_solution'], False)
# Mark as solution and verify
q.solution = a
q.save()
self.refresh()
data = AnswerMetricsMappingType.search().values_dict()[0]
eq_(data['is_solution'], True)
def top_contributors_questions(start=None,
end=None,
locale=None,
product=None,
count=10,
page=1):
"""Get the top Support Forum contributors."""
# Get the user ids and contribution count of the top contributors.
query = (AnswerMetricsMappingType.search().facet('creator_id',
filtered=True,
size=BIG_NUMBER))
# Adding answer to your own question, isn't a contribution.
query = query.filter(by_asker=False)
query = _apply_filters(query, start, end, locale, product)
return _get_creator_counts(query, count, page)
def get_data(self, request):
super(TopContributorsQuestions, self).get_data(request)
# This is the base of all the metrics. Each metric branches off from
# this to get a particular metric type, since we can't do Aggregates.
query = AnswerMetricsMappingType.search()
base_filters = self.get_filters()
# This branch is to get the total number of answers for each user.
answer_query = (
query
.filter(base_filters)
.facet('creator_id', filtered=True, size=BIG_NUMBER))
# This branch gets the number of answers that are solutions for each user.
solutions_filter = base_filters & F(is_solution=True)
solutions_query = (
query
.filter(solutions_filter)
.facet('creator_id', filtered=True, size=BIG_NUMBER))
# This branch gets the number of helpful votes across all answers for
# each user. It is a raw facet because elasticutils only supports the
# term facet type in non-raw facets. Because it is raw facet, we have
# to also put the filter in the facet ourselves.
helpful_query = (
query
.facet_raw(
creator_id={
'terms_stats': {
'key_field': 'creator_id',
'value_field': 'helpful_count',
},
'facet_filter': query._process_filters(base_filters.filters),
}))
# Collect three lists of objects that correlates users and the appropriate metric count
creator_answer_counts = answer_query.facet_counts()['creator_id']['terms']
creator_solutions_counts = solutions_query.facet_counts()['creator_id']['terms']
creator_helpful_counts = helpful_query.facet_counts()['creator_id']['terms']
# Combine all the metric types into one big list.
combined = defaultdict(lambda: {
'answer_count': 0,
'solution_count': 0,
'helpful_vote_count': 0,
})
for d in creator_answer_counts:
combined[d['term']]['user_id'] = d['term']
combined[d['term']]['answer_count'] = d['count']
for d in creator_solutions_counts:
combined[d['term']]['user_id'] = d['term']
combined[d['term']]['solution_count'] = d['count']
for d in creator_helpful_counts:
combined[d['term']]['user_id'] = d['term']
# Since this is a term_stats filter, not just a term filter, it is total, not count.
combined[d['term']]['helpful_vote_count'] = int(d['total'])
# Sort by answer count, and get just the ids into a list.
sort_key = self.query_values['ordering']
if sort_key[0] == '-':
sort_reverse = True
sort_key = sort_key[1:]
else:
sort_reverse = False
top_contributors = combined.values()
top_contributors.sort(key=lambda d: d[sort_key], reverse=sort_reverse)
user_ids = [c['user_id'] for c in top_contributors]
full_count = len(user_ids)
# Paginate those user ids.
page_start = (self.query_values['page'] - 1) * self.query_values['page_size']
page_end = page_start + self.query_values['page_size']
user_ids = user_ids[page_start:page_end]
# Get full user objects for every id on this page.
users = UserMappingType.reshape(
UserMappingType
.search()
.filter(id__in=user_ids)
.values_dict('id', 'username', 'display_name', 'avatar', 'last_contribution_date')
[:self.query_values['page_size']])
# For ever user object found, mix in the metrics counts for that user,
# and then reshape the data to make more sense to clients.
data = []
for u in users:
d = combined[u['id']]
d['user'] = u
d['last_contribution_date'] = d['user'].get('last_contribution_date', None)
d.pop('user_id', None)
d['user'].pop('id', None)
d['user'].pop('last_contribution_date', None)
data.append(d)
# One last sort, since ES didn't return the users in any particular order.
data.sort(key=lambda d: d[sort_key], reverse=sort_reverse)
# Add ranks to the objects.
for i, contributor in enumerate(data, 1):
contributor['rank'] = page_start + i
return {
'results': data,
'count': full_count,
'filters': self.query_values,
'allowed_orderings': self.get_allowed_orderings(),
'warnings': self.warnings,
}
def get_data(self, request):
super(TopContributorsQuestions, self).get_data(request)
# This is the base of all the metrics. Each metric branches off from
# this to get a particular metric type, since we can't do Aggregates.
query = AnswerMetricsMappingType.search()
base_filters = self.get_filters()
# This branch is to get the total number of answers for each user.
answer_query = (
query
.filter(base_filters)
.facet('creator_id', filtered=True, size=BIG_NUMBER))
# This branch gets the number of answers that are solutions for each user.
solutions_filter = base_filters & F(is_solution=True)
solutions_query = (
query
.filter(solutions_filter)
.facet('creator_id', filtered=True, size=BIG_NUMBER))
# This branch gets the number of helpful votes across all answers for
# each user. It is a raw facet because elasticutils only supports the
# term facet type in non-raw facets. Because it is raw facet, we have
# to also put the filter in the facet ourselves.
helpful_query = (
query
.facet_raw(
creator_id={
'terms_stats': {
'key_field': 'creator_id',
'value_field': 'helpful_count',
},
'facet_filter': query._process_filters(base_filters.filters),
}))
# Collect three lists of objects that correlates users and the appropriate metric count
creator_answer_counts = answer_query.facet_counts()['creator_id']['terms']
creator_solutions_counts = solutions_query.facet_counts()['creator_id']['terms']
creator_helpful_counts = helpful_query.facet_counts()['creator_id']['terms']
# Combine all the metric types into one big list.
combined = defaultdict(lambda: {
'answer_count': 0,
'solution_count': 0,
'helpful_vote_count': 0,
})
for d in creator_answer_counts:
combined[d['term']]['user_id'] = d['term']
combined[d['term']]['answer_count'] = d['count']
for d in creator_solutions_counts:
combined[d['term']]['user_id'] = d['term']
combined[d['term']]['solution_count'] = d['count']
for d in creator_helpful_counts:
combined[d['term']]['user_id'] = d['term']
# Since this is a term_stats filter, not just a term filter, it is total, not count.
combined[d['term']]['helpful_vote_count'] = int(d['total'])
# Sort by answer count, and get just the ids into a list.
sort_key = self.query_values['ordering']
if sort_key[0] == '-':
sort_reverse = True
sort_key = sort_key[1:]
else:
sort_reverse = False
top_contributors = combined.values()
top_contributors.sort(key=lambda d: d[sort_key], reverse=sort_reverse)
user_ids = [c['user_id'] for c in top_contributors]
full_count = len(user_ids)
# Paginate those user ids.
page_start = (self.query_values['page'] - 1) * self.query_values['page_size']
page_end = page_start + self.query_values['page_size']
user_ids = user_ids[page_start:page_end]
# Get full user objects for every id on this page.
users = UserMappingType.reshape(
UserMappingType
.search()
.filter(id__in=user_ids)
.values_dict('id', 'username', 'display_name', 'avatar', 'last_contribution_date')
[:self.query_values['page_size']])
# For ever user object found, mix in the metrics counts for that user,
# and then reshape the data to make more sense to clients.
data = []
for u in users:
d = combined[u['id']]
d['user'] = u
d['last_contribution_date'] = d['user'].get('last_contribution_date', None)
d.pop('user_id', None)
d['user'].pop('id', None)
d['user'].pop('last_contribution_date', None)
data.append(d)
# One last sort, since ES didn't return the users in any particular order.
data.sort(key=lambda d: d[sort_key], reverse=sort_reverse)
# Add ranks to the objects.
for i, contributor in enumerate(data, 1):
contributor['rank'] = page_start + i
return {
'results': data,
'count': full_count,
'filters': self.query_values,
'allowed_orderings': self.get_allowed_orderings(),
'warnings': self.warnings,
}
def get_data(self, request):
super(TopContributorsQuestions, self).get_data(request)
# This is the base of all the metrics. Each metric branches off from
# this to get a particular metric type, since we can't do Aggregates.
query = AnswerMetricsMappingType.search()
base_filters = self.get_filters()
# This branch is to get the total number of answers for each user.
answer_query = query.filter(base_filters).facet("creator_id",
filtered=True,
size=BIG_NUMBER)
# This branch gets the number of answers that are solutions for each user.
solutions_filter = base_filters & F(is_solution=True)
solutions_query = query.filter(solutions_filter).facet("creator_id",
filtered=True,
size=BIG_NUMBER)
# This branch gets the number of helpful votes across all answers for
# each user. It is a raw facet because elasticutils only supports the
# term facet type in non-raw facets. Because it is raw facet, we have
# to also put the filter in the facet ourselves.
helpful_query = query.facet_raw(
creator_id={
"terms_stats": {
"key_field": "creator_id",
"value_field": "helpful_count",
},
"facet_filter": query._process_filters(base_filters.filters),
})
# Collect three lists of objects that correlates users and the appropriate metric count
creator_answer_counts = answer_query.facet_counts(
)["creator_id"]["terms"]
creator_solutions_counts = solutions_query.facet_counts(
)["creator_id"]["terms"]
creator_helpful_counts = helpful_query.facet_counts(
)["creator_id"]["terms"]
# Combine all the metric types into one big list.
combined = defaultdict(lambda: {
"answer_count": 0,
"solution_count": 0,
"helpful_vote_count": 0,
})
for d in creator_answer_counts:
combined[d["term"]]["user_id"] = d["term"]
combined[d["term"]]["answer_count"] = d["count"]
for d in creator_solutions_counts:
combined[d["term"]]["user_id"] = d["term"]
combined[d["term"]]["solution_count"] = d["count"]
for d in creator_helpful_counts:
combined[d["term"]]["user_id"] = d["term"]
# Since this is a term_stats filter, not just a term filter, it is total, not count.
combined[d["term"]]["helpful_vote_count"] = int(d["total"])
# Sort by answer count, and get just the ids into a list.
sort_key = self.query_values["ordering"]
if sort_key[0] == "-":
sort_reverse = True
sort_key = sort_key[1:]
else:
sort_reverse = False
top_contributors = list(combined.values())
top_contributors.sort(key=lambda d: d[sort_key], reverse=sort_reverse)
user_ids = [c["user_id"] for c in top_contributors]
full_count = len(user_ids)
# Paginate those user ids.
page_start = (self.query_values["page"] -
1) * self.query_values["page_size"]
page_end = page_start + self.query_values["page_size"]
user_ids = user_ids[page_start:page_end]
# Get full user objects for every id on this page.
users = UserMappingType.reshape(
UserMappingType.search().filter(id__in=user_ids).values_dict(
"id", "username", "display_name", "avatar",
"last_contribution_date")[:self.query_values["page_size"]])
# For ever user object found, mix in the metrics counts for that user,
# and then reshape the data to make more sense to clients.
data = []
for u in users:
d = combined[u["id"]]
d["user"] = u
d["last_contribution_date"] = d["user"].get(
"last_contribution_date", None)
d.pop("user_id", None)
d["user"].pop("id", None)
d["user"].pop("last_contribution_date", None)
data.append(d)
# One last sort, since ES didn't return the users in any particular order.
data.sort(key=lambda d: d[sort_key], reverse=sort_reverse)
# Add ranks to the objects.
for i, contributor in enumerate(data, 1):
contributor["rank"] = page_start + i
return {
"results": data,
"count": full_count,
"filters": self.query_values,
"allowed_orderings": self.get_allowed_orderings(),
"warnings": self.warnings,
}