def user_stats_bulk(request):
"""
Get statistics for selected users and concepts
since:
time as timestamp - get stats changed since
users:
list of identifiers of users
concepts (Optional):
list of identifiers of concepts
language:
language of concepts
"""
language = get_language(request)
users = load_query_json(request.GET, "users")
since = None
if 'since' in request.GET:
since = datetime.datetime.fromtimestamp(int(request.GET['since']))
concepts = None
if "concepts" in request.GET:
concepts = Concept.objects.filter(lang=language, active=True,
identifier__in=load_query_json(request.GET, "concepts"))
stats = UserStat.objects.get_user_stats(users, language, concepts=concepts, since=since)
data = {"users": []}
for user, s in stats.items():
data["users"].append({
"user_id": user,
"concepts": s,
})
return render_json(request, data, template='concepts_json.html', help_text=user_stats_bulk.__doc__)
def user_stats_bulk(request):
"""
Get statistics for selected users and concepts
since:
time as timestamp - get stats changed since
users:
list of identifiers of users
concepts (Optional):
list of identifiers of concepts
language:
language of concepts
"""
language = get_language(request)
users = load_query_json(request.GET, "users")
if request.user.is_staff:
if not hasattr(request.user, 'userprofile') or User.objects.filter(
pk__in=users, userprofile__classes__owner=request.user.
userprofile).count() < len(users):
return render_json(
request, {
'error':
_('Some requested users are not in owned classes'),
'error_type': 'permission_denied'
},
template='concepts_json.html',
status=401)
since = None
if 'since' in request.GET:
since = datetime.datetime.fromtimestamp(int(request.GET['since']))
concepts = None
if "concepts" in request.GET:
concepts = Concept.objects.filter(lang=language,
active=True,
identifier__in=load_query_json(
request.GET, "concepts"))
stats = UserStat.objects.get_user_stats(users,
language,
concepts=concepts,
since=since)
data = {"users": []}
for user, s in stats.items():
data["users"].append({
"user_id": user,
"concepts": s,
})
return render_json(request,
data,
template='concepts_json.html',
help_text=user_stats_bulk.__doc__)
def user_stats(request):
"""
JSON of user stats of the user
GET parameters:
html (bool):
turn on the HTML version of the API, defaults to false
user (int):
identifier of the user, defaults to logged user
concepts (list):
list of identifiers of concepts, defaults to all concepts
lang (str):
language of requested concepts, defaults to language from django
"""
user = get_user_id(request)
language = get_language(request)
concepts = None # meaning all concept
if "concepts" in request.GET:
concepts = Concept.objects.filter(lang=language,
active=True,
identifier__in=load_query_json(
request.GET, "concepts"))
data = UserStat.objects.get_user_stats(user, language, concepts)
return render_json(request,
data,
template='concepts_json.html',
help_text=user_stats.__doc__)
def user_stats_bulk(request):
"""
Get statistics for selected users and concepts
since:
time as timestamp - get stats changed since
users:
list of identifiers of users
concepts (Optional):
list of identifiers of concepts
language:
language of concepts
"""
language = get_language(request)
users = load_query_json(request.GET, "users")
since = None
if 'since' in request.GET:
since = datetime.datetime.fromtimestamp(int(request.GET['since']))
concepts = None
if "concepts" in request.GET:
concepts = Concept.objects.filter(lang=language,
active=True,
identifier__in=load_query_json(
request.GET, "concepts"))
stats = UserStat.objects.get_user_stats(users,
language,
concepts=concepts,
since=since)
data = {"users": []}
for user, s in stats.items():
data["users"].append({
"user_id": user,
"concepts": s,
})
return render_json(request,
data,
template='concepts_json.html',
help_text=user_stats_bulk.__doc__)
def user_stats_bulk(request):
"""
Get statistics for selected users and concepts
since:
time as timestamp - get stats changed since
users:
list of identifiers of users
concepts (Optional):
list of identifiers of concepts
language:
language of concepts
"""
language = get_language(request)
users = load_query_json(request.GET, "users")
if request.user.is_staff:
if not hasattr(request.user, 'userprofile') or User.objects.filter(pk__in=users,
userprofile__classes__owner=request.user.userprofile).count() < len(users):
return render_json(request, {
'error': _('Some requested users are not in owned classes'),
'error_type': 'permission_denied'
}, template='concepts_json.html', status=401)
since = None
if 'since' in request.GET:
since = datetime.datetime.fromtimestamp(int(request.GET['since']))
concepts = None
if "concepts" in request.GET:
concepts = Concept.objects.filter(lang=language, active=True,
identifier__in=load_query_json(request.GET, "concepts"))
stats = UserStat.objects.get_user_stats(users, language, concepts=concepts, since=since)
data = {"users": []}
for user, s in stats.items():
data["users"].append({
"user_id": user,
"concepts": s,
})
return render_json(request, data, template='concepts_json.html', help_text=user_stats_bulk.__doc__)
def to_practice_counts(request):
"""
Get number of items available to practice.
filters: -- use this or body
json as in BODY
language:
language of the items
BODY
json in following format:
{
"#identifier": [] -- custom identifier (str) and filter
...
}
"""
data = None
if request.method == "POST":
data = json.loads(request.body.decode("utf-8"))["filters"]
if "filters" in request.GET:
data = load_query_json(request.GET, "filters")
if data is None or len(data) == 0:
return render_json(request, {},
template='models_json.html',
help_text=to_practice_counts.__doc__)
language = get_language(request)
timer('to_practice_counts')
filter_names, filter_filters = list(zip(*sorted(data.items())))
reachable_leaves = Item.objects.filter_all_reachable_leaves_many(
filter_filters, language)
response = {
group_id: {
'filter': data[group_id],
'number_of_items': len(items),
}
for group_id, items in zip(filter_names, reachable_leaves)
}
LOGGER.debug(
"to_practice_counts - getting items in groups took %s seconds",
(timer('to_practice_counts')))
return render_json(request,
response,
template='models_json.html',
help_text=to_practice_counts.__doc__)
def user_stats(request):
"""
JSON of user stats of the user
GET parameters:
html (bool):
turn on the HTML version of the API, defaults to false
user (int):
identifier of the user, defaults to logged user
concepts (list):
list of identifiers of concepts, defaults to all concepts
lang (str):
language of requested concepts, defaults to language from django
"""
user = get_user_id(request)
language = get_language(request)
concepts = None # meaning all concept
if "concepts" in request.GET:
concepts = Concept.objects.filter(lang=language, active=True,
identifier__in=load_query_json(request.GET, "concepts"))
data = UserStat.objects.get_user_stats(user, language, concepts)
return render_json(request, data, template='concepts_json.html', help_text=user_stats.__doc__)
def to_practice_counts(request):
"""
Get number of items available to practice.
filters: -- use this or body
json as in BODY
language:
language of the items
BODY
json in following format:
{
"#identifier": [] -- custom identifier (str) and filter
...
}
"""
data = None
if request.method == "POST":
data = json.loads(request.body.decode("utf-8"))["filters"]
if "filters" in request.GET:
data = load_query_json(request.GET, "filters")
if data is None:
return render_json(request, {}, template='models_json.html', help_text=to_practice_counts.__doc__)
language = get_language(request)
timer('to_practice_counts')
filter_names, filter_filters = list(zip(*sorted(data.items())))
reachable_leaves = Item.objects.filter_all_reachable_leaves_many(filter_filters, language)
response = {
group_id: {
'filter': data[group_id],
'number_of_items': len(items),
}
for group_id, items in zip(filter_names, reachable_leaves)
}
LOGGER.debug("flashcard_counts - getting flashcards in groups took %s seconds", (timer('to_practice_counts')))
return render_json(request, response, template='models_json.html', help_text=to_practice_counts.__doc__)
def practice(request):
"""
Return the given number of questions to practice adaptively. In case of
POST request, try to save the answer(s).
GET parameters:
filter:
list of lists of identifiers (may be prefixed by minus sign to
mark complement)
language:
language (str) of items
avoid:
list of item ids to avoid
limit:
number of returned questions (default 10, maximum 100)
time:
time in format '%Y-%m-%d_%H:%M:%S' used for practicing
user:
identifier for the practicing user (only for stuff users)
stats:
turn on the enrichment of the objects by some statistics
html:
turn on the HTML version of the API
BODY:
see answer resource
"""
if request.user.id is None: # Google Bot
return render_json(
request, {
'error': _('There is no user available for the practice.'),
'error_type': 'user_undefined'
},
status=400,
template='models_json.html')
limit = min(int(request.GET.get('limit', 10)), 100)
# prepare
user = get_user_id(request)
time = get_time(request)
avoid = load_query_json(request.GET, "avoid", "[]")
practice_filter = get_filter(request)
practice_context = PracticeContext.objects.from_content(practice_filter)
environment = get_environment()
item_selector = get_item_selector()
if is_time_overridden(request):
environment.shift_time(time)
# save answers
if request.method == 'POST':
_save_answers(request, practice_context, False)
elif request.method == 'GET':
PracticeSet.objects.filter(answer__user_id=request.user.id).update(
finished=True)
if limit > 0:
item_ids = Item.objects.filter_all_reachable_leaves(
practice_filter, get_language(request))
item_ids = list(set(item_ids) - set(avoid))
limit_size = get_config('proso_models',
'practice.limit_item_set_size_to_select_from',
default=None)
if limit_size is not None and limit_size < len(item_ids):
item_ids = sample(item_ids, limit_size)
if len(item_ids) == 0:
return render_json(request, {
'error':
_('There is no item for the given filter to practice.'),
'error_type':
'empty_practice'
},
status=404,
template='models_json.html')
selected_items, meta = item_selector.select(environment,
user,
item_ids,
time,
practice_context.id,
limit,
items_in_queue=len(avoid))
result = []
for item, item_meta in zip(selected_items, meta):
question = {
'object_type': 'question',
'payload': Item.objects.item_id_to_json(item),
}
if item_meta is not None:
question['meta'] = item_meta
result.append(question)
else:
result = []
return render_json(request,
result,
template='models_json.html',
help_text=practice.__doc__)
def user_stats(request):
"""
Get user statistics for selected groups of items
time:
time in format '%Y-%m-%d_%H:%M:%S' used for practicing
user:
identifier of the user (only for stuff users)
username:
username of user (only for users with public profile)
filters: -- use this or body
json as in BODY
mastered:
use model to compute number of mastered items - can be slowed
language:
language of the items
BODY
json in following format:
{
"#identifier": [] -- custom identifier (str) and filter
...
}
"""
timer('user_stats')
response = {}
data = None
if request.method == "POST":
data = json.loads(request.body.decode("utf-8"))["filters"]
if "filters" in request.GET:
data = load_query_json(request.GET, "filters")
if data is None:
return render_json(request, {},
template='models_user_stats.html',
help_text=user_stats.__doc__)
environment = get_environment()
if is_time_overridden(request):
environment.shift_time(get_time(request))
user_id = get_user_id(request)
language = get_language(request)
filter_names, filter_filters = list(zip(*sorted(data.items())))
reachable_leaves = Item.objects.filter_all_reachable_leaves_many(
filter_filters, language)
all_leaves = sorted(list(set(flatten(reachable_leaves))))
answers = environment.number_of_answers_more_items(all_leaves, user_id)
correct_answers = environment.number_of_correct_answers_more_items(
all_leaves, user_id)
if request.GET.get("mastered"):
timer('user_stats_mastered')
mastery_threshold = get_mastery_trashold()
predictions = Item.objects.predict_for_overview(
environment, user_id, all_leaves)
mastered = dict(
list(zip(all_leaves,
[p >= mastery_threshold for p in predictions])))
LOGGER.debug(
"user_stats - getting predictions for items took %s seconds",
(timer('user_stats_mastered')))
for identifier, items in zip(filter_names, reachable_leaves):
if len(items) == 0:
response[identifier] = {
"filter": data[identifier],
"number_of_items": 0,
}
else:
response[identifier] = {
"filter":
data[identifier],
"number_of_items":
len(items),
"number_of_practiced_items":
sum(answers[i] > 0 for i in items),
"number_of_answers":
sum(answers[i] for i in items),
"number_of_correct_answers":
sum(correct_answers[i] for i in items),
}
if request.GET.get("mastered"):
response[identifier]["number_of_mastered_items"] = sum(
mastered[i] for i in items)
return render_json(request,
response,
template='models_user_stats.html',
help_text=user_stats.__doc__)
def answers_per_month(request):
try:
from pylab import rcParams
import matplotlib.pyplot as plt
import pandas
import seaborn as sns
except ImportError:
return HttpResponse('Can not import python packages for analysis.',
status=503)
categories = load_query_json(request.GET, "categories", "[]")
translated = Item.objects.translate_identifiers(categories,
get_language(request))
translated_inverted = {item: name for name, item in translated.items()}
children = pandas.DataFrame([{
'item': item,
'category': translated_inverted[category]
} for category, items in Item.objects.get_reachable_children(
list(translated.values()), get_language(request)).items()
for item in items])
with connection.cursor() as cursor:
cursor.execute('''
SELECT item_id, date_part('month', time), COUNT(1)
FROM proso_models_answer
GROUP BY 1, 2
''')
data = []
for item, month, answers in cursor:
data.append({
'item': item,
'month': month,
'answers': answers,
})
data = pandas.DataFrame(data)
if len(children) == 0:
data['category'] = data['item'].apply(lambda i: 'category/all')
else:
data = pandas.merge(data, children, on='item', how='inner')
if 'percentage' in request.GET:
def _percentage(group):
total = group['answers'].sum()
return group.groupby('category').apply(lambda g: 100 * g[
'answers'].sum() / total).reset_index().rename(
columns={0: 'answers'})
data = data.groupby('month').apply(_percentage).reset_index()
def _apply(group):
group['answers_cumsum'] = group['answers'].cumsum()
return group
data = data.sort_values(by=['category'],
ascending=False).groupby('month').apply(_apply)
data['month'] = data['month'].astype(int)
sns.set(style='white')
rcParams['figure.figsize'] = 15, 10
palette = sns.color_palette("hls", max(5, len(categories)))
fig = plt.figure()
for i, category in enumerate(sorted(data['category'].unique())):
item_data = data[data['category'] == category]
sns.barplot(x='month',
y='answers_cumsum',
data=item_data,
label=category.split('/')[1],
color=palette[i % len(palette)],
ci=None)
plt.ylabel('Answers' + (' (%)' if 'percentage' in request.GET else ''))
plt.xlabel('Month')
plt.title('Answers per Month')
if 'percentage' in request.GET:
plt.ylim(0, 100)
plt.legend(loc='center left', bbox_to_anchor=(1, 0.5))
response = HttpResponse(content_type='image/png')
canvas = FigureCanvas(fig)
canvas.print_png(response)
return response
def practice(request):
"""
Return the given number of questions to practice adaptively. In case of
POST request, try to save the answer(s).
GET parameters:
filter:
list of lists of identifiers (may be prefixed by minus sign to
mark complement)
language:
language (str) of flashcards
avoid:
list of item ids to avoid
limit:
number of returned questions (default 10, maximum 100)
time:
time in format '%Y-%m-%d_%H:%M:%S' used for practicing
user:
identifier for the practicing user (only for stuff users)
stats:
turn on the enrichment of the objects by some statistics
html:
turn on the HTML version of the API
BODY:
see answer resource
"""
if request.user.id is None: # Google Bot
return render_json(request, {
'error': _('There is no user available for the practice.'),
'error_type': 'user_undefined'
}, status=400, template='models_json.html')
limit = min(int(request.GET.get('limit', 10)), 100)
# prepare
user = get_user_id(request)
time = get_time(request)
avoid = load_query_json(request.GET, "avoid", "[]")
practice_filter = get_filter(request)
practice_context = PracticeContext.objects.from_content(practice_filter)
environment = get_environment()
item_selector = get_item_selector()
if is_time_overridden(request):
environment.shift_time(time)
# save answers
if request.method == 'POST':
_save_answers(request, practice_context)
if len(practice_filter) > 0:
item_ids = Item.objects.filter_all_reachable_leaves(practice_filter, get_language(request))
else:
item_ids = Item.objects.get_all_available_leaves()
item_ids = list(set(item_ids) - set(avoid))
if len(item_ids) == 0:
return render_json(request, {
'error': _('There is no item for the given filter to practice.'),
'error_type': 'empty_practice'
}, status=404, template='models_json.html')
selected_items, meta = item_selector.select(environment, user, item_ids, time, practice_context.id, limit, items_in_queue=len(avoid))
result = []
for item, item_meta in zip(selected_items, meta):
question = {
'object_type': 'question',
'payload': Item.objects.item_id_to_json(item),
}
if item_meta is not None:
question['meta'] = item_meta
result.append(question)
return render_json(request, result, template='models_json.html', help_text=practice.__doc__)
def user_stats(request):
"""
Get user statistics for selected groups of items
time:
time in format '%Y-%m-%d_%H:%M:%S' used for practicing
user:
identifier of the user (only for stuff users)
username:
username of user (only for users with public profile)
filters: -- use this or body
json as in BODY
mastered:
use model to compute number of mastered items - can be slowed
language:
language of the items
BODY
json in following format:
{
"#identifier": [] -- custom identifier (str) and filter
...
}
"""
timer('user_stats')
response = {}
data = None
if request.method == "POST":
data = json.loads(request.body.decode("utf-8"))["filters"]
if "filters" in request.GET:
data = load_query_json(request.GET, "filters")
if data is None:
return render_json(request, {}, template='models_user_stats.html', help_text=user_stats.__doc__)
environment = get_environment()
if is_time_overridden(request):
environment.shift_time(get_time(request))
user_id = get_user_id(request)
language = get_language(request)
filter_names, filter_filters = list(zip(*sorted(data.items())))
reachable_leaves = Item.objects.filter_all_reachable_leaves_many(filter_filters, language)
all_leaves = flatten(reachable_leaves)
answers = dict(list(zip(all_leaves, environment.number_of_answers_more_items(all_leaves, user_id))))
correct_answers = dict(list(zip(all_leaves, environment.number_of_correct_answers_more_items(all_leaves, user_id))))
if request.GET.get("mastered"):
timer('user_stats_mastered')
mastery_threshold = get_mastery_trashold()
predictions = get_predictive_model().predict_more_items(environment, user_id, all_leaves, get_time(request))
mastered = dict(list(zip(all_leaves, [p >= mastery_threshold for p in predictions])))
LOGGER.debug("user_stats - getting predictions for flashcards took %s seconds", (timer('user_stats_mastered')))
for identifier, items in zip(filter_names, reachable_leaves):
if len(items) == 0:
response[identifier] = {
"filter": data[identifier],
"number_of_flashcards": 0,
}
else:
response[identifier] = {
"filter": data[identifier],
"number_of_flashcards": len(items),
"number_of_practiced_flashcards": sum(answers[i] > 0 for i in items),
"number_of_answers": sum(answers[i] for i in items),
"number_of_correct_answers": sum(correct_answers[i] for i in items),
}
if request.GET.get("mastered"):
response[identifier]["number_of_mastered_flashcards"]= sum(mastered[i] for i in items)
return render_json(request, response, template='models_user_stats.html', help_text=user_stats.__doc__)
def answers_per_month(request):
try:
from pylab import rcParams
import matplotlib.pyplot as plt
import pandas
import seaborn as sns
except ImportError:
return HttpResponse('Can not import python packages for analysis.', status=503)
categories = load_query_json(request.GET, "categories", "[]")
translated = Item.objects.translate_identifiers(categories, get_language(request))
translated_inverted = {item: name for name, item in translated.items()}
children = pandas.DataFrame([
{'item': item, 'category': translated_inverted[category]}
for category, items in Item.objects.get_reachable_children(
list(translated.values()), get_language(request)
).items()
for item in items
])
with connection.cursor() as cursor:
cursor.execute(
'''
SELECT item_id, date_part('month', time), COUNT(1)
FROM proso_models_answer
GROUP BY 1, 2
'''
)
data = []
for item, month, answers in cursor:
data.append({
'item': item,
'month': month,
'answers': answers,
})
data = pandas.DataFrame(data)
if len(children) == 0:
data['category'] = data['item'].apply(lambda i: 'category/all')
else:
data = pandas.merge(data, children, on='item', how='inner')
if 'percentage' in request.GET:
def _percentage(group):
total = group['answers'].sum()
return group.groupby('category').apply(lambda g: 100 * g['answers'].sum() / total).reset_index().rename(columns={0: 'answers'})
data = data.groupby('month').apply(_percentage).reset_index()
def _apply(group):
group['answers_cumsum'] = group['answers'].cumsum()
return group
data = data.sort_values(by=['category'], ascending=False).groupby('month').apply(_apply)
data['month'] = data['month'].astype(int)
sns.set(style='white')
rcParams['figure.figsize'] = 15, 10
palette = sns.color_palette("hls", max(5, len(categories)))
fig = plt.figure()
for i, category in enumerate(sorted(data['category'].unique())):
item_data = data[data['category'] == category]
sns.barplot(
x='month',
y='answers_cumsum',
data=item_data,
label=category.split('/')[1],
color=palette[i % len(palette)],
ci=None
)
plt.ylabel('Answers' + (' (%)' if 'percentage' in request.GET else ''))
plt.xlabel('Month')
plt.title('Answers per Month')
if 'percentage' in request.GET:
plt.ylim(0, 100)
plt.legend(loc='center left', bbox_to_anchor=(1, 0.5))
response = HttpResponse(content_type='image/png')
canvas = FigureCanvas(fig)
canvas.print_png(response)
return response
def get_filter(request):
return load_query_json(request.GET, "filter", "[]")