def handle_gc(self, options):
timer('recompute_gc')
print(' -- collecting garbage')
to_gc = [
str(x.id) for x in EnvironmentInfo.objects.filter(
status=EnvironmentInfo.STATUS_DISABLED).all()
]
if not to_gc:
print(' -- no environment info to collect')
return
to_gc_str = ','.join(to_gc)
with closing(connection.cursor()) as cursor:
cursor.execute(
'DELETE FROM proso_models_variable WHERE info_id IN (%s)' %
to_gc_str)
variables = cursor.rowcount
cursor.execute(
'DELETE FROM proso_models_audit WHERE info_id IN (%s)' %
to_gc_str)
audits = cursor.rowcount
cursor.execute(
'DELETE FROM proso_models_environmentinfo WHERE id IN (%s)' %
to_gc_str)
infos = cursor.rowcount
print(' -- collecting garbage, time:', timer('recompute_gc'),
'seconds, deleted', variables, 'variables,', audits,
'audit records,', infos, 'environment info records')
def _save_answers(request, practice_context):
timer('_save_answers')
json_objects = _get_answers(request)
answers = []
for json_object in json_objects:
if 'answer_class' not in json_object:
raise BadRequestException('The answer does not contain key "answer_class".')
answer_class = Answer.objects.answer_class(json_object['answer_class'])
answers.append(answer_class.objects.from_json(json_object, practice_context, request.user.id))
LOGGER.debug("saving of %s answers took %s seconds", len(answers), timer('_save_answers'))
return answers
def handle_recompute(self, options):
timer('recompute_all')
info = self.load_environment_info(options['initial'], options['config_name'])
if options['finish']:
with transaction.atomic():
to_process = self.number_of_answers_to_process(info)
if self.number_of_answers_to_process(info) >= options['batch_size']:
raise CommandError("There is more then allowed number of answers (%s) to process." % to_process)
self.recompute(info, options)
else:
self.recompute(info, options)
print(' -- total time:', timer('recompute_all'), 'seconds')
def _save_answers(request, practice_context):
timer('_save_answers')
json_objects = _get_answers(request)
answers = []
for json_object in json_objects:
if 'answer_class' not in json_object:
raise BadRequestException(
'The answer does not contain key "answer_class".')
answer_class = Answer.objects.answer_class(json_object['answer_class'])
answers.append(
answer_class.objects.from_json(json_object, practice_context,
request.user.id))
LOGGER.debug("saving of %s answers took %s seconds", len(answers),
timer('_save_answers'))
return answers
def handle_recompute(self, options):
timer('recompute_all')
info = self.load_environment_info(options['initial'],
options['config_name'])
if options['finish']:
with transaction.atomic():
to_process = self.number_of_answers_to_process(info)
if self.number_of_answers_to_process(
info) >= options['batch_size']:
raise CommandError(
"There is more then allowed number of answers (%s) to process."
% to_process)
self.recompute(info, options)
else:
self.recompute(info, options)
print(' -- total time:', timer('recompute_all'), 'seconds')
def handle_gc(self, options):
timer('recompute_gc')
print(' -- collecting garbage')
to_gc = [str(x.id) for x in EnvironmentInfo.objects.filter(status=EnvironmentInfo.STATUS_DISABLED).all()]
if not to_gc:
print(' -- no environment info to collect')
return
to_gc_str = ','.join(to_gc)
with closing(connection.cursor()) as cursor:
cursor.execute('DELETE FROM proso_models_variable WHERE info_id IN (%s)' % to_gc_str)
variables = cursor.rowcount
cursor.execute('DELETE FROM proso_models_audit WHERE info_id IN (%s)' % to_gc_str)
audits = cursor.rowcount
cursor.execute('DELETE FROM proso_models_environmentinfo WHERE id IN (%s)' % to_gc_str)
infos = cursor.rowcount
print(' -- collecting garbage, time:', timer('recompute_gc'), 'seconds, deleted', variables, 'variables,', audits, 'audit records,', infos, 'environment info records')
def to_practice_counts(request):
"""
Get number of items available to practice.
filters: -- use this or body
json as in BODY
language:
language of the items
BODY
json in following format:
{
"#identifier": [] -- custom identifier (str) and filter
...
}
"""
data = None
if request.method == "POST":
data = json.loads(request.body.decode("utf-8"))["filters"]
if "filters" in request.GET:
data = load_query_json(request.GET, "filters")
if data is None:
return render_json(request, {},
template='models_json.html',
help_text=to_practice_counts.__doc__)
language = get_language(request)
timer('to_practice_counts')
filter_names, filter_filters = list(zip(*sorted(data.items())))
reachable_leaves = Item.objects.filter_all_reachable_leaves_many(
filter_filters, language)
response = {
group_id: {
'filter': data[group_id],
'number_of_items': len(items),
}
for group_id, items in zip(filter_names, reachable_leaves)
}
LOGGER.debug(
"flashcard_counts - getting flashcards in groups took %s seconds",
(timer('to_practice_counts')))
return render_json(request,
response,
template='models_json.html',
help_text=to_practice_counts.__doc__)
def to_practice_counts(request):
"""
Get number of items available to practice.
filters: -- use this or body
json as in BODY
language:
language of the items
BODY
json in following format:
{
"#identifier": [] -- custom identifier (str) and filter
...
}
"""
data = None
if request.method == "POST":
data = json.loads(request.body.decode("utf-8"))["filters"]
if "filters" in request.GET:
data = load_query_json(request.GET, "filters")
if data is None:
return render_json(request, {}, template='models_json.html', help_text=to_practice_counts.__doc__)
language = get_language(request)
timer('to_practice_counts')
filter_names, filter_filters = list(zip(*sorted(data.items())))
reachable_leaves = Item.objects.filter_all_reachable_leaves_many(filter_filters, language)
response = {
group_id: {
'filter': data[group_id],
'number_of_items': len(items),
}
for group_id, items in zip(filter_names, reachable_leaves)
}
LOGGER.debug("flashcard_counts - getting flashcards in groups took %s seconds", (timer('to_practice_counts')))
return render_json(request, response, template='models_json.html', help_text=to_practice_counts.__doc__)
def user_stats(request):
"""
Get user statistics for selected groups of items
time:
time in format '%Y-%m-%d_%H:%M:%S' used for practicing
user:
identifier of the user (only for stuff users)
username:
username of user (only for users with public profile)
filters: -- use this or body
json as in BODY
mastered:
use model to compute number of mastered items - can be slowed
language:
language of the items
BODY
json in following format:
{
"#identifier": [] -- custom identifier (str) and filter
...
}
"""
timer('user_stats')
response = {}
data = None
if request.method == "POST":
data = json.loads(request.body.decode("utf-8"))["filters"]
if "filters" in request.GET:
data = load_query_json(request.GET, "filters")
if data is None:
return render_json(request, {},
template='models_user_stats.html',
help_text=user_stats.__doc__)
environment = get_environment()
if is_time_overridden(request):
environment.shift_time(get_time(request))
user_id = get_user_id(request)
language = get_language(request)
filter_names, filter_filters = list(zip(*sorted(data.items())))
reachable_leaves = Item.objects.filter_all_reachable_leaves_many(
filter_filters, language)
all_leaves = flatten(reachable_leaves)
answers = dict(
list(
zip(all_leaves,
environment.number_of_answers_more_items(all_leaves,
user_id))))
correct_answers = dict(
list(
zip(
all_leaves,
environment.number_of_correct_answers_more_items(
all_leaves, user_id))))
if request.GET.get("mastered"):
timer('user_stats_mastered')
mastery_threshold = get_mastery_trashold()
predictions = get_predictive_model().predict_more_items(
environment, user_id, all_leaves, get_time(request))
mastered = dict(
list(zip(all_leaves,
[p >= mastery_threshold for p in predictions])))
LOGGER.debug(
"user_stats - getting predictions for flashcards took %s seconds",
(timer('user_stats_mastered')))
for identifier, items in zip(filter_names, reachable_leaves):
if len(items) == 0:
response[identifier] = {
"filter": data[identifier],
"number_of_flashcards": 0,
}
else:
response[identifier] = {
"filter":
data[identifier],
"number_of_flashcards":
len(items),
"number_of_practiced_flashcards":
sum(answers[i] > 0 for i in items),
"number_of_answers":
sum(answers[i] for i in items),
"number_of_correct_answers":
sum(correct_answers[i] for i in items),
}
if request.GET.get("mastered"):
response[identifier]["number_of_mastered_flashcards"] = sum(
mastered[i] for i in items)
return render_json(request,
response,
template='models_user_stats.html',
help_text=user_stats.__doc__)
def recompute(self, info, options):
print(' -- preparing phase')
timer('recompute_prepare')
environment = self.load_environment(info)
users, items = self.load_user_and_item_ids(info, options['batch_size'])
environment.prefetch(users, items)
predictive_model = get_predictive_model()
print(' -- preparing phase, time:', timer('recompute_prepare'),
'seconds')
timer('recompute_model')
print(' -- model phase')
with closing(connection.cursor()) as cursor:
cursor.execute(
'''
SELECT
id,
user_id,
item_id,
item_asked_id,
item_answered_id,
time,
response_time,
guess
FROM proso_models_answer
ORDER BY id
OFFSET %s LIMIT %s
''', [info.load_progress, options['batch_size']])
progress_bar = progress.bar(cursor,
every=max(1, cursor.rowcount / 100),
expected_size=cursor.rowcount)
info.load_progress += cursor.rowcount
for (answer_id, user, item, asked, answered, time, response_time,
guess) in progress_bar:
predictive_model.predict_and_update(environment,
user,
item,
asked == answered,
time.replace(tzinfo=None),
item_answered=answered,
item_asked=asked,
guess=guess,
answer_id=answer_id)
environment.process_answer(user, item, asked, answered, time,
answer_id, response_time, guess)
print(' -- model phase, time:', timer('recompute_model'), 'seconds')
timer('recompute_flush')
print(' -- flushing phase')
environment.flush(clean=options['finish'])
print(' -- flushing phase, time:', timer('recompute_flush'),
'seconds, total number of answers:', info.load_progress)
if options['finish']:
timer('recompute_finish')
print(' -- finishing phase')
try:
previous_info = EnvironmentInfo.objects.get(
status=EnvironmentInfo.STATUS_ACTIVE)
previous_info.status = EnvironmentInfo.STATUS_DISABLED
previous_info.save()
cache.delete(ENVIRONMENT_INFO_CACHE_KEY)
except EnvironmentInfo.DoesNotExist:
pass
info.status = EnvironmentInfo.STATUS_ACTIVE
print(' -- finishing phase, time:', timer('recompute_finish'),
'seconds')
info.save()
def recompute(self, info, options):
print(' -- preparing phase')
timer('recompute_prepare')
environment = self.load_environment(info)
users, items = self.load_user_and_item_ids(info, options['batch_size'])
environment.prefetch(users, items)
predictive_model = get_predictive_model()
print(' -- preparing phase, time:', timer('recompute_prepare'), 'seconds')
timer('recompute_model')
print(' -- model phase')
with closing(connection.cursor()) as cursor:
cursor.execute(
'''
SELECT
id,
user_id,
item_id,
item_asked_id,
item_answered_id,
time,
response_time,
guess
FROM proso_models_answer
ORDER BY id
OFFSET %s LIMIT %s
''', [info.load_progress, options['batch_size']])
progress_bar = progress.bar(cursor, every=max(1, cursor.rowcount / 100), expected_size=cursor.rowcount)
info.load_progress += cursor.rowcount
for (answer_id, user, item, asked, answered, time, response_time, guess) in progress_bar:
predictive_model.predict_and_update(
environment,
user,
item,
asked == answered,
time,
item_answered=answered,
item_asked=asked,
guess=guess,
answer_id=answer_id)
environment.process_answer(user, item, asked, answered, time, answer_id, response_time, guess)
print(' -- model phase, time:', timer('recompute_model'), 'seconds')
timer('recompute_flush')
print(' -- flushing phase')
environment.flush(clean=options['finish'])
print(' -- flushing phase, time:', timer('recompute_flush'), 'seconds, total number of answers:', info.load_progress)
if options['finish']:
timer('recompute_finish')
print(' -- finishing phase')
try:
previous_info = EnvironmentInfo.objects.get(status=EnvironmentInfo.STATUS_ACTIVE)
previous_info.status = EnvironmentInfo.STATUS_DISABLED
previous_info.save()
cache.delete(ENVIRONMENT_INFO_CACHE_KEY)
except EnvironmentInfo.DoesNotExist:
pass
info.status = EnvironmentInfo.STATUS_ACTIVE
print(' -- finishing phase, time:', timer('recompute_finish'), 'seconds')
info.save()
def user_stats(request):
"""
Get user statistics for selected groups of items
time:
time in format '%Y-%m-%d_%H:%M:%S' used for practicing
user:
identifier of the user (only for stuff users)
username:
username of user (only for users with public profile)
filters: -- use this or body
json as in BODY
mastered:
use model to compute number of mastered items - can be slowed
language:
language of the items
BODY
json in following format:
{
"#identifier": [] -- custom identifier (str) and filter
...
}
"""
timer('user_stats')
response = {}
data = None
if request.method == "POST":
data = json.loads(request.body.decode("utf-8"))["filters"]
if "filters" in request.GET:
data = load_query_json(request.GET, "filters")
if data is None:
return render_json(request, {}, template='models_user_stats.html', help_text=user_stats.__doc__)
environment = get_environment()
if is_time_overridden(request):
environment.shift_time(get_time(request))
user_id = get_user_id(request)
language = get_language(request)
filter_names, filter_filters = list(zip(*sorted(data.items())))
reachable_leaves = Item.objects.filter_all_reachable_leaves_many(filter_filters, language)
all_leaves = flatten(reachable_leaves)
answers = dict(list(zip(all_leaves, environment.number_of_answers_more_items(all_leaves, user_id))))
correct_answers = dict(list(zip(all_leaves, environment.number_of_correct_answers_more_items(all_leaves, user_id))))
if request.GET.get("mastered"):
timer('user_stats_mastered')
mastery_threshold = get_mastery_trashold()
predictions = get_predictive_model().predict_more_items(environment, user_id, all_leaves, get_time(request))
mastered = dict(list(zip(all_leaves, [p >= mastery_threshold for p in predictions])))
LOGGER.debug("user_stats - getting predictions for flashcards took %s seconds", (timer('user_stats_mastered')))
for identifier, items in zip(filter_names, reachable_leaves):
if len(items) == 0:
response[identifier] = {
"filter": data[identifier],
"number_of_flashcards": 0,
}
else:
response[identifier] = {
"filter": data[identifier],
"number_of_flashcards": len(items),
"number_of_practiced_flashcards": sum(answers[i] > 0 for i in items),
"number_of_answers": sum(answers[i] for i in items),
"number_of_correct_answers": sum(correct_answers[i] for i in items),
}
if request.GET.get("mastered"):
response[identifier]["number_of_mastered_flashcards"]= sum(mastered[i] for i in items)
return render_json(request, response, template='models_user_stats.html', help_text=user_stats.__doc__)