def newsletter_registration(email):
configuration = Configuration.get_solo()
headers = {
"accept": "application/json",
"content-type": "application/json",
"api-key": settings.SIB_API_KEY,
}
data = {
"email": email,
# "attributes": {
# "FIRSTNAME": "",
# "LASTNAME": ""
# },
"includeListIds": [int(settings.SIB_NEWSLETTER_LIST_ID)],
"templateId": int(settings.SIB_NEWSLETTER_DOI_TEMPLATE_ID),
"redirectionUrl":
configuration.application_frontend_url + "?newsletter=confirmed",
"updateEnabled": True,
}
return requests.post(settings.SIB_CONTACT_DOI_ENDPOINT,
headers=headers,
data=json.dumps(data))
def changelist_view(self, request, extra_context=None):
"""
show chart of answers per day
https://dev.to/danihodovic/integrating-chart-js-with-django-admin-1kjb
Corresponding template in templates/admin/stats/dailystat/change_list.html
"""
if request.POST.get("run_generate_daily_stats_script", False):
management.call_command("generate_daily_stats")
# custom form
current_field = str(
request.POST.get("field",
constants.AGGREGATION_FIELD_CHOICE_LIST[0]))
current_scale = str(
request.POST.get("scale",
constants.AGGREGATION_SCALE_CHOICE_LIST[0]))
current_since_date = str(
request.POST.get("since_date",
constants.AGGREGATION_SINCE_DATE_DEFAULT))
# Aggregate answers per day
# chart_data_query = DailyStat.objects.extra(select={"day": "date(date)"}) # sqlite
chart_data_query = DailyStat.objects.agg_timeseries(
current_field, scale=current_scale, since_date=current_since_date)
chart_data_list = list(chart_data_query)
# Serialize and attach the chart data to the template context
chart_data_json = json.dumps(chart_data_list, cls=DjangoJSONEncoder)
extra_context = extra_context or {
"configuration": Configuration.get_solo(),
"chart_data": chart_data_json,
"field_choice_list": constants.AGGREGATION_FIELD_CHOICE_LIST,
"current_field": current_field,
"scale_choice_list": constants.AGGREGATION_SCALE_CHOICE_LIST,
"current_scale": current_scale,
"since_date_min": constants.AGGREGATION_SINCE_DATE_DEFAULT,
"current_since_date": current_since_date,
}
# Call the superclass changelist_view to render the page
return super().changelist_view(request, extra_context=extra_context)
def changelist_view(self, request, extra_context=None):
"""
Corresponding template in templates/admin/api/question/change_list_with_import.html
"""
notion_questions_import_scope_choices = [(
scope_value,
scope_label,
"notion_questions_scope_" + str(scope_value) + "_last_imported",
) for (
scope_value,
scope_label,
) in api_constants.NOTION_QUESTIONS_IMPORT_SCOPE_CHOICES[1:]]
notion_questions_import_response = []
if request.POST.get("run_import_questions_from_notion_script", False):
out = StringIO()
scope = request.POST.get("run_import_questions_from_notion_script")
management.call_command("import_questions_from_notion",
scope,
stdout=out)
notion_questions_import_response = out.getvalue()
notion_questions_import_response = notion_questions_import_response.split(
"\n")
notion_questions_import_response = [
elem.split("///") if ("///" in elem) else elem
for elem in notion_questions_import_response
]
extra_context = extra_context or {
"configuration": Configuration.get_solo(),
"notion_questions_import_scope_choices":
notion_questions_import_scope_choices,
"notion_questions_import_response":
notion_questions_import_response,
}
# Call the superclass changelist_view to render the page
return super().changelist_view(request, extra_context=extra_context)
def handle(self, *args, **options):
# init
start_time = time.time()
current_datetime = datetime.now()
current_datetime_string = current_datetime.strftime("%Y-%m-%d-%H-%M")
current_datetime_string_pretty = current_datetime.strftime(
"%Y-%m-%d %H:%M")
branch_name = f"update-data-{current_datetime_string}"
pull_request_name = f"Update: data ({current_datetime_string_pretty})"
# update configuration first
configuration = Configuration.get_solo()
configuration.github_data_last_exported = timezone.now()
configuration.save()
print("--- Step 1 done : init (%s seconds) ---" %
round(time.time() - start_time, 1))
# update & commit data files
try:
#####################################
# data/configuration.yaml
start_time = time.time()
configuration_yaml = utilities.serialize_model_to_yaml(
"core", model_label="configuration", flat=True)
configuration_element = utilities_github.create_file_element(
file_path="data/configuration.yaml",
file_content=configuration_yaml)
print("--- Step 2.1 done : configuration.yaml (%s seconds) ---" %
round(time.time() - start_time, 1))
#####################################
# # data/categories.yaml
start_time = time.time()
# categories_yaml = utilities.serialize_model_to_yaml("api", model_label="category", flat=True) # noqa
# categories_element = utilities_github.create_file_element(
# file_path="data/categories.yaml",
# file_content=categories_element
# )
print("--- Step 2.2 done : categories.yaml (skipped) ---")
#####################################
# data/tags.yaml
start_time = time.time()
tags_yaml = utilities.serialize_model_to_yaml("api",
model_label="tag",
flat=True)
tags_element = utilities_github.create_file_element(
file_path="data/tags.yaml", file_content=tags_yaml)
print("--- Step 2.3 done : tags.yaml (%s seconds) ---" %
round(time.time() - start_time, 1))
#####################################
# data/questions.yaml
start_time = time.time()
questions_yaml = utilities.serialize_model_to_yaml(
"api", model_label="question", flat=True)
questions_element = utilities_github.create_file_element(
file_path="data/questions.yaml", file_content=questions_yaml)
print("--- Step 2.4 done : questions.yaml (%s seconds) ---" %
round(time.time() - start_time, 1))
#####################################
# data/quizzes.yaml
start_time = time.time()
quizzes_yaml = utilities.serialize_model_to_yaml(
"api", model_label="quiz", flat=True)
quizzes_element = utilities_github.create_file_element(
file_path="data/quizzes.yaml", file_content=quizzes_yaml)
print("--- Step 2.5 done : quizzes.yaml (%s seconds) ---" %
round(time.time() - start_time, 1))
#####################################
# data/quiz-questions.yaml
start_time = time.time()
quiz_questions_yaml = utilities.serialize_model_to_yaml(
"api", model_label="quizquestion", flat=True)
quiz_questions_element = utilities_github.create_file_element(
file_path="data/quiz-questions.yaml",
file_content=quiz_questions_yaml)
print("--- Step 2.6 done : quiz-questions.yaml (%s seconds) ---" %
round(time.time() - start_time, 1))
#####################################
# data/quiz-relationships.yaml
start_time = time.time()
quiz_relationships_yaml = utilities.serialize_model_to_yaml(
"api", model_label="quizrelationship", flat=True)
quiz_relationships_element = utilities_github.create_file_element(
file_path="data/quiz-relationships.yaml",
file_content=quiz_relationships_yaml,
)
print(
"--- Step 2.7 done : quiz-relationships.yaml (%s seconds) ---"
% round(time.time() - start_time, 1))
#####################################
# update frontend file with timestamp
# frontend/src/constants.js
start_time = time.time()
old_frontend_constants_file_content = utilities_github.get_file(
file_path="frontend/src/constants.js", )
new_frontend_constants_file_content_string = utilities.update_frontend_last_updated_datetime( # noqa
old_frontend_constants_file_content.decoded_content.decode(),
current_datetime_string_pretty,
)
new_frontend_constants_file_element = utilities_github.create_file_element(
file_path="frontend/src/constants.js",
file_content=new_frontend_constants_file_content_string,
)
print("--- Step 2.8 done : constants.js (%s seconds) ---" %
round(time.time() - start_time, 1))
#####################################
# commit files
start_time = time.time()
utilities_github.update_multiple_files(
branch_name=branch_name,
commit_message="Data: data update",
file_element_list=[
configuration_element,
tags_element,
questions_element,
quizzes_element,
quiz_questions_element,
quiz_relationships_element,
new_frontend_constants_file_element,
],
)
print("--- Step 3 done : committed to branch (%s seconds) ---" %
round(time.time() - start_time, 1))
#####################################
# create pull request
start_time = time.time()
if not settings.DEBUG:
# create pull request
pull_request_message = ("Mise à jour de la donnée :"
"<ul>"
"<li>data/configuration.yaml</li>"
"<li>data/tags.yaml</li>"
"<li>data/questions.yaml</li>"
"<li>data/quizzes.yaml</li>"
"<li>data/quiz-questions.yaml</li>"
"<li>data/quiz-relationships.yaml</li>"
"</ul>")
pull_request = utilities_github.create_pull_request(
pull_request_title=pull_request_name,
pull_request_message=pull_request_message,
branch_name=branch_name,
pull_request_labels="automerge",
)
print(
"--- Step 4 done : created Pull Request (%s seconds) ---" %
round(time.time() - start_time, 1))
# return
self.stdout.write(pull_request.html_url)
except Exception as e:
print(e)
self.stdout.write(str(e))
def handle(self, *args, **options):
# init
start_time = time.time()
current_datetime = datetime.now()
current_datetime_string = current_datetime.strftime("%Y-%m-%d-%H-%M")
current_datetime_string_pretty = current_datetime.strftime("%Y-%m-%d %H:%M")
branch_name = f"update-stats-{current_datetime_string}"
pull_request_name = f"Update: stats ({current_datetime_string_pretty})"
# update configuration first
configuration = Configuration.get_solo()
configuration.github_stats_last_exported = timezone.now()
configuration.save()
print(
"--- Step 1 done : init (%s seconds) ---"
% round(time.time() - start_time, 1)
)
# update & commit stats files
try:
#####################################
# data/stats.yaml
start_time = time.time()
stats_dict = {
**utilities_stats.question_stats(),
**utilities_stats.quiz_stats(),
**utilities_stats.answer_stats(),
**utilities_stats.category_stats(),
**utilities_stats.tag_stats(),
**utilities_stats.contribution_stats(),
}
stats_yaml = yaml.safe_dump(stats_dict, allow_unicode=True, sort_keys=False)
stats_element = utilities_github.create_file_element(
file_path="data/stats.yaml", file_content=stats_yaml
)
print(
"--- Step 2.1 done : stats.yaml (%s seconds) ---"
% round(time.time() - start_time, 1)
)
#####################################
# data/difficulty-levels.yaml
start_time = time.time()
difficulty_levels_list = utilities_stats.difficulty_aggregate()
difficulty_levels_yaml = yaml.safe_dump(
difficulty_levels_list, allow_unicode=True, sort_keys=False
)
difficulty_levels_element = utilities_github.create_file_element(
file_path="data/difficulty-levels.yaml",
file_content=difficulty_levels_yaml,
)
print(
"--- Step 2.2 done : difficulty-levels.yaml (%s seconds) ---"
% round(time.time() - start_time, 1)
)
#####################################
# data/authors.yaml
start_time = time.time()
authors_list = utilities_stats.author_aggregate()
authors_yaml = yaml.safe_dump(
authors_list, allow_unicode=True, sort_keys=False
)
authors_element = utilities_github.create_file_element(
file_path="data/authors.yaml", file_content=authors_yaml
)
print(
"--- Step 2.3 done : authors.yaml (%s seconds) ---"
% round(time.time() - start_time, 1)
)
#####################################
# data/languages.yaml
start_time = time.time()
languages_list = utilities_stats.language_aggregate()
languages_yaml = yaml.safe_dump(
languages_list, allow_unicode=True, sort_keys=False
)
languages_element = utilities_github.create_file_element(
file_path="data/languages.yaml", file_content=languages_yaml
)
print(
"--- Step 2.4 done : languages.yaml (%s seconds) ---"
% round(time.time() - start_time, 1)
)
#####################################
# data/quiz-stats.yaml
start_time = time.time()
quiz_detail_stats_list = utilities_stats.quiz_detail_stats()
quiz_detail_stats_yaml = yaml.safe_dump(
quiz_detail_stats_list, allow_unicode=True, sort_keys=False
)
quiz_stats_element = utilities_github.create_file_element(
file_path="data/quiz-stats.yaml", file_content=quiz_detail_stats_yaml
)
print(
"--- Step 2.5 done : quiz-stats.yaml (%s seconds) ---"
% round(time.time() - start_time, 1)
)
#####################################
# update frontend file with timestamp
# frontend/src/constants.js
start_time = time.time()
old_frontend_constants_file_content = utilities_github.get_file(
file_path="frontend/src/constants.js",
)
new_frontend_constants_file_content_string = utilities.update_frontend_last_updated_datetime( # noqa
old_frontend_constants_file_content.decoded_content.decode(),
current_datetime_string_pretty,
)
new_frontend_constants_file_element = utilities_github.create_file_element(
file_path="frontend/src/constants.js",
file_content=new_frontend_constants_file_content_string,
)
print(
"--- Step 2.6 done : constants.js (%s seconds) ---"
% round(time.time() - start_time, 1)
)
#####################################
# commit files
start_time = time.time()
utilities_github.update_multiple_files(
branch_name=branch_name,
commit_message="Data: stats update",
file_element_list=[
stats_element,
difficulty_levels_element,
authors_element,
languages_element,
quiz_stats_element,
new_frontend_constants_file_element,
],
)
print(
"--- Step 3 done : committed to branch (%s seconds) ---"
% round(time.time() - start_time, 1)
)
#####################################
# create pull request
start_time = time.time()
if not settings.DEBUG:
# create pull request
pull_request_message = (
"Mise à jour des stats :"
"<ul>"
"<li>data/stats.yaml</li>"
"<li>data/difficulty-levels.yaml</li>"
"<li>data/authors.yaml</li>"
"<li>data/languages.yaml</li>"
"<li>data/tags.yaml</li>"
"<li>data/quiz-stats.yaml</li>"
"</ul>"
)
pull_request = utilities_github.create_pull_request(
pull_request_title=pull_request_name,
pull_request_message=pull_request_message,
branch_name=branch_name,
pull_request_labels="automerge",
)
print(
"--- Step 4 done : created Pull Request (%s seconds) ---"
% round(time.time() - start_time, 1)
)
# return
self.stdout.write(pull_request.html_url)
except Exception as e:
print(e)
self.stdout.write(str(e))
def handle(self, *args, **options):
#########################################################
# Init
#########################################################
scope = options["scope"]
notion_questions_list = []
questions_ids_duplicate = []
questions_ids_missing = []
tags_created = []
questions_created = []
questions_updated = set()
questions_updates = {}
validation_errors = []
all_categories_list = list(Category.objects.all())
all_tags_name_list = list(Tag.objects.all().values_list("name",
flat=True))
#########################################################
# Fetch questions from Notion
#########################################################
start_time = time.time()
try:
notion_questions_list = utilities_notion.get_questions_table_rows()
except: # noqa
self.stdout.write("Erreur accès Notion. token_v2 expiré ?")
return
print(
"--- Step 1 done : fetch questions from Notion (%s seconds) ---" %
round(time.time() - start_time, 1))
#########################################################
# Check question ids (duplicates & missing)
#########################################################
start_time = time.time()
# order by notion_questions_list by id
notion_questions_list = sorted(
notion_questions_list,
key=lambda question: question.get_property("id") or 0)
# check if id duplicates
notion_questions_id_list = [
question.get_property("id") for question in notion_questions_list
if question.get_property("id")
]
questions_ids_duplicate = [
item for item, count in collections.Counter(
notion_questions_id_list).items() if count > 1
]
# check if id 'missing'
for n in range(1, notion_questions_id_list[-1]):
if n not in notion_questions_id_list:
questions_ids_missing.append(n)
print(
"--- Step 2 done : check question ids (duplicates & missing) : %s seconds ---"
% round(time.time() - start_time, 1))
#########################################################
# Loop on questions and create, update, store validation_errors
#########################################################
start_time = time.time()
# reduce scope because of timeouts on Heroku (30 seconds)
if scope:
min_question_id = 200 * (scope - 1)
max_question_id = 200 * scope
notion_questions_list_scope = notion_questions_list[
min_question_id:max_question_id]
else:
notion_questions_list_scope = notion_questions_list
print(f"processing {len(notion_questions_list_scope)} questions")
print(
f"First question id : {notion_questions_list_scope[0].get_property('id')}"
)
print(
f"Last question id : {notion_questions_list_scope[-1].get_property('id')}"
)
for notion_question_row in notion_questions_list_scope:
question_validation_errors = []
notion_question_tag_objects = []
notion_question_last_updated = notion_question_row.get_property(
"Last edited time").date() # noqa
# check question has id
if notion_question_row.get_property("id") is None:
question_validation_errors.append(
ValidationError({"id": "Question sans id. vide ?"}))
# ignore questions not updated recently
elif options.get("skip-old") and (
notion_question_last_updated <
(datetime.now().date() -
timedelta(days=SKIP_QUESTIONS_LAST_UPDATED_SINCE_DAYS))):
pass
else:
# build question_dict from notion_row
notion_question_dict = self.transform_notion_question_row_to_question_dict(
notion_question_row)
# cleanup relation category
# - check category exists
# error if unknown category : api.models.DoesNotExist: Category matching query does not exist. # noqa
if notion_question_dict["category"] is not None:
notion_question_category_id = next(
(c.id for c in all_categories_list
if c.name == notion_question_dict["category"]),
None,
)
if notion_question_category_id:
notion_question_dict[
"category_id"] = notion_question_category_id
del notion_question_dict["category"]
else:
question_validation_errors.append(
ValidationError({
"category":
f"Question {notion_question_dict['id']}."
f"Category '{notion_question_dict['category']}' inconnue"
}))
# cleanup relation tags
# - if no tags, notion returns [""]
# - check tags exist, create them if not
# - then delete the "tags" key, we will make the M2M join later
notion_question_tag_name_list = []
if notion_question_dict["tags"] != [""]:
notion_question_tag_name_list = [
tag for tag in notion_question_dict["tags"]
if not tag.startswith("Quiz")
]
new_tags = [
new_tag for new_tag in notion_question_tag_name_list
if new_tag not in all_tags_name_list
]
# create missing tags
if len(new_tags):
Tag.objects.bulk_create(
[Tag(name=new_tag) for new_tag in new_tags])
all_tags_name_list += new_tags
tags_created += new_tags
del notion_question_dict["tags"]
# create or update
# - if the question does not have validation_errors
if len(question_validation_errors):
validation_errors += question_validation_errors
# - if the question has been updated recently
elif options.get("skip-old") and (
notion_question_last_updated <
(datetime.now().date() -
timedelta(days=SKIP_QUESTIONS_LAST_UPDATED_SINCE_DAYS))):
pass
else:
# the question doesn't have errors : ready to create/update
try:
db_question, created = Question.objects.get_or_create(
id=notion_question_dict["id"],
defaults=notion_question_dict)
# store info
if created:
notion_question_tag_objects = Tag.objects.filter(
name__in=notion_question_tag_name_list)
db_question.tags.set(notion_question_tag_objects)
questions_created.append(db_question.id)
else:
questions_updates_key = f"Question {db_question.id}"
# update basic fields
question_changes_list = self.update_question(
db_question, notion_question_dict)
if len(question_changes_list):
if questions_updates_key in questions_updates:
questions_updates[
questions_updates_key] += question_changes_list
else:
questions_updates[
questions_updates_key] = question_changes_list
questions_updated.add(db_question.id)
# update tags
question_tag_changes_string = self.update_question_tags(
db_question, notion_question_tag_name_list)
if question_tag_changes_string:
if questions_updates_key in questions_updates:
questions_updates[
questions_updates_key].append(
question_tag_changes_string)
else:
questions_updates[questions_updates_key] = [
question_tag_changes_string
]
questions_updated.add(db_question.id)
except ValidationError as e:
validation_errors.append(
f"Question {notion_question_dict['id']}: {e}")
except IntegrityError as e:
validation_errors.append(
f"Question {notion_question_dict['id']}: {e}")
print("--- Step 3 done : loop on questions : %s seconds ---" %
round(time.time() - start_time, 1))
#########################################################
# Build and send stats
#########################################################
start_time = time.time()
# done
questions_notion_count = (
f"Nombre de questions dans Notion : {len(notion_questions_list)}")
questions_scope_count = f"Nombre de questions prises en compte ici : {len(notion_questions_list_scope)}" # noqa
questions_scope_count += f" (de id {notion_questions_list_scope[0].get_property('id')} à id {notion_questions_list_scope[-1].get_property('id')})" # noqa
questions_ids_duplicate_message = f"ids 'en double' : {', '.join([str(question_id) for question_id in questions_ids_duplicate])}" # noqa
questions_ids_missing_message = f"ids 'manquants' : {', '.join([str(question_id) for question_id in questions_ids_missing])}" # noqa
tags_created_message = f"Nombre de tags ajoutés : {len(tags_created)}"
if len(tags_created):
tags_created_message += (
"\n" +
f"Détails : {', '.join([str(tag_name) for tag_name in tags_created])}"
)
questions_created_message = (
f"Nombre de questions ajoutées : {len(questions_created)}")
if len(questions_created):
questions_created_message += (
"\n" +
f"Détails : {', '.join([str(question_id) for question_id in questions_created])}"
)
questions_updated_message = (
f"Nombre de questions modifiées : {len(questions_updated)}")
if len(questions_updates):
questions_updated_message += "\n" + "\n".join([
key + "///" + "///".join(questions_updates[key])
for key in questions_updates
])
# check if any published quiz have non-validated questions
published_quizs = Quiz.objects.prefetch_related(
"questions").published()
for pq in published_quizs:
pq_not_validated_questions = pq.questions_not_validated_list
for question in pq_not_validated_questions:
validation_errors.append(
f"Quiz {pq.id}: Question {question.id} is not validated but quiz is published"
)
validation_errors_message = "Erreurs : "
if len(validation_errors):
validation_errors_message += (
f"Erreurs : {len(validation_errors)}" + "\n" +
"\n".join([str(error) for error in validation_errors]))
Contribution.objects.create(
text="Erreur(s) lors de l'import",
description=validation_errors_message,
type="erreur application",
)
# if not settings.DEBUG and scope == 0:
# utilities_notion.add_import_stats_row(
# len(notion_questions_list),
# len(questions_created),
# len(questions_updated),
# )
print("--- Step 4 done : build and send stats : %s seconds ---" %
round(time.time() - start_time, 1))
#########################################################
# Update configuration
#########################################################
configuration = Configuration.get_solo()
if scope:
setattr(
configuration,
f"notion_questions_scope_{scope}_last_imported",
timezone.now(),
)
else:
for scope in constants.NOTION_QUESTIONS_IMPORT_SCOPE_LIST[1:]:
setattr(
configuration,
f"notion_questions_scope_{scope}_last_imported",
timezone.now(),
)
configuration.save()
self.stdout.write("\n".join([
">>> Info sur les questions",
questions_notion_count,
questions_scope_count,
questions_ids_duplicate_message,
questions_ids_missing_message,
"",
">>> Info sur les tags ajoutés",
tags_created_message,
"",
">>> Info sur les questions ajoutées",
questions_created_message,
"",
">>> Info sur les questions modifiées",
questions_updated_message,
"",
">>> Erreurs lors de l'import",
validation_errors_message,
]))
from django.utils import timezone
from django.core.management import BaseCommand
from core.models import Configuration
from stats.models import (
# QuestionAggStat,
QuestionAnswerEvent,
QuestionFeedbackEvent,
QuizAnswerEvent,
# QuizFeedbackEvent,
DailyStat,
)
from api.models import Question
configuration = Configuration.get_solo()
class Command(BaseCommand):
"""
Usage:
python manage.py generate_daily_stats
Daily stats
- total number of answers
- total number of answers from questions
- total number of answers from quizs
- total number of quizs played
- total number of feedbacks (like/dislike)
answers per hour ?