def save_learning_result(xapi_statement: XapiStatement) -> LearningResult:
"""Creates a new Learning Result from a XAPI Statement and saves it to the database."""
learning_result: LearningResult = LearningResultMC.from_dict(
actor_account_name=xapi_statement.actor.account.name,
actor_object_type=xapi_statement.actor.object_type.value,
category_id=xapi_statement.context.context_activities.category[0].id,
category_object_type=xapi_statement.context.context_activities.
category[0].object_type.value,
choices=json.dumps(
[x.serialize() for x in xapi_statement.object.definition.choices]),
completion=xapi_statement.result.completion,
correct_responses_pattern=json.dumps(
xapi_statement.object.definition.correct_responses_pattern),
created_time=datetime.utcnow().timestamp(),
duration=xapi_statement.result.duration,
extensions=json.dumps(xapi_statement.object.definition.extensions),
interaction_type=xapi_statement.object.definition.interaction_type,
object_definition_description=xapi_statement.object.definition.
description.en_us,
object_definition_type=xapi_statement.object.definition.type,
object_object_type=xapi_statement.object.object_type.value,
response=xapi_statement.result.response,
score_max=xapi_statement.result.score.max,
score_min=xapi_statement.result.score.min,
score_raw=xapi_statement.result.score.raw,
score_scaled=xapi_statement.result.score.scaled,
success=xapi_statement.result.success,
verb_id=xapi_statement.verb.id,
verb_display=xapi_statement.verb.display.en_us)
db.session.add(learning_result)
DatabaseService.commit()
return learning_result
def init_app_common(cfg: Type[Config] = Config, is_csm: bool = False) -> Flask:
""" Initializes common Flask parts, e.g. CORS, configuration, database, migrations and custom corpora."""
spec_dir: str = Config.CSM_DIRECTORY if is_csm else Config.MC_SERVER_DIRECTORY
connexion_app: FlaskApp = connexion.FlaskApp(
__name__,
port=(cfg.CORPUS_STORAGE_MANAGER_PORT if is_csm else cfg.HOST_PORT),
specification_dir=spec_dir)
spec_path: str = Config.API_SPEC_CSM_FILE_PATH if is_csm else Config.API_SPEC_MCSERVER_FILE_PATH
parser = prance.ResolvingParser(
spec_path, lazy=True, strict=False) # str(Path(spec_path).absolute())
parser.parse()
connexion_app.add_api(parser.specification)
apply_event_handlers(connexion_app)
app: Flask = connexion_app.app
# allow CORS requests for all API routes
CORS(app) # , resources=r"/*"
app.config.from_object(cfg)
app.app_context().push()
db.init_app(app)
migrate.init_app(app, db)
if is_csm or cfg.TESTING:
db.create_all()
if is_csm:
from mcserver.app.services.databaseService import DatabaseService
DatabaseService.init_db_alembic()
from mcserver.app.services.textService import TextService
TextService.init_proper_nouns_list()
TextService.init_stop_words_latin()
if is_csm:
full_init(app, cfg)
return app
def get(
id: str, type: FileType, solution_indices: List[int]
) -> Union[ETagResponseMixin, ConnexionResponse]:
"""The GET method for the file REST API. It provides the URL to download a specific file."""
clean_tmp_folder()
exercise: Exercise = DatabaseService.query(Exercise,
filter_by=dict(eid=id),
first=True)
file_name: str = id + "." + str(type)
mime_type: str = MimeType[type].value
if exercise is None:
# try and see if a file is already cached on disk
if not os.path.exists(os.path.join(Config.TMP_DIRECTORY, file_name)):
return connexion.problem(404, Config.ERROR_TITLE_NOT_FOUND,
Config.ERROR_MESSAGE_EXERCISE_NOT_FOUND)
return send_from_directory(Config.TMP_DIRECTORY,
file_name,
mimetype=mime_type,
as_attachment=True)
exercise.last_access_time = datetime.utcnow().timestamp()
DatabaseService.commit()
if solution_indices:
file_name = id + "-" + str(uuid.uuid4()) + "." + str(type)
existing_file: DownloadableFile = next(
(x for x in FileService.downloadable_files
if x.id + "." + str(x.file_type) == file_name), None)
if existing_file is None:
existing_file = FileService.make_tmp_file_from_exercise(
type, exercise, solution_indices)
return send_from_directory(Config.TMP_DIRECTORY,
existing_file.file_name,
mimetype=mime_type,
as_attachment=True)
def get(last_update_time: int) -> Union[Response, ConnexionResponse]:
"""The GET method for the corpus list REST API. It provides metadata for all available texts."""
ui_cts: UpdateInfo = DatabaseService.query(
UpdateInfo, filter_by=dict(resource_type=ResourceType.cts_data.name), first=True)
if ui_cts and ui_cts.last_modified_time >= last_update_time / 1000:
corpora: List[Corpus] = DatabaseService.query(Corpus)
return NetworkService.make_json_response([x.to_dict() for x in corpora])
return NetworkService.make_json_response(None)
def delete(cid: int) -> Union[Response, ConnexionResponse]:
"""The DELETE method for the corpus REST API. It deletes metadata for a specific text."""
corpus: Corpus = DatabaseService.query(Corpus,
filter_by=dict(cid=cid),
first=True)
if not corpus:
return connexion.problem(404, Config.ERROR_TITLE_NOT_FOUND,
Config.ERROR_MESSAGE_CORPUS_NOT_FOUND)
db.session.delete(corpus)
DatabaseService.commit()
return NetworkService.make_json_response(True)
def full_init(app: Flask, cfg: Type[Config] = Config) -> None:
""" Fully initializes the application, including logging."""
from mcserver.app.services import DatabaseService
DatabaseService.init_db_update_info()
from mcserver.app.services.corpusService import CorpusService
CorpusService.init_corpora()
from mcserver.app.services import ExerciseService
ExerciseService.update_exercises(is_csm=True)
if not cfg.TESTING:
CorpusService.init_graphannis_logging()
start_updater(app)
def patch(cid: int, **kwargs) -> Union[Response, ConnexionResponse]:
"""The PUT method for the corpus REST API. It provides updates metadata for a specific text."""
corpus: Corpus = DatabaseService.query(Corpus,
filter_by=dict(cid=cid),
first=True)
if not corpus:
return connexion.problem(404, Config.ERROR_TITLE_NOT_FOUND,
Config.ERROR_MESSAGE_CORPUS_NOT_FOUND)
for k, v in kwargs.items():
if v is not None:
setattr(corpus, k, v)
DatabaseService.commit()
return NetworkService.make_json_response(corpus.to_dict())
def post(h5p_data: dict):
""" The POST method for the H5P REST API. It offers client-side H5P exercises for download as ZIP archives. """
h5p_form: H5PForm = H5PForm.from_dict(h5p_data)
language: Language = determine_language(h5p_form.lang)
exercise: Exercise = DatabaseService.query(
Exercise, filter_by=dict(eid=h5p_form.eid), first=True)
if not exercise:
return connexion.problem(404, Config.ERROR_TITLE_NOT_FOUND,
Config.ERROR_MESSAGE_EXERCISE_NOT_FOUND)
text_field_content: str = get_text_field_content(exercise,
h5p_form.solution_indices)
if not text_field_content:
return connexion.problem(422, Config.ERROR_TITLE_UNPROCESSABLE_ENTITY,
Config.ERROR_MESSAGE_UNPROCESSABLE_ENTITY)
response_dict: dict = TextService.json_template_mark_words
response_dict = get_response(response_dict, language,
TextService.json_template_drag_text, exercise,
text_field_content,
TextService.feedback_template)
file_name_no_ext: str = str(h5p_form.exercise_type_path)
file_name: str = f"{file_name_no_ext}.{FileType.ZIP}"
target_dir: str = Config.TMP_DIRECTORY
make_h5p_archive(file_name_no_ext, response_dict, target_dir, file_name)
return send_from_directory(target_dir,
file_name,
mimetype=MimeType.zip.value,
as_attachment=True)
def get(cid: int) -> Union[Response, ConnexionResponse]:
"""The GET method for the corpus REST API. It provides metadata for a specific text."""
corpus: Corpus = DatabaseService.query(Corpus,
filter_by=dict(cid=cid),
first=True)
if not corpus:
return connexion.problem(404, Config.ERROR_TITLE_NOT_FOUND,
Config.ERROR_MESSAGE_CORPUS_NOT_FOUND)
return NetworkService.make_json_response(corpus.to_dict())
def map_exercise_data_to_database(
exercise_data: ExerciseData, exercise_type: str, instructions: str,
xml_guid: str, correct_feedback: str, partially_correct_feedback: str,
incorrect_feedback: str, general_feedback: str,
exercise_type_translation: str, search_values: str,
solutions: List[Solution], conll: str, work_author: str,
work_title: str, urn: str, language: str):
"""Maps the exercise data so we can save it to the database."""
# sort the nodes according to the ordering links
AnnotationService.sort_nodes(graph_data=exercise_data.graph)
# add content to solutions
solutions: List[Solution] = adjust_solutions(exercise_data=exercise_data,
solutions=solutions,
exercise_type=exercise_type)
quiz_solutions: str = json.dumps([x.to_dict() for x in solutions])
tc: TextComplexity = TextComplexityService.text_complexity(
TextComplexityMeasure.all.name, urn, False, exercise_data.graph)
new_exercise: Exercise = ExerciseMC.from_dict(
conll=conll,
correct_feedback=correct_feedback,
eid=xml_guid,
exercise_type=exercise_type,
exercise_type_translation=exercise_type_translation,
general_feedback=general_feedback,
incorrect_feedback=incorrect_feedback,
instructions=instructions,
language=language,
last_access_time=datetime.utcnow().timestamp(),
partially_correct_feedback=partially_correct_feedback,
search_values=search_values,
solutions=quiz_solutions,
text_complexity=tc.all,
work_author=work_author,
work_title=work_title,
urn=urn)
# add the mapped exercise to the database
db.session.add(new_exercise)
ui_exercises: UpdateInfo = DatabaseService.query(
UpdateInfo,
filter_by=dict(resource_type=ResourceType.exercise_list.name),
first=True)
ui_exercises.last_modified_time = datetime.utcnow().timestamp()
DatabaseService.commit()
return new_exercise
def get(lang: str,
frequency_upper_bound: int,
last_update_time: int,
vocabulary: str = ""):
"""The GET method for the exercise list REST API. It provides metadata for all available exercises."""
vocabulary_set: Set[str]
ui_exercises: UpdateInfo = DatabaseService.query(
UpdateInfo,
filter_by=dict(resource_type=ResourceType.exercise_list.name),
first=True)
if ui_exercises.last_modified_time < last_update_time / 1000:
return NetworkService.make_json_response([])
try:
vc: VocabularyCorpus = VocabularyCorpus[vocabulary]
vocabulary_set = FileService.get_vocabulary_set(
vc, frequency_upper_bound)
except KeyError:
vocabulary_set = set()
lang: Language
try:
lang = Language(lang)
except ValueError:
lang = Language.English
exercises: List[Exercise] = DatabaseService.query(
Exercise, filter_by=dict(language=lang.value))
matching_exercises: List[MatchingExercise] = [
MatchingExercise.from_dict(x.to_dict()) for x in exercises
]
if len(vocabulary_set):
for exercise in matching_exercises:
conll: List[TokenList] = conllu.parse(exercise.conll)
lemmata: List[str] = [
tok["lemma"] for sent in conll for tok in sent.tokens
]
exercise.matching_degree = sum(
(1 if x in vocabulary_set else 0)
for x in lemmata) / len(lemmata) * 100
ret_val: List[dict] = [
NetworkService.serialize_exercise(x, compress=True)
for x in matching_exercises
]
return NetworkService.make_json_response(ret_val)
def get(eid: str) -> Union[Response, ConnexionResponse]:
exercise: TExercise = DatabaseService.query(Exercise,
filter_by=dict(eid=eid),
first=True)
if not exercise:
return connexion.problem(404, Config.ERROR_TITLE_NOT_FOUND,
Config.ERROR_MESSAGE_EXERCISE_NOT_FOUND)
ar: AnnisResponse = CorpusService.get_corpus(cts_urn=exercise.urn,
is_csm=False)
if not ar.graph_data.nodes:
return connexion.problem(404, Config.ERROR_TITLE_NOT_FOUND,
Config.ERROR_MESSAGE_CORPUS_NOT_FOUND)
exercise.last_access_time = datetime.utcnow().timestamp()
DatabaseService.commit()
exercise_type: ExerciseType = ExerciseType(exercise.exercise_type)
ar.solutions = json.loads(exercise.solutions)
ar.uri = NetworkService.get_exercise_uri(exercise)
ar.exercise_id = exercise.eid
ar.exercise_type = exercise_type.value
return NetworkService.make_json_response(ar.to_dict())
def clean_tmp_folder():
""" Cleans the files directory regularly. """
ui_file: UpdateInfo = DatabaseService.query(
UpdateInfo,
filter_by=dict(resource_type=ResourceType.file_api_clean.name),
first=True)
ui_datetime: datetime = datetime.fromtimestamp(ui_file.last_modified_time)
if (datetime.utcnow() -
ui_datetime).total_seconds() > Config.INTERVAL_FILE_DELETE:
for file in [
x for x in os.listdir(Config.TMP_DIRECTORY)
if x not in ".gitignore"
]:
file_to_delete_type: str = os.path.splitext(file)[1].replace(
".", "")
file_to_delete: DownloadableFile = next((
x for x in FileService.downloadable_files
if x.file_name == file and x.file_type == file_to_delete_type),
None)
if file_to_delete is not None:
FileService.downloadable_files.remove(file_to_delete)
os.remove(os.path.join(Config.TMP_DIRECTORY, file))
ui_file.last_modified_time = datetime.utcnow().timestamp()
DatabaseService.commit()
def get(eid: str, lang: str,
solution_indices: List[int]) -> Union[Response, ConnexionResponse]:
""" The GET method for the H5P REST API. It provides JSON templates for client-side H5P exercise layouts. """
language: Language = determine_language(lang)
exercise: Exercise = DatabaseService.query(Exercise,
filter_by=dict(eid=eid),
first=True)
if not exercise:
return connexion.problem(404, Config.ERROR_TITLE_NOT_FOUND,
Config.ERROR_MESSAGE_EXERCISE_NOT_FOUND)
text_field_content: str = get_text_field_content(exercise,
solution_indices)
if not text_field_content:
return connexion.problem(422, Config.ERROR_TITLE_UNPROCESSABLE_ENTITY,
Config.ERROR_MESSAGE_UNPROCESSABLE_ENTITY)
response_dict: dict = TextService.json_template_mark_words
response_dict = get_response(response_dict, language,
TextService.json_template_drag_text, exercise,
text_field_content,
TextService.feedback_template)
return NetworkService.make_json_response(response_dict)
def update_exercises(is_csm: bool) -> None:
"""Deletes old exercises."""
if DatabaseService.has_table(Config.DATABASE_TABLE_EXERCISE):
exercises: List[Exercise] = DatabaseService.query(Exercise)
now: datetime = datetime.utcnow()
for exercise in exercises:
exercise_datetime: datetime = datetime.fromtimestamp(
exercise.last_access_time)
# delete exercises that have not been accessed for a while, are not compatible anymore, or contain
# corrupted / empty data
if (now - exercise_datetime).total_seconds() > Config.INTERVAL_EXERCISE_DELETE or \
not exercise.urn or not json.loads(exercise.solutions):
db.session.delete(exercise)
DatabaseService.commit()
# manually add text complexity measures for old exercises
elif not exercise.text_complexity:
ar: AnnisResponse = CorpusService.get_corpus(exercise.urn,
is_csm=is_csm)
tc: TextComplexity = TextComplexityService.text_complexity(
TextComplexityMeasure.all.name, exercise.urn, is_csm,
ar.graph_data)
exercise.text_complexity = tc.all
DatabaseService.commit()