def calculate_idr_average(param):
"""
A function to get the average idr of
the items:
It gets a list of each item's idr, and
then calculates the average.
:param: a json in the Reliabilty Measures
standard json format
:return: a float: the average idr
"""
service_key = get_service_config(11)
catch_error = get_error(param)
if catch_error[0]:
return {service_key: catch_error[1]}
inp = update_input(param)
idr_dict = list(calculate_idr(inp).values())[0]
if idr_dict == get_keyword_value("bad_mean"):
return {service_key: get_keyword_value("bad_mean")}
idr_list = list(list(idr_dict.values()))
num_items = len(idr_list)
idr_avg = sum(idr_list) / num_items
idr_avg = round(idr_avg, 3)
return {service_key: idr_avg}
def calculate_topic_averages(param):
"""
A function to calculate the average number of
correct responses in every topic:
It gets the total number of right responses
per topic and then divides them by the total
number of students.
:param: a json in the Reliability Measures
standard json format
:return: a list of dictionaries, a list of
each topic and its average number
of right responses
"""
service_key = get_service_config(16)
catch_error = get_error(param)
if catch_error[0]:
return {service_key: catch_error[1]}
inp = update_input(param)
topic_avgs = get_item_topics(inp)
topic_responses = calculate_topic_rights(inp)[get_service_config(15)]
num_topics = len(topic_avgs)
num_students = len(topic_responses)
check_topics = get_item_topics(inp)
if check_topics == get_keyword_value("no_topics"):
return {service_key: get_keyword_value("no_topics")}
for i in range(0, num_topics):
avg_rights = 0
for k in topic_responses:
rights = topic_responses[k][i][get_keyword_value("topic_rights")]
avg_rights += rights
avg_rights /= num_students
avg_rights = round(avg_rights, 3)
topic_avgs[i][get_keyword_value("topic_rights")] = avg_rights
for i in range(0, num_topics):
del topic_avgs[i][get_keyword_value("topic_ids")]
return {service_key: topic_avgs}
def test_get_config(self):
expected = "Reliability Measures"
assert get_config("application_org") == expected
# read the secret
expected = {
"db_host": "",
"db_user": "",
"db_password": "",
"db_name": "DB"
}
assert get_config("db_provider") == expected
expected = "kr20"
assert get_service_config(1) == expected
expected = "Item discrimination"
assert get_service_config(2, "name") == expected
# not found
expected = None
assert get_service_config(5, "title") == expected
def get_assumptions(param):
"""
A function to get the items for which a
student was assumed to have a response
of 0:
It gets a list of all item ids listed from
every student's responses, then iterates
through every student. If a student doesn't
have a response for an item in the id list,
then that item is assumed to have a response
of 0 for that student.
:param: a json in the Reliabilty Measures
standard json format
:return: a dictionary of dictionaries:
a dictionary with student ids as keys
and a list of item ids as values
"""
service_key = get_service_config(13)
catch_error = get_error(param)
if catch_error[0]:
return {service_key: catch_error[1]}
inp = update_input(param)
student_list = get_student_list(inp)
id_list = get_item_ids(inp)
assumptions_dict = {}
for i in student_list: # For each student i
checklist = id_list.copy()
dupes = []
for k in i[get_keyword_value("item_responses")]: # For each question k
for j in id_list: # For each item ID j
if k[get_keyword_value("item_id")] == j: # If item IDs match
if j in checklist:
checklist.remove(j)
else:
dupes.append(j)
if dupes:
assumptions_dict[i[get_keyword_value("id")]] = {}
assumptions_dict[i[get_keyword_value("id")]][get_keyword_value(
"dupes")] = dupes
if len(checklist) != 0:
assumptions_dict[i[get_keyword_value("id")]] = {}
assumptions_dict[i[get_keyword_value("id")]][get_keyword_value(
"assumed")] = checklist.copy()
if not assumptions_dict:
return {service_key: get_keyword_value("no_assumptions")}
return {service_key: assumptions_dict}
def calculate_weighted_scores(param):
"""
A function to get the weighted score of each student:
For each student, it gets the weight of every item
they got correct by getting its difficulty and dividing
it by the sum of all items' difficulties.
:param: a json in the Reliabilty Measures
standard json format
:return: a dictionary of floats: a dictionary with
student ids as keys and their weighted score
as values
"""
service_key = get_service_config(7)
catch_error = get_error(param)
if catch_error[0]:
return {service_key: catch_error[1]}
inp = update_input(param)
student_ids = get_student_ids(inp)
sorted_resp = get_sorted_responses(inp)
scoring_method = get_scoring_method(inp)
num_items = len(sorted_resp[0])
difficulty_list = list(
list(calculate_difficulty(inp).values())[0].values())
difficulty_sum = sum(difficulty_list)
weighted_scores_dict = {}
for curr_id in student_ids:
weighted_scores_dict[curr_id] = None
j = 0
for i in weighted_scores_dict:
weighted = 0
for k in range(0, num_items):
if sorted_resp[j][k] == 1:
weighted += difficulty_list[k]
weighted /= difficulty_sum
if scoring_method[0] == get_keyword_value("percentage"):
weighted = round(weighted * 100, 3)
elif scoring_method[0] == get_keyword_value("absolute"):
weighted = round(weighted * num_items, 3)
elif scoring_method[0] == get_keyword_value("scaled"):
weighted = round(weighted * scoring_method[1], 3)
else:
weighted = round(weighted, 3)
weighted_scores_dict[i] = weighted
j += 1
return {service_key: weighted_scores_dict}
def calculate_topic_rights(param):
"""
A function to calculate the number of
correct responses in every topic for each
student:
It gets a list of every topic with its
corresponding item id. If a student gets
that item correct, then the number of right
responses for that topic increases.
:param: a json in the Reliability Measures
standard json format
:return: a dictionary of dictionaries:
a dictionary with student ids as keys
and a list of topics and their number
of right responses
"""
service_key = get_service_config(15)
catch_error = get_error(param)
if catch_error[0]:
return {service_key: catch_error[1]}
inp = update_input(param)
student_list = get_student_list(inp)
check_topics = get_item_topics(inp)
topic_rights = {}
if check_topics == get_keyword_value("no_topics"):
return {service_key: get_keyword_value("no_topics")}
for i in student_list:
topic_trees = get_item_topics(inp)
stud_id = i[get_keyword_value("id")]
responses = i[get_keyword_value("item_responses")]
for k in responses:
item_id = k[get_keyword_value("item_id")]
item_resp = k[get_keyword_value("response")]
for j in range(0, len(topic_trees)):
topic_ids = topic_trees[j][get_keyword_value("topic_ids")]
if item_resp == 1 and item_id in topic_ids:
topic_trees[j][get_keyword_value("topic_rights")] += 1
for k in range(0, len(topic_trees)):
del topic_trees[k][get_keyword_value("topic_ids")]
topic_rights[stud_id] = topic_trees
return {service_key: topic_rights}
def analyze_test(param):
"""
A function to get an exam's analysis:
It calls every service used to analyze
an exam and then returns the analysis.
:param: a json in the Reliabilty Measures
standard json format
:return: a dictionary of dictionaries:
a dictionary with the results
of the services as values
"""
service_key = get_service_config(6)
catch_error = get_error(param)
if catch_error[0]:
return {service_key: catch_error[1]}
inp = update_input(param)
# use microservice calls here when all are hosted
val_kr20 = calculate_kr20(inp)
val_idr = calculate_idr(inp)
val_difficulty = calculate_difficulty(inp)
val_scores = calculate_scores(inp)
val_average = calculate_average(inp)
val_weighted_s = calculate_weighted_scores(inp)
val_weighted_avg = calculate_weighted_average(inp)
val_excludes = get_exclude_recos(inp)
val_diff_avg = calculate_difficulty_average(inp)
val_idr_avg = calculate_idr_average(inp)
val_num_correct = calculate_num_correct(inp)
val_assumptions = get_assumptions(inp)
val_topic_rights = calculate_topic_rights(inp)
val_topic_avgs = calculate_topic_averages(inp)
val_group_analysis = analyze_groups(inp)
# join all results
result = {}
items = [
val_kr20, val_idr, val_difficulty, val_scores, val_average,
val_weighted_s, val_weighted_avg, val_excludes, val_diff_avg,
val_idr_avg, val_num_correct, val_assumptions, val_topic_rights,
val_group_analysis, val_topic_avgs
]
for item in items:
result.update(item)
return {service_key: result}
def calculate_scores(param):
"""
A function to get the score of each student:
For each student, it gets the number of correct
responses and divides it by the number of questions.
:param: a json in the Reliability Measures
standard json format
:return: a dictionary of floats: a dictionary with
student ids as keys and their score as values
"""
service_key = get_service_config(4)
catch_error = get_error(param)
if catch_error[0]:
return {service_key: catch_error[1]}
inp = update_input(param)
sorted_resp = get_sorted_responses(inp)
student_ids = get_student_ids(inp)
scoring_method = get_scoring_method(inp)
num_items = len(sorted_resp[0])
score_dict = {}
for curr_id in student_ids:
score_dict[curr_id] = None
k = 0
for i in score_dict:
num_right = sum(sorted_resp[k])
score = num_right / num_items
if scoring_method[0] == get_keyword_value("percentage"):
score = round(score * 100, 3)
elif scoring_method[0] == get_keyword_value("absolute"):
score = round(score * num_items, 3)
elif scoring_method[0] == get_keyword_value("scaled"):
score = round(score * scoring_method[1], 3)
else:
score = round(score, 3)
score_dict[i] = score
k += 1
return {service_key: score_dict}
def calculate_difficulty(param):
"""
A function to get the difficulty of
each item on the exam:
It calculates how many students got an
item correct, and then divides it by
the total number of students.
:param: a json in the Reliabilty Measures
standard json format
:return: a dictionary of floats: a dictionary
with item ids as keys and the
difficulty as values
"""
service_key = get_service_config(3)
catch_error = get_error(param)
if catch_error[0]:
return {service_key: catch_error[1]}
inp = update_input(param)
sorted_resp = get_sorted_responses(inp)
num_students = len(sorted_resp)
num_items = len(sorted_resp[0])
id_list = get_item_ids(inp)
difficulty_list = []
difficulty_dict = {}
for i in range(0, num_items): # For each question i
numRight = 0
for k in range(0, num_students): # For each student k
studentAnswer = sorted_resp[k][i]
numRight += studentAnswer
difficulty = 1 - numRight / num_students
difficulty = round(difficulty, 3)
difficulty_list.append(difficulty)
k = 0
for i in id_list:
difficulty_dict[i] = difficulty_list[k]
k += 1
return {service_key: difficulty_dict}
def calculate_kr20(param):
"""
A function to get the kr20 value of an exam:
First it get the number of items divided by the
number of items - 1. Then it multiplies that by
1 - the summation of the product of the
proportion of those who got an item right by the
proportion of those who got it wrong divided by the
variance of the students' scores.
:param: a json in the Reliabilty Measures
standard json format
:return: a float: the kr20
"""
service_key = get_service_config(1)
catch_error = get_error(param)
if catch_error[0]:
return {service_key: catch_error[1]}
inp = update_input(param)
sorted_resp = get_sorted_responses(inp)
num_students = len(sorted_resp)
num_items = len (sorted_resp[0])
pq_list = []
score_std = get_score_std(inp)
if score_std <= 0:
return {service_key: get_keyword_value("bad_std")}
for i in range(0, num_items):
p = 0
for k in range(0, num_students):
p += sorted_resp[k][i]
p /= num_students
q = 1 - p
pq_list.append(p * q)
pq_sum = sum(pq_list)
kr20_value = (num_items /(num_items - 1)) * (1 - (pq_sum / (score_std ** 2)))
kr20_value = round(kr20_value, 3)
return {service_key: kr20_value}
def calculate_difficulty_average(param):
"""
A function to get the average difficulty
of items on the exam:
It gets the difficulty of every item and then
calculates the average of those difficulties.
:param: a json in the Reliabilty Measures
standard json format
:return: a float: the difficulty average
"""
service_key = get_service_config(10)
catch_error = get_error(param)
if catch_error[0]:
return {service_key: catch_error[1]}
inp = update_input(param)
diff_list = list(list(calculate_difficulty(inp).values())[0].values())
num_items = len(diff_list)
diff_avg = sum(diff_list) / num_items
diff_avg = round(diff_avg, 3)
return {service_key: diff_avg}
def get_exclude_recos(param):
"""
A function to get a recommendation of
items to exclude from the exam based on
their idr values:
It get every item's idr, and if it's less than
0.09, it adds it to the exclude recommendations.
If the number of recommendations is greater
than half the number of items, only recommend
items with idr values less than 0.
:param: a json in the Reliabilty Measures
standard json format
:return: a list of strings: a list of item ids
"""
service_key = get_service_config(9)
catch_error = get_error(param)
if catch_error[0]:
return {service_key: catch_error[1]}
inp = update_input(param)
idr_dict = list(calculate_idr(inp).values())[0]
if idr_dict == get_keyword_value("bad_mean"):
return {service_key: get_keyword_value("bad_mean")}
exclude_list = []
for i in idr_dict:
if idr_dict[i] <= get_keyword_value("exclude_threshold_1"):
exclude_list.append(i)
if len(exclude_list) >= len(idr_dict)*get_keyword_value("exclude_length_1"):
exclude_list = []
for i in idr_dict:
if idr_dict[i] < get_keyword_value("exclude_threshold_2"):
exclude_list.append(i)
# if len(exclude_list) >= len(idr_dict)*get_keyword_value("exclude_length_2"):
# return {service_key: get_keyword_value("bad_exam")}
return {service_key: exclude_list}
def calculate_average(param):
"""
A function to get the average score of
the students:
It gets every students score and then
calculates the average of those scores.
:param: a json in the Reliability Measures
standard json format
:return: a float: the score average
"""
service_key = get_service_config(5)
catch_error = get_error(param)
if catch_error[0]:
return {service_key: catch_error[1]}
inp = update_input(param)
score_list = list(list(calculate_scores(inp).values())[0].values())
num_students = len(score_list)
average = sum(score_list) / num_students
average = round(average, 3)
return {service_key: average}
def test_kr20(self):
expected = -1.167
kr20 = calculate_kr20(self.data)[get_service_config(1)]
assert kr20 == expected
def analyze_groups(param):
"""
A function to get an exam's analysis by
students' group:
It groups all students by group and
then iterates over the groups, calling
every service used to analyze an exam.
:param: a json in the Reliabilty Measures
standard json format
:return: a dictionary of nested dictionaries:
a dictionary with groups as
keys and the exam analysis as values
"""
service_key = get_service_config(14)
catch_error = get_error(param)
if catch_error[0]:
return {service_key: catch_error[1]}
inp = update_input(param)
assumptions_key = get_service_config(13)
assumptions = get_assumptions(inp)[assumptions_key]
students_dict = sort_students_by_group(inp)
group_list = get_group_list(inp)
group_analysis = {}
if group_list == get_keyword_value("no_group"):
return {service_key: get_keyword_value("no_group")}
for i in students_dict:
curr_students = students_dict[i]
catch_error = get_error(curr_students)
if catch_error[0]:
group_analysis[i] = catch_error[1]
continue
student_list = get_student_list(curr_students)
val_kr20 = calculate_kr20(curr_students)
val_idr = calculate_idr(curr_students)
val_difficulty = calculate_difficulty(curr_students)
val_scores = calculate_scores(curr_students)
val_average = calculate_average(curr_students)
val_weighted_s = calculate_weighted_scores(curr_students)
val_weighted_avg = calculate_weighted_average(curr_students)
val_excludes = get_exclude_recos(curr_students)
val_diff_avg = calculate_difficulty_average(curr_students)
val_idr_avg = calculate_idr_average(curr_students)
val_num_correct = calculate_num_correct(curr_students)
val_topic_rights = calculate_topic_rights(curr_students)
val_topic_avgs = calculate_topic_averages(curr_students)
curr_assumptions = {}
for k in assumptions:
for j in student_list:
if k == j[get_keyword_value("id")]:
curr_assumptions[k] = assumptions[k]
val_assumptions = {assumptions_key: curr_assumptions}
result = {
'overall_quiz': {
'average': val_average['average'],
'kr20': val_kr20['kr20'],
'weighted_avg': val_weighted_avg['weighted_avg']
},
'overall_items': {
'diff_avg': val_diff_avg['diff_avg'],
'idr_avg': val_idr_avg['idr_avg']
},
'item_analysis': [],
'student_scores': []
}
for k in val_difficulty['difficulty']:
curr_idr = val_idr['idr']
if curr_idr is not str:
curr_idr = val_idr['idr'][k]
result['item_analysis'].append({
'item_id':
k,
'difficulty':
val_difficulty['difficulty'][k],
'idr':
curr_idr,
'num_correct':
val_num_correct['num_correct'][k]
})
for k in val_scores['scores']:
result['student_scores'].append({
'student':
k,
'score':
val_scores['scores'][k],
'weighted_score':
val_weighted_s['weighted_scores'][k]
})
items = [
val_excludes, val_assumptions, val_topic_rights, val_topic_avgs
]
for item in items:
result.update(item)
group_analysis[i] = result
return {service_key: group_analysis}
def calculate_idr(param):
"""
A function to get the idr of each item:
For every item, it calculates the mean score
of students who got the answer right and subtracts
it by the mean score of those who got it wrong.
Then it multiplies that by the square root of
the number of students who got the item right
multiplied by the total of those who got it wrong.
Then it divides that by the number of students
multiplied by the std of the students' scores.
:param: a json in the Reliabilty Measures
standard json format
:return: a dictionary of floats: a dictionary with
item ids as keys and idr as values
"""
service_key = get_service_config(2)
catch_error = get_error(param)
if catch_error[0]:
return {service_key: catch_error[1]}
inp = update_input(param)
sorted_resp = get_sorted_responses(inp)
num_students = len(sorted_resp)
num_items = len(sorted_resp[0])
id_list = get_item_ids(inp)
score_std = get_score_std(inp)
idr_list = []
idr_dict = {}
if score_std < 0:
return {service_key: get_keyword_value("bad_std")}
for i in range(0, num_items): # For each question i
right_list = []
wrong_list = []
num_right = 0
num_wrong = 0
for k in range(0, num_students): # For each student k
if sorted_resp[k][i] == 1: # If student k gets question i correct
score = sum(sorted_resp[k]) / num_items
right_list.append(
score) # Then add their score to the "right" list
num_right += 1
elif sorted_resp[k][i] == 0: # If student k gets question i wrong
score = sum(sorted_resp[k]) / num_items
wrong_list.append(
score) # Then add their score to the "wrong" list
num_wrong += 1
if num_right == num_students or num_wrong == num_students:
idr_list.append(0)
continue
if len(right_list) == 1:
right_mean = right_list[0]
elif len(right_list) > 1:
right_mean = mean(right_list)
if len(wrong_list) == 1:
wrong_mean = wrong_list[0]
elif len(wrong_list) > 1:
wrong_mean = mean(wrong_list)
if not right_mean or not wrong_mean:
return {service_key: get_keyword_value("bad_mean")}
idr = ((right_mean - wrong_mean) *
sqrt(num_right * num_wrong)) / num_students * score_std
idr = round(idr, 3)
idr_list.append(idr)
k = 0
for i in id_list:
idr_dict[i] = idr_list[k]
k += 1
return {service_key: idr_dict}
def analyze_test(param):
"""
A function to get an exam's analysis:
It calls every service used to analyze
an exam and then returns the analysis.
:param: a json in the Reliabilty Measures
standard json format
:return: a dictionary of dictionaries:
a dictionary with the results
of the services as values
"""
service_key = get_service_config(6)
catch_error = get_error(param)
if catch_error[0]:
return {service_key: catch_error[1]}
inp = update_input(param)
# use microservice calls here when all are hosted
val_kr20 = calculate_kr20(inp)
val_idr = calculate_idr(inp)
val_difficulty = calculate_difficulty(inp)
val_scores = calculate_scores(inp)
val_average = calculate_average(inp)
val_weighted_s = calculate_weighted_scores(inp)
val_weighted_avg = calculate_weighted_average(inp)
val_excludes = get_exclude_recos(inp)
val_diff_avg = calculate_difficulty_average(inp)
val_idr_avg = calculate_idr_average(inp)
val_num_correct = calculate_num_correct(inp)
val_assumptions = get_assumptions(inp)
val_topic_rights = calculate_topic_rights(inp)
val_topic_avgs = calculate_topic_averages(inp)
val_group_analysis = analyze_groups(inp)
# join all results
result = {
'overall_quiz': {
'average': val_average['average'],
'kr20': val_kr20['kr20'],
'weighted_avg': val_weighted_avg['weighted_avg']
},
'overall_items': {
'diff_avg': val_diff_avg['diff_avg'],
'idr_avg': val_idr_avg['idr_avg']
},
'item_analysis': [],
'student_scores': []
}
for i in val_difficulty['difficulty']:
curr_idr = val_idr['idr']
if type(curr_idr) is not str:
curr_idr = val_idr['idr'][i]
result['item_analysis'].append({
'item_id':
i,
'difficulty':
val_difficulty['difficulty'][i],
'idr':
curr_idr,
'num_correct':
val_num_correct['num_correct'][i]
})
for i in val_scores['scores']:
result['student_scores'].append({
'student':
i,
'score':
val_scores['scores'][i],
'weighted_score':
val_weighted_s['weighted_scores'][i]
})
items = [
val_excludes, val_assumptions, val_topic_rights, val_group_analysis,
val_topic_avgs
]
for item in items:
result.update(item)
return {service_key: result}
