def test_update_answer_criteria_scores(self):
criterion_comparison_results = {
1:
calculate_score(comparison_pairs=[
ComparisonPair(key1=1, key2=2, winner=ComparisonWinner.key1)
])
}
scores = update_answer_criteria_scores([], 1,
criterion_comparison_results)
self.assertEqual(len(scores), 2)
for score in scores:
self.assertIsNone(score.id)
score = AnswerCriterionScore(answer_id=1, criterion_id=1, id=2)
scores = update_answer_criteria_scores([score], 1,
criterion_comparison_results)
self.assertEqual(len(scores), 2)
self.assertEqual(scores[0].id, 2)
self.assertIsNone(scores[1].id)
criterion_comparison_results = {
1:
calculate_score(comparison_pairs=[
ComparisonPair(key1=1, key2=2, winner=ComparisonWinner.key1)
]),
2:
calculate_score(comparison_pairs=[
ComparisonPair(key1=1, key2=2, winner=ComparisonWinner.key1)
])
}
score = AnswerCriterionScore(answer_id=1, criterion_id=1, id=2)
scores = update_answer_criteria_scores([score], 1,
criterion_comparison_results)
self.assertEqual(len(scores), 4)
def test_update_answer_criteria_scores(self):
criterion_comparison_results = {
1: calculate_score(comparison_pairs=[
ComparisonPair(key1=1,key2=2, winner=ComparisonWinner.key1)
])
}
scores = update_answer_criteria_scores([], 1, criterion_comparison_results)
self.assertEqual(len(scores), 2)
for score in scores:
self.assertIsNone(score.id)
score = AnswerCriterionScore(answer_id=1, criterion_id=1, id=2)
scores = update_answer_criteria_scores([score], 1, criterion_comparison_results)
self.assertEqual(len(scores), 2)
self.assertEqual(scores[0].id, 2)
self.assertIsNone(scores[1].id)
criterion_comparison_results = {
1: calculate_score(comparison_pairs=[
ComparisonPair(key1=1,key2=2, winner=ComparisonWinner.key1)
]),
2: calculate_score(comparison_pairs=[
ComparisonPair(key1=1,key2=2, winner=ComparisonWinner.key1)
])
}
score = AnswerCriterionScore(answer_id=1, criterion_id=1, id=2)
scores = update_answer_criteria_scores([score], 1, criterion_comparison_results)
self.assertEqual(len(scores), 4)
def test_update_scores(self):
criterion_comparison_results = {
1: calculate_score(comparison_pairs=[
ComparisonPair(1,2, winning_key=1)
])
}
scores = update_scores([], 1, criterion_comparison_results)
self.assertEqual(len(scores), 2)
for score in scores:
self.assertIsNone(score.id)
score = Score(answer_id=1, criterion_id=1, id=2)
scores = update_scores([score], 1, criterion_comparison_results)
self.assertEqual(len(scores), 2)
self.assertEqual(scores[0].id, 2)
self.assertIsNone(scores[1].id)
criterion_comparison_results = {
1: calculate_score(comparison_pairs=[
ComparisonPair(1,2, winning_key=1)
]),
2: calculate_score(comparison_pairs=[
ComparisonPair(1,2, winning_key=1)
])
}
score = Score(answer_id=1, criterion_id=1, id=2)
scores = update_scores([score], 1, criterion_comparison_results)
self.assertEqual(len(scores), 4)
def test_update_answer_scores(self):
comparison_results = calculate_score(comparison_pairs=[
ComparisonPair(key1=1, key2=2, winner=ComparisonWinner.key1)
])
scores = update_answer_scores([], 1, comparison_results)
self.assertEqual(len(scores), 2)
for score in scores:
self.assertIsNone(score.id)
score = AnswerScore(answer_id=1, id=2)
scores = update_answer_scores([score], 1, comparison_results)
self.assertEqual(len(scores), 2)
self.assertEqual(scores[0].id, 2)
self.assertIsNone(scores[1].id)
comparison_results = calculate_score(comparison_pairs=[
ComparisonPair(key1=1, key2=2, winner=ComparisonWinner.key1),
ComparisonPair(key1=3, key2=4, winner=ComparisonWinner.key1)
])
score = AnswerScore(answer_id=1, id=2)
scores = update_answer_scores([score], 1, comparison_results)
self.assertEqual(len(scores), 4)
def test_update_answer_scores(self):
comparison_results = calculate_score(comparison_pairs=[
ComparisonPair(key1=1,key2=2, winner=ComparisonWinner.key1)
])
scores = update_answer_scores([], 1, comparison_results)
self.assertEqual(len(scores), 2)
for score in scores:
self.assertIsNone(score.id)
score = AnswerScore(answer_id=1, id=2)
scores = update_answer_scores([score], 1, comparison_results)
self.assertEqual(len(scores), 2)
self.assertEqual(scores[0].id, 2)
self.assertIsNone(scores[1].id)
comparison_results = calculate_score(comparison_pairs=[
ComparisonPair(key1=1,key2=2, winner=ComparisonWinner.key1),
ComparisonPair(key1=3,key2=4, winner=ComparisonWinner.key1)
])
score = AnswerScore(answer_id=1, id=2)
scores = update_answer_scores([score], 1, comparison_results)
self.assertEqual(len(scores), 4)
def calculate_scores(cls, assignment_id):
from . import AnswerScore, AnswerCriterionScore, \
AssignmentCriterion, ScoringAlgorithm
# get all comparisons for this assignment and only load the data we need
comparisons = Comparison.query \
.filter(Comparison.assignment_id == assignment_id) \
.all()
assignment_criteria = AssignmentCriterion.query \
.with_entities(AssignmentCriterion.criterion_id) \
.filter_by(assignment_id=assignment_id, active=True) \
.all()
comparison_criteria = []
comparison_pairs = []
answer_ids = set()
for comparison in comparisons:
answer_ids.add(comparison.answer1_id)
answer_ids.add(comparison.answer2_id)
comparison_criteria.extend(comparison.comparison_criteria)
comparison_pairs.append(comparison.convert_to_comparison_pair())
# calculate answer score
comparison_results = calculate_score(
package_name=ScoringAlgorithm.elo.value,
comparison_pairs=comparison_pairs,
log=current_app.logger)
scores = AnswerScore.query \
.filter(AnswerScore.answer_id.in_(answer_ids)) \
.all()
updated_answer_scores = update_answer_scores(scores, assignment_id,
comparison_results)
db.session.add_all(updated_answer_scores)
# calculate answer criterion scores
criterion_comparison_results = {}
for assignment_criterion in assignment_criteria:
comparison_pairs = []
for comparison_criterion in comparison_criteria:
if comparison_criterion.criterion_id != assignment_criterion.criterion_id:
continue
comparison_pairs.append(
comparison_criterion.convert_to_comparison_pair())
criterion_comparison_results[
assignment_criterion.criterion_id] = calculate_score(
package_name=ScoringAlgorithm.elo.value,
comparison_pairs=comparison_pairs,
log=current_app.logger)
scores = AnswerCriterionScore.query \
.filter(AnswerCriterionScore.answer_id.in_(answer_ids)) \
.all()
updated_answer_criteria_scores = update_answer_criteria_scores(
scores, assignment_id, criterion_comparison_results)
db.session.add_all(updated_answer_criteria_scores)
db.session.commit()
def test_calculate_score(self):
# test comparitive judgement score algorithm
self.package_name = "comparative_judgement"
results = calculate_score(
package_name=self.package_name,
comparison_pairs=self.comparisons
)
self.assertEqual(len(results.items()), 3)
self.assertIsInstance(results.get(1), ScoredObject)
self.assertIsInstance(results.get(1).score, float)
self.assertIsInstance(results.get(1).variable1, float)
self.assertIsNone(results.get(1).variable2)
self.assertEqual(results.get(1).rounds, 2)
self.assertEqual(results.get(1).opponents, 2)
self.assertEqual(results.get(1).wins, 2)
self.assertEqual(results.get(1).loses, 0)
self.assertIsInstance(results.get(2), ScoredObject)
self.assertIsInstance(results.get(2).score, float)
self.assertIsInstance(results.get(2).variable1, float)
self.assertIsNone(results.get(2).variable2)
self.assertEqual(results.get(2).rounds, 2)
self.assertEqual(results.get(2).opponents, 2)
self.assertEqual(results.get(2).wins, 1)
self.assertEqual(results.get(2).loses, 1)
self.assertIsInstance(results.get(3), ScoredObject)
self.assertIsInstance(results.get(3).score, float)
self.assertIsInstance(results.get(3).variable1, float)
self.assertIsNone(results.get(3).variable2)
self.assertEqual(results.get(3).rounds, 2)
self.assertEqual(results.get(3).opponents, 2)
self.assertEqual(results.get(3).wins, 0)
self.assertEqual(results.get(3).loses, 2)
# test elo rating algorithm
self.package_name = "elo_rating"
results = calculate_score(
package_name=self.package_name,
comparison_pairs=self.comparisons
)
self.assertEqual(len(results.items()), 3)
self.assertIsInstance(results.get(1), ScoredObject)
self.assertIsInstance(results.get(1).score, float)
self.assertIsInstance(results.get(1).variable1, float)
self.assertIsNone(results.get(1).variable2)
self.assertEqual(results.get(1).rounds, 2)
self.assertEqual(results.get(1).opponents, 2)
self.assertEqual(results.get(1).wins, 2)
self.assertEqual(results.get(1).loses, 0)
self.assertIsInstance(results.get(2), ScoredObject)
self.assertIsInstance(results.get(2).score, float)
self.assertIsInstance(results.get(2).variable1, float)
self.assertIsNone(results.get(2).variable2)
self.assertEqual(results.get(2).rounds, 2)
self.assertEqual(results.get(2).opponents, 2)
self.assertEqual(results.get(2).wins, 1)
self.assertEqual(results.get(2).loses, 1)
self.assertIsInstance(results.get(3), ScoredObject)
self.assertIsInstance(results.get(3).score, float)
self.assertIsInstance(results.get(3).variable1, float)
self.assertIsNone(results.get(3).variable2)
self.assertEqual(results.get(3).rounds, 2)
self.assertEqual(results.get(3).opponents, 2)
self.assertEqual(results.get(3).wins, 0)
self.assertEqual(results.get(3).loses, 2)
# test true skill rating algorithm
self.package_name = "true_skill_rating"
results = calculate_score(
package_name=self.package_name,
comparison_pairs=self.comparisons
)
self.assertEqual(len(results.items()), 3)
self.assertIsInstance(results.get(1), ScoredObject)
self.assertIsInstance(results.get(1).score, float)
self.assertIsInstance(results.get(1).variable1, float)
self.assertIsInstance(results.get(1).variable2, float)
self.assertEqual(results.get(1).rounds, 2)
self.assertEqual(results.get(1).opponents, 2)
self.assertEqual(results.get(1).wins, 2)
self.assertEqual(results.get(1).loses, 0)
self.assertIsInstance(results.get(2), ScoredObject)
self.assertIsInstance(results.get(2).score, float)
self.assertIsInstance(results.get(2).variable1, float)
self.assertIsInstance(results.get(2).variable2, float)
self.assertEqual(results.get(2).rounds, 2)
self.assertEqual(results.get(2).opponents, 2)
self.assertEqual(results.get(2).wins, 1)
self.assertEqual(results.get(2).loses, 1)
self.assertIsInstance(results.get(3), ScoredObject)
self.assertIsInstance(results.get(3).score, float)
self.assertIsInstance(results.get(3).variable1, float)
self.assertIsInstance(results.get(3).variable2, float)
self.assertEqual(results.get(3).rounds, 2)
self.assertEqual(results.get(3).opponents, 2)
self.assertEqual(results.get(3).wins, 0)
self.assertEqual(results.get(3).loses, 2)
def calculate_scores(cls, assignment_id):
from . import AnswerScore, AnswerCriterionScore, \
AssignmentCriterion, ScoringAlgorithm
assignment = Assignment.get(assignment_id)
# get all comparisons for this assignment and only load the data we need
comparisons = Comparison.query \
.filter(Comparison.assignment_id == assignment_id) \
.all()
assignment_criteria = AssignmentCriterion.query \
.with_entities(AssignmentCriterion.criterion_id) \
.filter_by(assignment_id=assignment_id, active=True) \
.all()
comparison_criteria = []
comparison_pairs = []
answer_ids = set()
for comparison in comparisons:
answer_ids.add(comparison.answer1_id)
answer_ids.add(comparison.answer2_id)
comparison_criteria.extend(comparison.comparison_criteria)
comparison_pairs.append(comparison.convert_to_comparison_pair())
# calculate answer score
comparison_results = calculate_score(
package_name=assignment.scoring_algorithm.value,
comparison_pairs=comparison_pairs,
log=current_app.logger
)
scores = AnswerScore.query \
.filter(AnswerScore.answer_id.in_(answer_ids)) \
.all()
updated_answer_scores = update_answer_scores(scores, assignment_id, comparison_results)
db.session.add_all(updated_answer_scores)
# calculate answer criterion scores
criterion_comparison_results = {}
for assignment_criterion in assignment_criteria:
comparison_pairs = []
for comparison_criterion in comparison_criteria:
if comparison_criterion.criterion_id != assignment_criterion.criterion_id:
continue
comparison_pairs.append(comparison_criterion.convert_to_comparison_pair())
criterion_comparison_results[assignment_criterion.criterion_id] = calculate_score(
package_name=assignment.scoring_algorithm.value,
comparison_pairs=comparison_pairs,
log=current_app.logger
)
scores = AnswerCriterionScore.query \
.filter(AnswerCriterionScore.answer_id.in_(answer_ids)) \
.all()
updated_answer_criteria_scores = update_answer_criteria_scores(scores, assignment_id, criterion_comparison_results)
db.session.add_all(updated_answer_criteria_scores)
db.session.commit()
CURRENT_FOLDER = os.getcwd() + '/scripts'
comparisons = []
with open(CURRENT_FOLDER + '/input.csv', 'rU') as csvfile:
spamreader = csv.reader(csvfile)
for row in spamreader:
if row:
comparisons.append(
ComparisonPair(int(row[KEY1]), int(row[KEY2]),
ComparisonWinner(row[WINNER])))
packages = [
ScoringAlgorithm.comparative_judgement.value, ScoringAlgorithm.elo.value,
ScoringAlgorithm.true_skill.value
]
for package_name in packages:
results = calculate_score(package_name=package_name,
comparison_pairs=comparisons)
with open(CURRENT_FOLDER + "/out_" + package_name + ".csv",
"w+") as csvfile:
out = csv.writer(csvfile)
out.writerow(["id", "score", "rounds", "wins", "normal score"])
for key, result in results.items():
out.writerow(
[result.key, result.score, result.rounds, result.wins, ""])
def test_calculate_score(self):
# test comparitive judgement score algorithm
self.package_name = "comparative_judgement"
results = calculate_score(
package_name=self.package_name,
comparison_pairs=self.comparisons
)
self.assertEqual(len(results.items()), 3)
self.assertIsInstance(results.get(1), ScoredObject)
self.assertIsInstance(results.get(1).score, float)
self.assertIsInstance(results.get(1).variable1, float)
self.assertIsNone(results.get(1).variable2)
self.assertEqual(results.get(1).rounds, 2)
self.assertEqual(results.get(1).opponents, 2)
self.assertEqual(results.get(1).wins, 2)
self.assertEqual(results.get(1).loses, 0)
self.assertIsInstance(results.get(2), ScoredObject)
self.assertIsInstance(results.get(2).score, float)
self.assertIsInstance(results.get(2).variable1, float)
self.assertIsNone(results.get(2).variable2)
self.assertEqual(results.get(2).rounds, 2)
self.assertEqual(results.get(2).opponents, 2)
self.assertEqual(results.get(2).wins, 1)
self.assertEqual(results.get(2).loses, 1)
self.assertIsInstance(results.get(3), ScoredObject)
self.assertIsInstance(results.get(3).score, float)
self.assertIsInstance(results.get(3).variable1, float)
self.assertIsNone(results.get(3).variable2)
self.assertEqual(results.get(3).rounds, 2)
self.assertEqual(results.get(3).opponents, 2)
self.assertEqual(results.get(3).wins, 0)
self.assertEqual(results.get(3).loses, 2)
# test elo rating algorithm
self.package_name = "elo_rating"
results = calculate_score(
package_name=self.package_name,
comparison_pairs=self.comparisons
)
self.assertEqual(len(results.items()), 3)
self.assertIsInstance(results.get(1), ScoredObject)
self.assertIsInstance(results.get(1).score, float)
self.assertIsInstance(results.get(1).variable1, float)
self.assertIsNone(results.get(1).variable2)
self.assertEqual(results.get(1).rounds, 2)
self.assertEqual(results.get(1).opponents, 2)
self.assertEqual(results.get(1).wins, 2)
self.assertEqual(results.get(1).loses, 0)
self.assertIsInstance(results.get(2), ScoredObject)
self.assertIsInstance(results.get(2).score, float)
self.assertIsInstance(results.get(2).variable1, float)
self.assertIsNone(results.get(2).variable2)
self.assertEqual(results.get(2).rounds, 2)
self.assertEqual(results.get(2).opponents, 2)
self.assertEqual(results.get(2).wins, 1)
self.assertEqual(results.get(2).loses, 1)
self.assertIsInstance(results.get(3), ScoredObject)
self.assertIsInstance(results.get(3).score, float)
self.assertIsInstance(results.get(3).variable1, float)
self.assertIsNone(results.get(3).variable2)
self.assertEqual(results.get(3).rounds, 2)
self.assertEqual(results.get(3).opponents, 2)
self.assertEqual(results.get(3).wins, 0)
self.assertEqual(results.get(3).loses, 2)
# test true skill rating algorithm
self.package_name = "true_skill_rating"
results = calculate_score(
package_name=self.package_name,
comparison_pairs=self.comparisons
)
self.assertEqual(len(results.items()), 3)
self.assertIsInstance(results.get(1), ScoredObject)
self.assertIsInstance(results.get(1).score, float)
self.assertIsInstance(results.get(1).variable1, float)
self.assertIsInstance(results.get(1).variable2, float)
self.assertEqual(results.get(1).rounds, 2)
self.assertEqual(results.get(1).opponents, 2)
self.assertEqual(results.get(1).wins, 2)
self.assertEqual(results.get(1).loses, 0)
self.assertIsInstance(results.get(2), ScoredObject)
self.assertIsInstance(results.get(2).score, float)
self.assertIsInstance(results.get(2).variable1, float)
self.assertIsInstance(results.get(2).variable2, float)
self.assertEqual(results.get(2).rounds, 2)
self.assertEqual(results.get(2).opponents, 2)
self.assertEqual(results.get(2).wins, 1)
self.assertEqual(results.get(2).loses, 1)
self.assertIsInstance(results.get(3), ScoredObject)
self.assertIsInstance(results.get(3).score, float)
self.assertIsInstance(results.get(3).variable1, float)
self.assertIsInstance(results.get(3).variable2, float)
self.assertEqual(results.get(3).rounds, 2)
self.assertEqual(results.get(3).opponents, 2)
self.assertEqual(results.get(3).wins, 0)
self.assertEqual(results.get(3).loses, 2)
with open(CURRENT_FOLDER+'/input.csv', 'rU') as csvfile:
spamreader = csv.reader(csvfile)
for row in spamreader:
if row:
comparisons.append(ComparisonPair(
int(row[KEY1]), int(row[KEY2]), ComparisonWinner(row[WINNER])
))
packages = [
ScoringAlgorithm.comparative_judgement.value,
ScoringAlgorithm.elo.value,
ScoringAlgorithm.true_skill.value
]
for package_name in packages:
results = calculate_score(
package_name=package_name,
comparison_pairs=comparisons
)
with open(CURRENT_FOLDER+"/out_"+package_name+".csv", "w+") as csvfile:
out = csv.writer(csvfile)
out.writerow(["id", "score", "rounds", "wins", "normal score"])
for key, result in results.items():
out.writerow([result.key, result.score,
result.rounds, result.wins, ""])