def test_Q1_all(self):
pointlist, textfdist, slist, rulelist = self.parse_Q1()
manuallist = [
9, 7, 9, 7, 7, 10, 7, 7, 7, 7, 7, 1, 2, 2, 0, 1, 2, 2, 1, 1, 8, 9,
4, 9, 7, 9, 0, 7, 4, 10, 7, 7
]
minmaxerr = 0
minrd = 0
minerrcount = 0
for i in range(10):
maxerr = 0
errcount = 0
var = 0
rd = random.uniform(0.32, 0.36)
debug_print_without_newline("rd = ")
print rd
# Create the appropriate class instance for Answer
# TODO: Remove abAnswer method overrides altogether and do everything via proper subclassing (RTFM!!!).
## OLD: ans = abAnswer(dist_threshold=rd, multisen_matchrate=0.3, sen_threshold=rd, multisen_threshold=0.4)
ans = None
if (USE_OVERRIDES):
ans = abAnswer(dist_threshold=rd,
multisen_matchrate=0.3,
sen_threshold=rd,
multisen_threshold=0.4)
else:
ans = Answer()
self.dist_threshold = rd
self.multisen_matchrate = 0.3
self.sen_threshold = rd
self.multisen_threshold = 0.4
# Test all cases in Q1 directory
for root, dirs, files in os.walk('algo/testdata/raw/Q1'):
if 'Standard' in dirs:
dirs.remove('Standard')
for idx in range(0, 32):
ansfile = 'Q1_SS' + str(idx + 1) + '.docx.txt'
filePath = os.path.join(root, ansfile)
fh = file(filePath, "r")
anstext = fh.read()
fh.close()
# TODO: Always use freq dist for student text (not standard)
if (USE_STUDENT_TEXT_DIST):
textfdist = get_student_text_distribution(
anstext, textfdist)
mark, marklist, ommited = ans.Analysis(
anstext, textfdist, slist, pointlist, rulelist)
err = mark - manuallist[idx]
maxerr += math.fabs(err)
var += err**2
errcount += 1 if math.fabs(err) > 3 else 0
print("%s\t%d\t%s\t%d" % (ansfile, mark, marklist, err))
if errcount < minerrcount:
minerrcount = errcount
minmaxerr = maxerr
minrd = rd
print "maxerr:%d, maxvar:%d, errcount:%d" % (maxerr, var, errcount)
print "minmaxerr:%d rd:%d count:%d" % (minmaxerr, minrd, minerrcount)
def __traversal_process(self, testdir):
ans = Answer()
for root, dirs, files in os.walk(testdir):
if 'Standard' in dirs:
dirs.remove('Standard')
for stdfile in files:
# Check for answer file (e.g., "ans_q8.txt")
if 'ans' in stdfile:
testno = stdfile[4:-4]
self.logger.info("no:%s" % testno)
stdPath = os.path.join(root, stdfile)
if not os.path.isfile(stdPath):
self.logger.error("Test file doesn't exist:%s" %
stdfile)
assert False
fh = file(stdPath, "r")
filetext = fh.read()
fh.close()
sinst = Standard()
pointlist, textfdist, slist = sinst.Analysis(filetext)
# Check schema file (e.g., "scheme_q8.txt")
schemename = 'scheme_' + testno + '.txt'
schemepath = os.path.join(root, schemename)
fr = file(schemepath, 'r')
scheme = self.__parsescheme(fr.read())
fr.close()
rulelist = self.__updaterulelist(scheme, pointlist)
print("ansfile\tmark\tmarklist")
for idx in range(0, 10):
# Check student response file (e.g., "stud9_q8.txt")
ansfile = 'stud' + str(idx + 1) + '_' + testno + '.txt'
ansPath = os.path.join(root, ansfile)
if os.path.isfile(ansPath):
fa = file(ansPath, 'r')
anstext = fa.read()
fa.close()
if anstext:
# TODO: Always use freq dist for student text (not standard)
if (USE_STUDENT_TEXT_DIST):
textfdist = get_student_text_distribution(
anstext, textfdist)
debug_print("Calling ans.Analysis%s" % str(
(anstext, textfdist, slist, pointlist,
rulelist)),
level=4)
mark, marklist, ommited = ans.Analysis(
anstext, textfdist, slist, pointlist,
rulelist)
else:
mark = 0
marklist = []
print("%s\t%d\t%s" % (ansfile, mark, marklist))
def test_Q1_all(self):
debug_print("test_Q1_all()", level=4)
pointlist, textfdist, slist, rulelist = self.parse_Q1()
manuallist = [
9, 7, 9, 7, 7, 10, 7, 7, 7, 7, 7, 1, 2, 2, 0, 1, 2, 2, 1, 1, 8, 9,
4, 9, 7, 9, 0, 7, 4, 10, 7, 7
]
minmaxerr = 0
minrd = 0
minerrcount = 0
total_good = 0
total_system = 0
total_manual = 0
# Optionally initialize random see (useful for debugging)
if RANDOM_SEED > 0:
debug_print("Setting random seed to %d" % RANDOM_SEED)
random.seed(RANDOM_SEED)
# Run ten different evaluations with different random sentence matching threshold
for i in range(NUM_TRIALS):
trial_num = i + 1
debug_print("trial %d" % trial_num, level=4)
maxerr = 0
errcount = 0
var = 0
# Create the appropriate class instance for Answer
# Note: default thresholds: dist_threshold 0.25, multisen_matchrate 0.3, sen_threshold 0.33, multisen_threshold 0.4
# TODO: Remove abAnswer method overrides altogether and do everything via proper subclassing (RTFM!!!).
## OLD: ans = abAnswer(dist_threshold=rd, multisen_matchrate=0.3, sen_threshold=rd, multisen_threshold=0.4)
ans = None
if (USE_OVERRIDES):
## OLD: ans = abAnswer(dist_threshold=rd, multisen_matchrate=0.3, sen_threshold=rd, multisen_threshold=0.4)
ans = abAnswer()
else:
ans = Answer()
if RANDOM_THRESHOLDS:
rd = random.uniform(0.32, 0.36)
debug_print_without_newline("rd = ")
print rd
ans.dist_threshold = rd
ans.multisen_matchrate = 0.3
ans.sen_threshold = rd
ans.multisen_threshold = 0.4
# Test all cases in Q1 directory
# TODO: replace os.walk with directory read
Q1_base_dir = 'algo/testdata/raw/Q1'
for root, dirs, files in os.walk('algo/testdata/raw/Q1'):
if root != Q1_base_dir:
continue
if 'Standard' in dirs:
dirs.remove('Standard')
for idx in range(0, MAX_ESSAYS):
# Read student answer
## OLD: ansfile = 'Q1_SS' + str(idx + 1) + '.docx.txt'
ext = '.annot.txt' if CHECK_LINKAGES else '.docx.txt'
ansfile = 'Q1_SS' + str(idx + 1) + ext
debug_print("Processing student answer '%s'" % ansfile,
level=3)
filePath = os.path.join(root, ansfile)
fh = file(filePath, "r")
anstext = fh.read()
debug_print("anstext: { %s }" % anstext, level=7)
fh.close()
# Check the text for optional annotations and isolate
if CHECK_LINKAGES:
self.answer_annotations = Annotations()
self.answer_annotations.extract_annotations(anstext)
## DUH: anstext = self.standard_annotations.text_proper
anstext = self.answer_annotations.text_proper
# Grade the essay and compare against manual
# TODO: Always use freq dist for student text (not standard)
if (USE_STUDENT_TEXT_DIST):
textfdist = get_student_text_distribution(
anstext, textfdist)
mark, marklist, ommited = ans.Analysis(
anstext, textfdist, slist, pointlist, rulelist)
err = mark - manuallist[idx]
# Check the system sentence linkages versus the annotations
if CHECK_LINKAGES:
num_good, num_system, num_manual = evaluate_linkages(
self.answer_annotations, ans.detailedmarklist,
marklist)
calculate_fscore("ss" + str(idx + 1), num_good,
num_system, num_manual)
total_good += num_good
total_system += num_system
total_manual += num_manual
# Check grammar, etc.
if ans.apply_grammar_checking:
print("answer critique: %s" % ans.text_critique)
# Update statistics
maxerr += math.fabs(err)
var += err**2
errcount += 1 if math.fabs(err) > 3 else 0
print("%s\t%d\t%s\t%d" % (ansfile, mark, marklist, err))
if errcount < minerrcount:
minerrcount = errcount
minmaxerr = maxerr
minrd = rd
print "maxerr:%d, maxvar:%d, errcount:%d" % (maxerr, var, errcount)
print "minmaxerr:%d rd:%d count:%d" % (minmaxerr, minrd, minerrcount)
# Summarize answer/standard correspondences with respect to manual annotations
if CHECK_LINKAGES:
calculate_fscore("Overall", total_good, total_system, total_manual)
def test_answer(self):
self.logger.info("Test Student Answer Analysis")
# Read in the correct answer to first question
# TODO: Create helper function for reading question info as same code sequence used elsewhere.
testStandardAnswerFile = "ans_Q1.txt"
stdFilePath = os.path.join("algo/testdata/raw/Q1",
testStandardAnswerFile)
self.logger.info("stdanswer filepath:%s" % stdFilePath)
if not os.path.isfile(stdFilePath):
self.logger.error("Standard Test file doesn't exist:%s" %
testStandardAnswerFile)
assert False
fh = file(stdFilePath, "r")
stdtext = fh.read()
fh.close()
# Perform text processing analysis over correct answer
sinst = Standard()
pointlist, textfdist, slist = sinst.Analysis(stdtext)
std_pointlist_no = [point['Point_No'] for point in pointlist]
self.logger.info("Points:%s" % std_pointlist_no)
# Read in the standard as if it were
# TODO: Just do an assignment for crying out loud! Such needless code repetiton!
# ex: anstext = stdtext
testAnswerFile = "ans_Q1.txt"
ansFilePath = os.path.join("algo/testdata/raw/Q1", testAnswerFile)
self.logger.info("answer filepath:%s" % ansFilePath)
if not os.path.isfile(ansFilePath):
self.logger.error("Answer file doesn't exist:%s" % testAnswerFile)
assert False
fh = file(ansFilePath, "r")
anstext = fh.read()
fh.close()
# Create some dummy grading rules
mockrulelist = [{
'Mark':
10,
'Point': ['P1.1', 'P1.2', 'P1.3', 'P2', 'P3', 'P4', 'P5']
}, {
'Mark': 7,
'Point': ['P1.1', 'P2', 'P3', 'P4', 'P5']
}, {
'Mark': 6,
'Point': ['P1.1', 'P2', 'P3', 'P4']
}, {
'Mark': 5,
'Point': ['P1.1', 'P2', 'P3']
}, {
'Mark': 3,
'Point': ['P1.1', 'P2']
}, {
'Mark': 2,
'Point': ['P1.1']
}]
pprint.pprint(mockrulelist)
# Create the answer class instance and optionally override global frequency distribution from answer text.
# TODO: Always use freq dist for student text (not standard).
ans = Answer()
if (USE_STUDENT_TEXT_DIST):
textfdist = get_student_text_distribution(anstext, textfdist)
# Preprocess the student answer and then compare resulting vectors against standard
# TODO: Raise an exception if the result is not as expected
mark, marklist, ommited = ans.Analysis(anstext, textfdist, slist,
pointlist, mockrulelist)
pprint.pprint(mark)
pprint.pprint(ommited)
self.logger.info("Test Student Answer Analysis Finished")
