def test_Q1_all(self): pointlist, textfdist, slist, rulelist = self.parse_Q1() manuallist = [ 9, 7, 9, 7, 7, 10, 7, 7, 7, 7, 7, 1, 2, 2, 0, 1, 2, 2, 1, 1, 8, 9, 4, 9, 7, 9, 0, 7, 4, 10, 7, 7 ] minmaxerr = 0 minrd = 0 minerrcount = 0 for i in range(10): maxerr = 0 errcount = 0 var = 0 rd = random.uniform(0.32, 0.36) debug_print_without_newline("rd = ") print rd # Create the appropriate class instance for Answer # TODO: Remove abAnswer method overrides altogether and do everything via proper subclassing (RTFM!!!). ## OLD: ans = abAnswer(dist_threshold=rd, multisen_matchrate=0.3, sen_threshold=rd, multisen_threshold=0.4) ans = None if (USE_OVERRIDES): ans = abAnswer(dist_threshold=rd, multisen_matchrate=0.3, sen_threshold=rd, multisen_threshold=0.4) else: ans = Answer() self.dist_threshold = rd self.multisen_matchrate = 0.3 self.sen_threshold = rd self.multisen_threshold = 0.4 # Test all cases in Q1 directory for root, dirs, files in os.walk('algo/testdata/raw/Q1'): if 'Standard' in dirs: dirs.remove('Standard') for idx in range(0, 32): ansfile = 'Q1_SS' + str(idx + 1) + '.docx.txt' filePath = os.path.join(root, ansfile) fh = file(filePath, "r") anstext = fh.read() fh.close() # TODO: Always use freq dist for student text (not standard) if (USE_STUDENT_TEXT_DIST): textfdist = get_student_text_distribution( anstext, textfdist) mark, marklist, ommited = ans.Analysis( anstext, textfdist, slist, pointlist, rulelist) err = mark - manuallist[idx] maxerr += math.fabs(err) var += err**2 errcount += 1 if math.fabs(err) > 3 else 0 print("%s\t%d\t%s\t%d" % (ansfile, mark, marklist, err)) if errcount < minerrcount: minerrcount = errcount minmaxerr = maxerr minrd = rd print "maxerr:%d, maxvar:%d, errcount:%d" % (maxerr, var, errcount) print "minmaxerr:%d rd:%d count:%d" % (minmaxerr, minrd, minerrcount)
def __traversal_process(self, testdir): ans = Answer() for root, dirs, files in os.walk(testdir): if 'Standard' in dirs: dirs.remove('Standard') for stdfile in files: # Check for answer file (e.g., "ans_q8.txt") if 'ans' in stdfile: testno = stdfile[4:-4] self.logger.info("no:%s" % testno) stdPath = os.path.join(root, stdfile) if not os.path.isfile(stdPath): self.logger.error("Test file doesn't exist:%s" % stdfile) assert False fh = file(stdPath, "r") filetext = fh.read() fh.close() sinst = Standard() pointlist, textfdist, slist = sinst.Analysis(filetext) # Check schema file (e.g., "scheme_q8.txt") schemename = 'scheme_' + testno + '.txt' schemepath = os.path.join(root, schemename) fr = file(schemepath, 'r') scheme = self.__parsescheme(fr.read()) fr.close() rulelist = self.__updaterulelist(scheme, pointlist) print("ansfile\tmark\tmarklist") for idx in range(0, 10): # Check student response file (e.g., "stud9_q8.txt") ansfile = 'stud' + str(idx + 1) + '_' + testno + '.txt' ansPath = os.path.join(root, ansfile) if os.path.isfile(ansPath): fa = file(ansPath, 'r') anstext = fa.read() fa.close() if anstext: # TODO: Always use freq dist for student text (not standard) if (USE_STUDENT_TEXT_DIST): textfdist = get_student_text_distribution( anstext, textfdist) debug_print("Calling ans.Analysis%s" % str( (anstext, textfdist, slist, pointlist, rulelist)), level=4) mark, marklist, ommited = ans.Analysis( anstext, textfdist, slist, pointlist, rulelist) else: mark = 0 marklist = [] print("%s\t%d\t%s" % (ansfile, mark, marklist))
def test_Q1_all(self): debug_print("test_Q1_all()", level=4) pointlist, textfdist, slist, rulelist = self.parse_Q1() manuallist = [ 9, 7, 9, 7, 7, 10, 7, 7, 7, 7, 7, 1, 2, 2, 0, 1, 2, 2, 1, 1, 8, 9, 4, 9, 7, 9, 0, 7, 4, 10, 7, 7 ] minmaxerr = 0 minrd = 0 minerrcount = 0 total_good = 0 total_system = 0 total_manual = 0 # Optionally initialize random see (useful for debugging) if RANDOM_SEED > 0: debug_print("Setting random seed to %d" % RANDOM_SEED) random.seed(RANDOM_SEED) # Run ten different evaluations with different random sentence matching threshold for i in range(NUM_TRIALS): trial_num = i + 1 debug_print("trial %d" % trial_num, level=4) maxerr = 0 errcount = 0 var = 0 # Create the appropriate class instance for Answer # Note: default thresholds: dist_threshold 0.25, multisen_matchrate 0.3, sen_threshold 0.33, multisen_threshold 0.4 # TODO: Remove abAnswer method overrides altogether and do everything via proper subclassing (RTFM!!!). ## OLD: ans = abAnswer(dist_threshold=rd, multisen_matchrate=0.3, sen_threshold=rd, multisen_threshold=0.4) ans = None if (USE_OVERRIDES): ## OLD: ans = abAnswer(dist_threshold=rd, multisen_matchrate=0.3, sen_threshold=rd, multisen_threshold=0.4) ans = abAnswer() else: ans = Answer() if RANDOM_THRESHOLDS: rd = random.uniform(0.32, 0.36) debug_print_without_newline("rd = ") print rd ans.dist_threshold = rd ans.multisen_matchrate = 0.3 ans.sen_threshold = rd ans.multisen_threshold = 0.4 # Test all cases in Q1 directory # TODO: replace os.walk with directory read Q1_base_dir = 'algo/testdata/raw/Q1' for root, dirs, files in os.walk('algo/testdata/raw/Q1'): if root != Q1_base_dir: continue if 'Standard' in dirs: dirs.remove('Standard') for idx in range(0, MAX_ESSAYS): # Read student answer ## OLD: ansfile = 'Q1_SS' + str(idx + 1) + '.docx.txt' ext = '.annot.txt' if CHECK_LINKAGES else '.docx.txt' ansfile = 'Q1_SS' + str(idx + 1) + ext debug_print("Processing student answer '%s'" % ansfile, level=3) filePath = os.path.join(root, ansfile) fh = file(filePath, "r") anstext = fh.read() debug_print("anstext: { %s }" % anstext, level=7) fh.close() # Check the text for optional annotations and isolate if CHECK_LINKAGES: self.answer_annotations = Annotations() self.answer_annotations.extract_annotations(anstext) ## DUH: anstext = self.standard_annotations.text_proper anstext = self.answer_annotations.text_proper # Grade the essay and compare against manual # TODO: Always use freq dist for student text (not standard) if (USE_STUDENT_TEXT_DIST): textfdist = get_student_text_distribution( anstext, textfdist) mark, marklist, ommited = ans.Analysis( anstext, textfdist, slist, pointlist, rulelist) err = mark - manuallist[idx] # Check the system sentence linkages versus the annotations if CHECK_LINKAGES: num_good, num_system, num_manual = evaluate_linkages( self.answer_annotations, ans.detailedmarklist, marklist) calculate_fscore("ss" + str(idx + 1), num_good, num_system, num_manual) total_good += num_good total_system += num_system total_manual += num_manual # Check grammar, etc. if ans.apply_grammar_checking: print("answer critique: %s" % ans.text_critique) # Update statistics maxerr += math.fabs(err) var += err**2 errcount += 1 if math.fabs(err) > 3 else 0 print("%s\t%d\t%s\t%d" % (ansfile, mark, marklist, err)) if errcount < minerrcount: minerrcount = errcount minmaxerr = maxerr minrd = rd print "maxerr:%d, maxvar:%d, errcount:%d" % (maxerr, var, errcount) print "minmaxerr:%d rd:%d count:%d" % (minmaxerr, minrd, minerrcount) # Summarize answer/standard correspondences with respect to manual annotations if CHECK_LINKAGES: calculate_fscore("Overall", total_good, total_system, total_manual)
def test_answer(self): self.logger.info("Test Student Answer Analysis") # Read in the correct answer to first question # TODO: Create helper function for reading question info as same code sequence used elsewhere. testStandardAnswerFile = "ans_Q1.txt" stdFilePath = os.path.join("algo/testdata/raw/Q1", testStandardAnswerFile) self.logger.info("stdanswer filepath:%s" % stdFilePath) if not os.path.isfile(stdFilePath): self.logger.error("Standard Test file doesn't exist:%s" % testStandardAnswerFile) assert False fh = file(stdFilePath, "r") stdtext = fh.read() fh.close() # Perform text processing analysis over correct answer sinst = Standard() pointlist, textfdist, slist = sinst.Analysis(stdtext) std_pointlist_no = [point['Point_No'] for point in pointlist] self.logger.info("Points:%s" % std_pointlist_no) # Read in the standard as if it were # TODO: Just do an assignment for crying out loud! Such needless code repetiton! # ex: anstext = stdtext testAnswerFile = "ans_Q1.txt" ansFilePath = os.path.join("algo/testdata/raw/Q1", testAnswerFile) self.logger.info("answer filepath:%s" % ansFilePath) if not os.path.isfile(ansFilePath): self.logger.error("Answer file doesn't exist:%s" % testAnswerFile) assert False fh = file(ansFilePath, "r") anstext = fh.read() fh.close() # Create some dummy grading rules mockrulelist = [{ 'Mark': 10, 'Point': ['P1.1', 'P1.2', 'P1.3', 'P2', 'P3', 'P4', 'P5'] }, { 'Mark': 7, 'Point': ['P1.1', 'P2', 'P3', 'P4', 'P5'] }, { 'Mark': 6, 'Point': ['P1.1', 'P2', 'P3', 'P4'] }, { 'Mark': 5, 'Point': ['P1.1', 'P2', 'P3'] }, { 'Mark': 3, 'Point': ['P1.1', 'P2'] }, { 'Mark': 2, 'Point': ['P1.1'] }] pprint.pprint(mockrulelist) # Create the answer class instance and optionally override global frequency distribution from answer text. # TODO: Always use freq dist for student text (not standard). ans = Answer() if (USE_STUDENT_TEXT_DIST): textfdist = get_student_text_distribution(anstext, textfdist) # Preprocess the student answer and then compare resulting vectors against standard # TODO: Raise an exception if the result is not as expected mark, marklist, ommited = ans.Analysis(anstext, textfdist, slist, pointlist, mockrulelist) pprint.pprint(mark) pprint.pprint(ommited) self.logger.info("Test Student Answer Analysis Finished")