def evaluate_student_number(phrasedir, summarydir, output): body = [] lectures = annotation.Lectures for i, lec in enumerate(lectures): for prompt in ['q1', 'q2']: key_prefix = os.path.join(phrasedir, str(lec), '%s.%s'%(prompt, method)) sum_prefix = os.path.join(summarydir, str(lec), '%s'%(prompt)) evalator = EvalStudent(key_prefix, sum_prefix, len(annotation.anotators)) scores = evalator.score_no() row = [lec] + scores body.append(row) head = ['week', 'precision', 'recall', 'f-measure'] row = ['ave'] for i in range(1, len(head)): scores = [float(xx[i]) for xx in body] row.append(numpy.mean(scores)) body.append(row) fio.WriteMatrix(output, body, head)
def getStudentResponses4Annotation(excelfile, cid, maxWeek, datadir): sheets = range(1, maxWeek + 1) for sheet in sheets: week = sheet for type in ['q1', 'q2', 'q3', 'q4']: head = ['student_id', 'sentence_id', 'responses'] body = [] student_summaryList = getStudentResponseList( excelfile, cid, week, type, True) if len(student_summaryList) == 0: continue filename = datadir + "response." + str(week) + "." + type + ".txt" old = "" i = 1 for summary, id in student_summaryList: row = [] summary = summary.replace('"', '\'') if len(summary.strip()) == 0: continue if id == old: row.append(" ") else: row.append(id) row.append(i) row.append(summary) body.append(row) i = i + 1 old = id fio.WriteMatrix(filename, body, head)
def gather_performance(output): sim_extractor = Similarity() allfeatures = sorted(sim_extractor.features.keys()) allbody = [] for k in range(len(allfeatures) + 1): #features = allfeatures#['WordEmbedding'] if k == len(allfeatures): #use all features features = allfeatures else: features = [allfeatures[k]] #features = allfeatures[0:k] + allfeatures[k+1:] name = '_'.join(features) resultfile = '../data/%s/simlearning.cv.svm.%s.txt' % (course, name) head, body = fio.ReadMatrix(resultfile, hasHead=True) #get the average allhead = ['name'] + head[2:] average = [name] for i in range(2, len(head)): #start from the third one values = [float(row[i]) for row in body] average.append(np.mean(values)) allbody.append(average) fio.WriteMatrix(output, allbody, allhead)
def PrintClusterRankSummary(datadir): sheets = range(0,maxWeek) lectures = fio.LoadDictJson('../data/CourseMIRROR/lectures.json') head = ['week', 'data', 'Point of Interest', "Muddiest Point"] body = [] for i, sheet in enumerate(sheets): row = [] week = i + 1 row.append(week) row.append(getDate(lectures, course, week)) for type in ['q1', 'q2', 'q3', 'q4']: path = datadir + str(i+1)+ '/' summaryfile = path + type + '.summary' if not fio.IsExist(summaryfile): continue summaries = [line.strip() for line in fio.ReadFile(summaryfile)] sourcefile = path + type + '.summary.source' sources = [line.split(',') for line in fio.ReadFile(sourcefile)] combinedSummary = [] for j, (summary, source) in enumerate(zip(summaries, sources)): summary = summary.replace('"', '\'') combinedSummary.append(str(j+1) + ") " + summary + " [" + str(len(source)) + "]") row.append('"' + chr(10).join(combinedSummary)+ '"') body.append(row) fio.WriteMatrix(datadir + "summary.txt", body, head)
def getStudentResponses4Quality(excelfile, cid, maxWeek, datadir): sheets = range(1, maxWeek + 1) for sheet in sheets: week = sheet for type in ['q1', 'q2', 'q3', 'q4']: head = ['student_id', 'responses'] body = [] student_summaries = getStudentResponse(excelfile, cid, week, type) if len(student_summaries) == 0: continue for id, summaryList in student_summaries.items(): summary = ' '.join(summaryList) row = [] summary = summary.replace('"', '\'') if len(summary.strip()) == 0: continue row.append(id) row.append(summary) body.append(row) filename = datadir + "response." + str(week) + "." + type + ".txt" fio.WriteMatrix(filename, body, head)
def getPhraseClusterAll(sennafile, weightfile, output, ratio=None, MalformedFlilter=False, source=None, np=None): NPCandidates, sources = getNPs(sennafile, MalformedFlilter, source=source, np=np) if len(NPCandidates) == 0: return NPs, matrix = fio.ReadMatrix(weightfile, hasHead=True) #change the similarity to distance matrix = Similarity2Distance(matrix) index = {} for i, NP in enumerate(NPs): index[NP] = i newMatrix = [] for NP1 in NPCandidates: assert (NP1 in index) i = index[NP1] row = [] for NP2 in NPCandidates: if NP2 not in index: print NP2, weightfile, np j = index[NP2] row.append(matrix[i][j]) newMatrix.append(row) V = len(NPCandidates) if ratio == "sqrt": K = int(math.sqrt(V)) elif float(ratio) > 1: K = int(ratio) else: K = int(ratio * V) if K < 1: K = 1 clusterid = ClusterWrapper.KMedoidCluster(newMatrix, K) body = [] for NP, id in zip(NPCandidates, clusterid): row = [] row.append(NP) row.append(id) body.append(row) fio.WriteMatrix(output, body, header=None)
def train_leave_one_lecture_out(model_dir, name='simlearn_cv'): # model_dir = '../data/IE256/%s/model/%s/'%(system, name) # fio.NewPath(model_dir) # # outputdir = '../data/IE256/%s/extraction/%s_output/'%(system, name) # fio.NewPath(outputdir) sim_extractor = Similarity() allfeatures = sorted(sim_extractor.features.keys()) if True: k = len(allfeatures) #for k in range(len(allfeatures)+1): #features = allfeatures#['WordEmbedding'] if k == len(allfeatures): #use all features features = allfeatures else: features = [allfeatures[k]] name = '_'.join(features) lectures = annotation.Lectures dict = defaultdict(int) MSE = [] for i, lec in enumerate(lectures): train = [x for x in lectures if x != lec] test = [lec] print train print test model_file = os.path.join(model_dir, '%d_%s.model' % (lec, name)) if fio.IsExist(model_file): with open(model_file, 'rb') as handle: clf = pickle.load(handle) else: train_X, train_Y = combine_files(train, features) clf = svm.SVR() clf.fit(train_X, train_Y) with open(model_file, 'wb') as handle: pickle.dump(clf, handle) for q in ['q1', 'q2']: test_X, test_Y = combine_files(test, features, prompts=[q]) predict_Y = clf.predict(test_X) mse = mean_squared_error(test_Y, predict_Y) MSE.append([lec, q, mse]) output = '../data/%s/simlearning.cv.%s.txt' % (course, name) fio.WriteMatrix(output, MSE, header=['lec', 'prompt', 'MSE'])
def getPhraseClusterPhrase(phrasefile, weightfile, output, ratio=None, method=None): NPCandidates = fio.ReadFile(phrasefile) if len(NPCandidates) == 0: return NPs, matrix = fio.ReadMatrix(weightfile, hasHead=True) #change the similarity to distance matrix = Similarity2Distance(matrix) index = {} for i, NP in enumerate(NPs): index[NP] = i newMatrix = [] for NP1 in NPCandidates: if NP1 not in index: continue i = index[NP1] row = [] for NP2 in NPCandidates: if NP2 not in index: print NP2, weightfile, method continue j = index[NP2] row.append(matrix[i][j]) newMatrix.append(row) V = len(NPCandidates) if ratio == "sqrt": K = int(math.sqrt(V)) elif float(ratio) >= 1: K = int(ratio) else: K = int(ratio * V) if K < 1: K = 1 K = min(K, V) clusterid = ClusterWrapper.KMedoidCluster(newMatrix, K) body = [] for NP, id in zip(NPCandidates, clusterid): row = [] row.append(NP) row.append(id) body.append(row) fio.WriteMatrix(output, body, header=None)
def correlation_analysis(course): phrasedir1 = '../data/%s/oracle_annotator_1/phrase/' % course phrasedir2 = '../data/%s/oracle_annotator_2/phrase/' % course outdir = '../data/%s/simlearning/' % course fio.NewPath(outdir) sim_extractor = Similarity() features = sorted(sim_extractor.features.keys()) head = features + ['score', 'predict'] body = [] lectures = annotation.Lectures name = '_'.join(features) for i, lec in enumerate(lectures): model_file = os.path.join(model_dir, '%d_%s.model' % (lec, name)) with open(model_file, 'rb') as handle: clf = pickle.load(handle) for q in ['q1', 'q2']: outfile = os.path.join(outdir, str(lec), '%s%s' % (q, sim_exe)) for phrasedir in [phrasedir1, phrasedir2]: path = phrasedir + str(lec) + '/' filename = os.path.join(path, q + sim_exe) data = fio.LoadDictJson(filename) for fdict, score, _ in data: row = [] for fname in features: x = fdict[fname] if str(x) == 'nan': x = 0.0 row.append(x) predict_score = clf.predict([row]) row.append(score) row.append(predict_score[0]) body.append(row) out_correlation = os.path.join(outdir, 'data.txt') print out_correlation fio.WriteMatrix(out_correlation, body, head)
def getPhraseCluster(phrasedir, method='lexicalOverlapComparer', ratio=None): sheets = range(0, 12) for sheet in sheets: week = sheet + 1 for type in ['POI', 'MP', 'LP']: weightfilename = phrasedir + str(week) + '/' + type + '.' + method print weightfilename NPs, matrix = fio.ReadMatrix(weightfilename, hasHead=True) #change the similarity to method for i, row in enumerate(matrix): for j, col in enumerate(row): matrix[i][j] = 1 - float( matrix[i][j]) if matrix[i][j] != "NaN" else 0 V = len(NPs) if ratio == None: K = int(math.sqrt(V)) else: K = int(ratio * V) K = 10 clusterid = ClusterWrapper.KMedoidCluster(matrix, K) # sorted_lists = sorted(zip(NPs, clusterid), key=lambda x: x[1]) # NPs, clusterid = [[x[i] for x in sorted_lists] for i in range(2)] dict = defaultdict(int) for id in clusterid: dict[id] = dict[id] + 1 body = [] for NP, id in zip(NPs, clusterid): row = [] row.append(NP) row.append(id) #row.append(dict[id]) body.append(row) if ratio == None: file = phrasedir + '/' + str( week ) + '/' + type + ".cluster.kmedoids." + "sqrt" + "." + method else: file = phrasedir + '/' + str( week) + '/' + type + ".cluster.kmedoids." + str( ratio) + "." + method fio.WriteMatrix(file, body, header=None)
def gather_rouge(output): datadir = '../data/%s/' % course #output = '../data/IE256/result.rouge.txt' models = [ 'QPS_NP', #'QPS_A1_N', 'QPS_A2_N', 'QPS_union', 'QPS_intersect', 'QPS_combine' ] methods = [ 'rouge_crf_optimumComparerLSATasa', 'rouge_crf_ct.svm.default', #'rouge_crf_svm', #'rouge_crf_svr', 'rouge_crf_ct.svm.default', #'rouge_crf_ct.svr.default', ] Header = [ 'method', 'model', 'R1-R', 'R1-P', 'R1-F', 'R2-R', 'R2-P', 'R2-F', 'RSU4-R', 'RSU4-P', 'RSU4-F', ] xbody = [] for method in methods: for model in models: filename = os.path.join(datadir, model, "%s.txt" % method) if not fio.IsExist(filename): continue head, body = fio.ReadMatrix(filename, hasHead=True) row = [method, model] row += body[-1][1:] xbody.append(row) fio.WriteMatrix(output, xbody, Header)
def writegraph_leave_one_lecture_out_lsa(model_dir, phrasedir, modelname='lsa'): lectures = annotation.Lectures for i, lec in enumerate(lectures): test = [lec] path = os.path.join(phrasedir, str(lec)) for q in ['q1', 'q2']: #write the output phrasefile = os.path.join(path, "%s.%s.key" % (q, method)) phrases = fio.LoadList(phrasefile) if modelname == 'lsa': similarties_results = os.path.join( path, "%s.%s.optimumComparerLSATasa" % (q, method)) elif modelname == 'svm': similarties_results = os.path.join(path, "%s.%s.svm" % (q, method)) simhead, simbody = fio.ReadMatrix(similarties_results, hasHead=True) assert (len(simhead) == len(phrases)) body = [] for i, p1 in enumerate(phrases): for j, p2 in enumerate(phrases): if j <= i: continue #undirect graph score = simbody[i][j] score = float(score) if score != 'NaN' else 0.0 #if score == 0.0: score = 0.000001 #if score < 0.5: continue if score == 0.0: continue #row = [i, j, '%f'%score] row = [i, j] body.append(row) output = os.path.join( path, "%s.%s.%s%s" % (q, method, modelname, net_exe)) fio.WriteMatrix(output, body)
def predict_IE256(train_course, model_dir, phrasedir, modelname='svm'): sim_extractor = Similarity() allfeatures = sorted(sim_extractor.features.keys()) features = allfeatures name = '_'.join(features) lectures = annotation.Lectures for i, lec in enumerate(lectures): test = [lec] print test model_file = os.path.join(model_dir, '%s_%s.model' % (train_course, name)) with open(model_file, 'rb') as handle: clf = pickle.load(handle) path = os.path.join(phrasedir, str(lec)) for q in ['q1', 'q2']: test_X, test_Y = combine_files_test(phrasedir, test, features, prompts=[q]) predict_Y = clf.predict(test_X) #write the output phrasefile = os.path.join(path, "%s.%s.key" % (q, method)) phrases = fio.LoadList(phrasefile) assert (len(predict_Y) == len(phrases) * len(phrases)) k = 0 body = [] for p1 in phrases: row = [] for p2 in phrases: row.append(predict_Y[k]) k += 1 body.append(row) output = os.path.join(path, "%s.%s.%s" % (q, method, modelname)) fio.WriteMatrix(output, body, phrases)
def correlation_analysis_noduplicate(): phrasedir1 = '../data/%s/oracle_annotator_1/phrase/' % course phrasedir2 = '../data/%s/oracle_annotator_2/phrase/' % course outdir = '../data/%s/simlearning/' % course fio.NewPath(outdir) sim_extractor = Similarity() features = sorted(sim_extractor.features.keys()) head = features + ['score'] body = [] lectures = annotation.Lectures for i, lec in enumerate(lectures): for q in ['q1', 'q2']: outfile = os.path.join(outdir, str(lec), '%s%s' % (q, sim_exe)) for phrasedir in [phrasedir1, phrasedir2]: path = phrasedir + str(lec) + '/' filename = os.path.join(path, q + sim_exe) data = fio.LoadDictJson(filename) for fdict, score, pd in data: if pd['p1'] == pd['p2']: print pd['p1'] continue row = [] for name in features: x = fdict[name] if str(x) == 'nan': x = 0.0 row.append(x) row.append(score) body.append(row) out_correlation = os.path.join(outdir, 'data.txt') fio.WriteMatrix(out_correlation, body, head)
def split_rouge(filename, prefix, N=2): head, body = fio.ReadMatrix(filename, hasHead=True) newbodies = [[] for i in range(N)] for i, row in enumerate(body[:-1]): newbodies[i % N].append(row) #compute the new average for k in range(len(newbodies)): row = ['ave'] for i in range(1, len(head)): scores = [float(xx[i]) for xx in newbodies[k]] row.append(numpy.mean(scores)) newbodies[k].append(row) for i, newbody in enumerate(newbodies): fio.WriteMatrix('%s_q%d.txt' % (prefix, i + 1), newbody, head)
def write_communite_to_clusters(communites, phrases, output): body = [] dict = {} for i, community in enumerate(communites): for node in community: row = [phrases[node], i + 1] body.append(row) dict[node] = 1 # k = len(communites) # #write phrases that are not in any communities # for i in range(len(phrases)): # if i in dict: continue # # row = [phrases[i], k] # k += 1 # body.append(row) fio.WriteMatrix(output, body, None)
crf_sub_output = '../data/%s/%s/extraction/all_output/' % (course, system) eval = CRFEval(class_index_dict_file, crf_sub_output) eval.get_label_accuracy() eval.get_mention_precision() eval.get_mention_recall() eval.get_mention_F_measure() for test in sorted(eval.dict): if test.startswith('overall'): continue #row = [system, eval.dict['overall_accuracy']['value'], eval.dict['overall_mention_precision']['value'],eval.dict['overall_mention_recall']['value'], eval.dict['overall_mention_F_measure']] row = [ system, test, eval.dict[test]['mention_precision']['value'], eval.dict[test]['mention_recall']['value'], eval.dict[test]['mention_F_measure'] ] body.append(row) eval.get_sentence_label_accuracy() print 'accuracy:%.4f (%d/%d)' % ( eval.dict['sentence_accuracy']['value'], eval.dict['sentence_accuracy']['correct'], eval.dict['sentence_accuracy']['total']) #print 'accuracy:%.4f\tprecision:%.4f\trecall:%.4f\tF-measure:%.4f'%(eval.dict['overall_accuracy']['value'], eval.dict['overall_mention_precision']['value'],eval.dict['overall_mention_recall']['value'], eval.dict['overall_mention_F_measure']) fio.WriteMatrix(output, body, head)
def TestRouge(): #ref = ["police killed the gunman"] #S1 = ["police kill the gunman"] #S2 = ["the gunman kill police"] S1_1 = ['Unable to finish the problem set before the quiz', 'Unable to understand why R is useful', 'Some questions were not answered', 'The problem set was confusing', 'Did not understand plots in R, or mean and median' ] S1_3=['There were two fundamental issues with the class. One, the students felt rushed and were unable to complete the required exercises, and two, the teaching assistant was seen as disorganized and rushed.'] S1_4 = ['The recitation was a good time to learn about the many useful features of R, and how to use them, e.g., plotting data sets and finding the mean and median. The questions in the problem set were well-organized to this end. The application to stock prices was a helpful demonstration.'] S1_2 = ['Learned some useful built-in functions of R', 'Practiced programming in R', 'The question types in the problem set were good', 'Application of R to stock values was helpful', 'The organization of the recitation was good'] S2_1=['Unfinished In-Class Questions', 'Knowing when to use R', 'Calculating Means/Medians', 'Using / Creating a Histogram', 'Using Rstudio'] S2_2 = ['Enjoyed Practice Session / In Class Questions', 'Liked learning about capabilities of rstudio', 'Liked Graphical Depictions / Diagrams', 'Liked Learning about Practical Applications of R', 'Enjoyed Calculating R'] S2_3=['Students had trouble knowing when to use R, and would have liked more examples. Some students had trouble using Rstudio. They were also concerned about the questions in the practice session, as they did not get to go over them.'] S2_4 = ['Students seemed to like using rstudio to calculate R, and creating graphics and seeing applications (such as stock values) for R. They liked the practice problems, and would have liked more of them.'] S1_5=['we cannot solve the last two questions of ps if we can solve them before quiz it will be very good ', 'Still not very clear how to use R/what data we were using at all times today when we were discussing R', 'Ps assistant couldn\'t manage her time properly.', 'question 3 in ps 1 was confusing', 'we can plot normal dist when the values are not continous but for it to work we need a lot independent variables , but like how much ?in the question it was 100', ] S1_6=['things we can do with r', 'Question types', 'I did not think that these questions can be created in lectures.', 'Interesting to combine the R with statistics on stock values', 'In the first part of class, we practised R programming, whuch was easy to understand.', ] S2_5=['The r experience was a bit Fast and hard to follow and the ps TA made simple questions look like a bit more complicated', 'in r function there can be more examples', 'Still not very clear how to use R/what data we were using at all times today when we were discussing R', 'inferring informations from histograph is a little confusing since it is new for me', 'Rstudio ps was not good I couldn\'t concentrate much', ] S2_6=['In the second part, we solved questions, which was helpful for us to remember our probability knowledge.', 'Most interesting thing was to learn that R is capable of many things like reading Excel or txt files.', 'Make a result of collection of data, look like normal distribution', 'the graphics in R and importing data to R', 'learning how to calculate in R', ] body = [] #getRouge(ref, S2) r1 = getRouge(S1_1, S2_1) r2 = getRouge(S1_2, S2_2) r3 = getRouge(S1_3, S2_3) r4 = getRouge(S1_4, S2_4) r5 = getRouge(S1_5, S2_5) r6 = getRouge(S1_6, S2_6) body.append(r1) body.append(r2) body.append(r3) body.append(r4) body.append(r5) body.append(r6) print body fio.WriteMatrix('log.txt', body)
def getOracleRougeSplit(oracledir, np, L, metric, outputdir): #sheets = range(0,1) sheets = range(0,12) body = [] for i, sheet in enumerate(sheets): week = i + 1 #Add a cache to make it faster Cache = {} cachefile = oracledir + str(week) + '/' + 'cache.json' print cachefile if fio.IsExist(cachefile): with open(cachefile, 'r') as fin: Cache = json.load(fin) row = [] for type in ['POI', 'MP', 'LP']: row.append(week) #read TA's summmary reffile = oracledir + str(week) + '/' + type + '.ref.summary' lines = fio.ReadFile(reffile) ref = [line.strip() for line in lines] Round = 1 while True: sumfile = oracledir + str(week) + '/' + type + '.' + str(np) + '.L' + str(L) + "." + str(metric) + '.R' + str(Round) +'.summary' if not fio.IsExist(sumfile): break Round = Round + 1 Round = Round - 1 sumfile = oracledir + str(week) + '/' + type + '.' + str(np) + '.L' + str(L) + "." + str(metric) + '.R' + str(Round) +'.summary' if fio.IsExist(sumfile): import os ssfile = oracledir + str(week) + '/' + type + '.' + str(np) + '.L' + str(L) + ".summary" cmd = 'cp ' + sumfile + ' ' + ssfile print cmd os.system(cmd) lines = fio.ReadFile(sumfile) TmpSum = [line.strip() for line in lines] cacheKey = getKey(ref, TmpSum) if cacheKey in Cache: scores = Cache[cacheKey] print "Hit" else: print "Miss", cacheKey print sumfile scores = getRouge(ref, TmpSum) Cache[cacheKey] = scores #exit() row = row + scores else: row = row + [0]*len(RougeHeader) body.append(row) print body print "RougeHeader", len(RougeHeader) header = ['week'] + RougeHeader*3 row = [] row.append("average") print len(header) for i in range(1, len(header)): scores = [float(xx[i]) for xx in body] row.append(numpy.mean(scores)) body.append(row) fio.WriteMatrix(outputdir + "rouge." + str(np) + '.L' + str(L) + "." + str(metric) + ".txt", body, header)
def writegraph_leave_one_lecture_out(model_dir, phrasedir, modelname='svr', traincourse=None): from sklearn import svm from sklearn.metrics import mean_squared_error, precision_recall_fscore_support, accuracy_score import QPS_simlearning sim_extractor = Similarity() allfeatures = sorted(sim_extractor.features.keys()) features = allfeatures name = '_'.join(features) lectures = annotation.Lectures for i, lec in enumerate(lectures): test = [lec] print test model_file = os.path.join(model_dir, '%d_%s.model' % (lec, name)) #model_file = os.path.join(model_dir, '%s_%s.model'%('IE256_2016', name)) with open(model_file, 'rb') as handle: clf = pickle.load(handle) path = os.path.join(phrasedir, str(lec)) for q in ['q1', 'q2']: test_X, test_Y = QPS_simlearning.combine_files_test(phrasedir, test, features, prompts=[q]) predict_Y = clf.predict(test_X) #write the output phrasefile = os.path.join(path, "%s.%s.key" % (q, method)) phrases = fio.LoadList(phrasefile) assert (len(predict_Y) == len(phrases) * len(phrases)) k = 0 body = [] for i, p1 in enumerate(phrases): for j, p2 in enumerate(phrases): if j <= i: k += 1 continue #undirect graph if modelname == 'svm': if predict_Y[k] == 1.0: #row = [i,j, '%.1f'%predict_Y[k]] row = [i, j] body.append(row) else: row = [i, j, '%.2f' % predict_Y[k]] body.append(row) k += 1 output = os.path.join( path, "%s.%s.%s%s" % (q, method, modelname, net_exe)) fio.WriteMatrix(output, body)
def train_leave_one_lecture_out_svm(model_dir, name='simlearn_cv'): # model_dir = '../data/IE256/%s/model/%s/'%(system, name) # fio.NewPath(model_dir) # # outputdir = '../data/IE256/%s/extraction/%s_output/'%(system, name) # fio.NewPath(outputdir) sim_extractor = Similarity() allfeatures = sorted(sim_extractor.features.keys()) #for k in range(len(allfeatures)+1): k = len(allfeatures) if True: #for k in range(len(allfeatures)): #if allfeatures[k] != 'optimumComparerLSATasa': continue if k == len(allfeatures): #use all features features = allfeatures else: features = [allfeatures[k]] #features = allfeatures[0:k] + allfeatures[k+1:] name = '_'.join(features) lectures = annotation.Lectures dict = defaultdict(int) MSE = [] for i, lec in enumerate(lectures): train = [x for x in lectures if x != lec] test = [lec] print train print test model_file = os.path.join(model_dir, '%d_%s.model' % (lec, name)) if fio.IsExist(model_file): with open(model_file, 'rb') as handle: clf = pickle.load(handle) else: train_X, train_Y = combine_files(train, features) clf = svm.SVC() clf.fit(train_X, train_Y) with open(model_file, 'wb') as handle: pickle.dump(clf, handle) for q in ['q1', 'q2']: test_X, test_Y = combine_files(test, features, prompts=[q]) predict_Y = clf.predict(test_X) prf = precision_recall_fscore_support(test_Y, predict_Y, average='weighted') accuracy = accuracy_score(test_Y, predict_Y) MSE.append([lec, q, accuracy] + [prf[0], prf[1], prf[2]]) output = '../data/%s/simlearning.cv.svm.%s.txt' % (course, name) fio.WriteMatrix(output, MSE, header=[ 'lec', 'prompt', 'accuracy', 'precision', 'recall', 'f-score' ])
def extractPhraseFromAnnotation(phrasedir, annotator, summarydir=None): for doc, lec, annotator in annotation.generate_all_files( annotation.datadir + 'json/', '.json', anotators=annotator, lectures=annotation.Lectures): print doc #load task task = annotation.Task() task.loadjson(doc) path = phrasedir + str(lec) + '/' fio.NewPath(path) #Add a cache to make it faster Cache = {} cachefile = phrasedir + str(lec) + '/' + 'cache.json' if fio.IsExist(cachefile): with open(cachefile, 'r') as fin: Cache = json.load(fin) for prompt in ['q1', 'q2']: filename = path + prompt + '.' + method + '.key' cluster_output = path + prompt + '.cluster.kmedoids.sqrt.oracle.%s' % method if summarydir: fio.NewPath(os.path.join(summarydir, str(lec))) summary_file = os.path.join(summarydir, str(lec), '%s.summary' % prompt) body = [] if summarydir: summaries = [] phrase_summary_dict = task.get_phrase_summary_textdict(prompt) extracted_phrases = [] phrase_annotation = task.get_phrase_annotation(prompt) for rank in sorted(phrase_annotation): rank_phrases = [] phrases = phrase_annotation[rank] for phrasedict in phrases: phrase = phrasedict['phrase'].lower() extracted_phrases.append(phrase) rank_phrases.append(phrase) row = [phrase, rank] body.append(row) if summarydir: rank_summary = phrase_summary_dict[rank] max_summary = get_max_phrase_by_ROUGE( rank_summary, rank_phrases, Cache) print max_summary summaries.append(max_summary) fio.SaveList(extracted_phrases, filename) fio.WriteMatrix(cluster_output, body, header=None) if summarydir: fio.SaveList(summaries, summary_file) with open(cachefile, 'w') as outfile: json.dump(Cache, outfile, indent=2)
def extractStatistics(annotator, output): students = set() body = [] for doc, lec, annotator in annotation.generate_all_files( annotation.datadir + 'json/', '.json', anotators=annotator, lectures=annotation.Lectures): print doc #load task task = annotation.Task() task.loadjson(doc) for prompt in ['q1', 'q2']: #for each lecture, prompt row = [lec, prompt] stu = set() #number of students wc = 0.0 dict = {} raw_responses = task.get_raw_response(prompt) for response_row in raw_responses[1:]: student_id, response = response_row[ 'student_id'], response_row['response'] student_id = student_id.lower() if student_id not in dict: dict[student_id] = [] dict[student_id].append(response) students.add(student_id) stu.add(student_id) wc += len(response.split()) response_number = len(dict) row.append(response_number) #number of responses row.append(wc) #word count row.append(wc / response_number) #averaged number of words per response phrase_summary_dict = task.get_phrase_summary_textdict(prompt) extracted_phrases = [] phrase_annotation = task.get_phrase_annotation(prompt) stu_h = set() ph_c = 0 for rank in sorted(phrase_annotation): phrases = phrase_annotation[rank] ph_c += len(phrases) for phrasedict in phrases: phrase = phrasedict['phrase'].lower() #phrase extracted_phrases.append(phrase) student_id = phrasedict['student_id'].lower().strip() stu_h.add(student_id) row.append(ph_c) #phrase count coverage = stu.intersection(stu_h) coverage_ratio = len(coverage) * 1.0 / len(stu) row.append(coverage_ratio) body.append(row) #add average head = [ 'lec', 'prompt', 'Response', 'Word', 'Word/Response', 'Highlights', 'Coverage' ] row = ['', 'ave'] for i in range(2, len(head)): scores = [float(xx[i]) for xx in body] row.append(np.mean(scores)) body.append(row) #add std row = ['', 'std'] for i in range(2, len(head)): scores = [float(xx[i]) for xx in body] row.append(np.std(scores)) body.append(row) fio.WriteMatrix(output, body, head) print(len(students))
def getRouge(datadir, maxWeek, output): sheets = range(0, maxWeek) body = [] allbody = [] #Krange = range(1, 25) #Krange = range(1, 25) Krange = [gK] for sheet in sheets: week = sheet + 1 dir = datadir + str(week) + '/' for type in ['q1', 'q2']: maxS = 0 maxK = -1 maxScore = [] Cache = {} cachefile = os.path.join(datadir, str(week), 'cache.json') print cachefile if fio.IsExist(cachefile): with open(cachefile, 'r') as fin: Cache = json.load(fin) allrow = [week] #Krange = [np.random.randint(1, 25)] for K in Krange: summary_file = dir + type + '.%d.summary' % K print summary_file if not fio.IsExist(summary_file): print summary_file continue #read TA's summmary refs = [] for i in range(2): reffile = os.path.join(datadir, str(week), type + '.ref.%d' % i) if not fio.IsExist(reffile): print reffile continue lines = fio.ReadFile(reffile) ref = [line.strip() for line in lines] refs.append(ref) if len(refs) == 0: continue lstref = refs[0] + refs[1] lines = fio.ReadFile(summary_file) TmpSum = [line.strip() for line in lines] cacheKey = OracleExperiment.getKey(lstref, TmpSum) if cacheKey in Cache: scores = Cache[cacheKey] print "Hit" else: print "Miss" print summary_file scores = OracleExperiment.getRouge_IE256(refs, TmpSum) Cache[cacheKey] = scores s = float(scores[RIndex]) allrow.append(s) if s >= maxS: maxS = s maxScore = scores maxK = K if maxK == -1: continue row = [week] row = row + maxScore + [maxK] body.append(row) allrow.append(maxK) allbody.append(allrow) try: fio.SaveDict2Json(Cache, cachefile) except: #fio.SaveDict(Cache, cachefile + '.dict') pass header = ['id'] + RougeHeader row = ['ave'] for i in range(1, len(header)): scores = [float(xx[i]) for xx in body] row.append(numpy.mean(scores)) body.append(row) fio.WriteMatrix(output, body, header) fio.WriteMatrix(output + '.all', allbody, ['week'] + Krange)
def getRouge(datadir, maxWeek, output): print datadir sheets = range(0, maxWeek) body = [] for sheet in sheets: week = sheet + 1 dir = datadir + str(week) + '/' for type in ['q1', 'q2']: summary_file = dir + type + "." + 'summary' print summary_file if not fio.IsExist(summary_file): print summary_file continue Cache = {} cachefile = os.path.join(datadir, str(week), 'cache.json') print cachefile if fio.IsExist(cachefile): with open(cachefile, 'r') as fin: Cache = json.load(fin) #read TA's summmary refs = [] for i in range(2): reffile = os.path.join(datadir, str(week), type + '.ref.%d' % i) if not fio.IsExist(reffile): print reffile continue lines = fio.ReadFile(reffile) ref = [line.strip() for line in lines] refs.append(ref) if len(refs) == 0: continue lstref = refs[0] + refs[1] lines = fio.ReadFile(summary_file) TmpSum = [line.strip() for line in lines] cacheKey = OracleExperiment.getKey(lstref, TmpSum) if cacheKey in Cache: scores = Cache[cacheKey] print "Hit" else: print "Miss" print summary_file scores = OracleExperiment.getRouge_IE256(refs, TmpSum) Cache[cacheKey] = scores row = [week] row = row + scores body.append(row) try: fio.SaveDict2Json(Cache, cachefile) except Exception as e: #fio.SaveDict(Cache, cachefile + '.dict') print e header = ['id'] + RougeHeader row = ['ave'] for i in range(1, len(header)): scores = [float(xx[i]) for xx in body] row.append(numpy.mean(scores)) body.append(row) fio.WriteMatrix(output, body, header)
def gather_rouge(output): courses = ['IE256', 'IE256_2016', 'CS0445'] rouges = [ ('LexRank', 'QPS_NP', 'rouge_LexRank'), ('PhraseSum', 'QPS_NP', 'rouge_crf_optimumComparerLSATasa'), ('SequenceSum', 'QPS_combine_coling', 'rouge_crf_optimumComparerLSATasa'), ('SimSum', 'QPS_combine_coling', 'rouge_crf_svm'), ('CDSum', 'QPS_combine_coling', 'rouge_crf_ct.svm.default'), ] baseline1 = ('PhraseSum', 'QPS_NP', 'rouge_crf_optimumComparerLSATasa') baseline2 = ('SequenceSum', 'QPS_combine_coling', 'rouge_crf_optimumComparerLSATasa') Header = [ 'course', 'name', 'R1-R', 'R1-P', 'R1-F', 'R2-R', 'R2-P', 'R2-F', 'RSU4-R', 'RSU4-P', 'RSU4-F', ] ROUGE_Head = [ 'id', 'R1-R', 'R1-P', 'R1-F', 'R2-R', 'R2-P', 'R2-F', 'RSU4-R', 'RSU4-P', 'RSU4-F' ] ROUGE_index = [ ROUGE_Head.index(name) for name in ROUGE_Head if name != 'id' ] xbody = [] for course in courses: for name, model, method in rouges: datadir = '../data/%s/' % course filename = os.path.join(datadir, model, "%s.txt" % method) if not fio.IsExist(filename): continue baseline1_name = os.path.join(datadir, baseline1[1], "%s.txt" % baseline1[2]) baseline2_name = os.path.join(datadir, baseline2[1], "%s.txt" % baseline2[2]) if name in ['LexRank', 'SequenceSum', 'SimSum', 'CDSum']: pvalues1 = get_pvalues(filename, baseline1_name, ROUGE_index) else: pvalues1 = [1] * len(ROUGE_index) if name in ['SimSum', 'CDSum']: pvalues2 = get_pvalues(filename, baseline2_name, ROUGE_index) else: pvalues2 = [1] * len(ROUGE_index) head, body = fio.ReadMatrix(filename, hasHead=True) row = [course, name] row += [ '%.3f%s%s' % (float(x), '*' if pvalues1[i] < 0.05 else '', '+' if pvalues2[i] < 0.05 else '') for i, x in enumerate(body[-1][1:]) ] xbody.append(row) fio.WriteMatrix(output, xbody, Header)