def histoDiversity(theData, theType, outFile):
draftDict = defaultdict(int)
finalDict = defaultdict(int)
for line in theData:
name = line[1]
if (0 != theType) and (int(theType) != int(name[-1])): continue
draftDiversity = float(line[4]) / float(line[3])
finalDiversity = float(line[6]) / float(line[5])
draftRounded = round(100 * draftDiversity)
finalRounded = round(100 * finalDiversity)
draftDict[int(draftRounded)] += 1
finalDict[int(finalRounded)] += 1
# print('%s %s %s %3d %3d' % \
# (name, name[-1], theType, int(draftRounded), (finalRounded) ))
which = 'DRAFT'
headerInfo = '\nInput from file: %s\n' % (inFileName)
headerInfo = 'Diversity is quotient of #uniq words and #words for ' + which + '\n'
headerInfo += 'Column 1: This quotient times 100\n'
headerInfo += 'Column 2: Percent of papers with this quotient\n'
headerInfo += 'Column 3: Raw number of papers with this quotient\n'
headerInfo += 'Column 4: The histogram\n'
label = 'DIVERSITY FOR TYPE %d\n' % (theType)
label += headerInfo
if len(draftDict) > 0:
histo, shortVersion = Histogram.histoTheData(label, draftDict, 1, 1, outFile)
outFile.write('%s\n' % (histo))
else:
# print('NO PAPERS FOR %s\n' % (label))
outFile.write('NO PAPERS FOR %s\n' % (label))
which = 'FINAL'
headerInfo = '\nInput from file: %s\n' % (inFileName)
headerInfo = 'Diversity is quotient of #uniq words and #words for ' + which + '\n'
headerInfo += 'Column 1: This quotient times 100\n'
headerInfo += 'Column 2: Percent of papers with this quotient\n'
headerInfo += 'Column 3: Raw number of papers with this quotient\n'
headerInfo += 'Column 4: The histogram\n'
label = 'DIVERSITY FOR TYPE %d\n' % (theType)
label += headerInfo
if len(finalDict) > 0:
histo, shortVersion = Histogram.histoTheData(label, finalDict, 1, 1, outFile)
outFile.write('%s\n' % (histo))
else:
# print('NO PAPERS FOR %s\n' % (label))
outFile.write('NO PAPERS FOR %s\n' % (label))
def histoWordCounts(theData, theType, which, outFile):
localDict = defaultdict(int)
for key, sent in sorted(theData.items()):
if (0 != theType) and (theType != sent.getType()): continue
if which != sent.getWhich(): continue
name = sent.getName()
sentenceLength = sent.getSentenceLength()
localDict[name] += sentenceLength
# print('SENT %14s %3d %3d' % (name, sentenceLength, localDict[name]))
localDict2 = defaultdict(int)
for name, wordCount in sorted(localDict.items()):
# print('%6s %5d' % (name, wordCount))
localDict2[wordCount] += 1
# for numSents, freq in sorted(localDict2.items()):
# print('%5d %5d' % (numSents, freq))
headerInfo = '\nInput from file: %s\n' % (inFileName)
headerInfo += 'Column 1: # of words ' + which + '\n'
headerInfo += 'Column 2: percent of total\n'
headerInfo += 'Column 3: raw numbers of total\n'
headerInfo += 'Column 4: the histogram\n'
label = 'WORD COUNTS FOR TYPE %d ' % (theType)
print('%s %s' % (label, which))
label += headerInfo
if len(localDict2) > 0:
histo, shortVersion = Histogram.histoTheData(label, localDict2, 50, 50, outFile)
# print('%s\n' % (histo))
outFile.write('%s\n' % (histo))
# shortStuff.append([type, shortVersion])
else:
# print('NO SENTENCES FOR %s\n' % (label))
outFile.write('NO SENTENCES FOR %s\n' % (label))
def histoInsertionsByPara(theData, theType, which, outFile):
localDict = defaultdict(int)
for key, sent in sorted(theData.items()):
if (0 != theType) and (theType != sent.getType()): continue
if which != sent.getWhich(): continue
if sent.isAligned(): continue
paraNum = sent.getRightParaSub()
localDict[paraNum] += 1
# for dist, freq in sorted(localDict.items()):
# print('%5d %5d' % (dist, freq))
headerInfo = '\nInput from file: %s\n' % (inFileName)
headerInfo += 'Column 1: paragraph numbers\n'
headerInfo += 'Column 2: percent of total\n'
headerInfo += 'Column 3: raw numbers of total\n'
headerInfo += 'Column 4: the histogram\n'
label = 'INSERTIONS BY PARAGRAPH INTO FINAL FOR TYPE %d ' % (theType)
print('%s %s' % (label, which))
label += headerInfo
if len(localDict) > 0:
histo, shortVersion = Histogram.histoTheData(label, localDict, 1, 1, outFile)
# print('%s\n' % (histo))
outFile.write('%s\n' % (histo))
# shortStuff.append([type, shortVersion])
else:
# print('NO INSERTIONS FOR %s\n' % (label))
outFile.write('NO INSERTIONS FOR %s\n' % (label))
def histoAlignmentFractionsByLevel(theData, theType, which, outFile):
localDict = defaultdict(int)
for key, sent in sorted(theData.items()):
if (0 != theType) and (theType != sent.getType()): continue
if which != sent.getWhich(): continue
name = sent.getName()
level = int(sent.getAlignmentLevel())
if -1 == level: level = 999
localDict[level] += 1
# for level, freqs in sorted(localDict.items()):
# print('%5d %5d' % (level, freqs))
headerInfo = '\nInput from file: %s\n' % (inFileName)
headerInfo += 'Column 1: fraction aligned in '+ which + '\n'
headerInfo += 'Column 2: percent of total\n'
headerInfo += 'Column 3: raw numbers of total\n'
headerInfo += 'Column 4: the histogram\n'
label = 'ALIGNMENTS FOR TYPE %d ' % (theType)
print('%s %s' % (label, which))
label += headerInfo
if len(localDict) > 0:
histo, shortVersion = Histogram.histoTheData(label, localDict, 1, 1, outFile)
# print('%s\n' % (histo))
outFile.write('%s\n' % (histo))
# shortStuff.append([type, shortVersion])
else:
# print('NO PAPERS FOR %s\n' % (label))
outFile.write('NO PAPERS FOR %s\n' % (label))
def histoInsertionsByEditDistFrac(theData, theType, which, outFile):
paperSet = set()
paperCount = 0
beginningInsertionCount = 0
localDict = defaultdict(int)
for key, sent in sorted(theData.items()):
if (0 != theType) and (theType != sent.getType()):
continue
if which != sent.getWhich():
continue
keySplit = key.split()
paperSet.add(keySplit[0])
if sent.isAligned():
continue
prevDist = sent.getPreviousDistance()
if prevDist < 0:
prevDist = 899
if "FINAL" == which:
if 0 == sent.getRightParaSub() and 0 == sent.getRightSentSub():
beginningInsertionCount += 1
# print('INITIAL INSERTION %s %s' % (key, sent))
localDict[prevDist] += 1
# for dist, freq in sorted(localDict.items()):
# print('%5d %5d' % (dist, freq))
paperCount = len(paperSet)
# print('PAPER COUNT %3s' % (paperCount))
headerInfo = "\nInput from file: %s\n" % (inFileName)
headerInfo += "There were initial insertions in %3d of %3d papers\n" % (beginningInsertionCount, paperCount)
headerInfo += "Column 1: edit dist fracs before insertion\n"
# headerInfo += " the '899' means insertion at beginning of paper\n"
headerInfo += "Column 2: percent of total\n"
headerInfo += "Column 3: raw numbers of total\n"
headerInfo += "Column 4: the histogram\n"
label = "INSERTIONS BY EDIT DIST FRAC OF PREVIOUS SENTENCE FOR TYPE %d " % (theType)
print("%s %s" % (label, which))
label += headerInfo
if len(localDict) > 0:
histo, shortVersion = Histogram.histoTheData(label, localDict, 4, 4, outFile)
# print('%s\n' % (histo))
outFile.write("%s\n" % (histo))
# shortStuff.append([type, shortVersion])
else:
# print('NO INSERTIONS FOR %s\n' % (label))
outFile.write("NO INSERTIONS FOR %s\n" % (label))
def histoSentenceCounts(theData, theType, which, outFile):
localDict = defaultdict(int)
for key, sent in sorted(theData.items()):
if (0 != theType) and (theType != sent.getType()):
continue
if which != sent.getWhich():
continue
name = sent.getName()
if "DRAFT" == which:
sentNum = int(sent.getLeftSentSub())
else:
sentNum = int(sent.getRightSentSub())
if sentNum > localDict[name]:
localDict[name] = sentNum
localDict2 = defaultdict(int)
for name, maxValue in sorted(localDict.items()):
# print('%6s %5d' % (name, maxValue))
localDict2[maxValue] += 1
# for numSents, freq in sorted(localDict2.items()):
# print('%5d %5d' % (numSents, freq))
headerInfo = "\nInput from file: %s\n" % (inFileName)
headerInfo += "Column 1: # of paragraphs " + which + "\n"
headerInfo += "Column 2: percent of total\n"
headerInfo += "Column 3: raw numbers of total\n"
headerInfo += "Column 4: the histogram\n"
label = "SENTENCE COUNTS FOR TYPE %d " % (theType)
print("%s %s" % (label, which))
label += headerInfo
if len(localDict2) > 0:
histo, shortVersion = Histogram.histoTheData(label, localDict2, 2, 2, outFile)
# print('%s\n' % (histo))
outFile.write("%s\n" % (histo))
# shortStuff.append([type, shortVersion])
else:
# print('NO SENTENCES FOR %s\n' % (label))
outFile.write("NO SENTENCES FOR %s\n" % (label))
def histoDeletionsByPara(theData, theType, which, lastParaNumDict, theFilter, outFile):
localDict = defaultdict(int)
for key, sent in sorted(theData.items()):
if (0 != theType) and (theType != sent.getType()): continue
if which != sent.getWhich(): continue
if sent.isAligned(): continue
paraNum = sent.getLeftParaSub()
docName = key.split()[0]
lastParaKey = docName + ' ' + which
lastParaNum = lastParaNumDict[lastParaKey]
if ('FIRST' == theFilter) and (0 != paraNum): continue
if ('LAST' == theFilter) and (lastParaNum != paraNum):
continue
if ('MIDDLE' == theFilter):
if (0 == paraNum) or (lastParaNum == paraNum):
continue
localDict[paraNum] += 1
# for dist, freq in sorted(localDict.items()):
# print('%5d %5d' % (dist, freq))
headerInfo = '\nInput from file: %s for %s\n' % (inFileName, theFilter)
headerInfo += 'Column 1: paragraph numbers\n'
headerInfo += 'Column 2: percent of total\n'
headerInfo += 'Column 3: raw numbers of total\n'
headerInfo += 'Column 4: the histogram\n'
label = 'DELETIONS BY PARAGRAPH FROM DRAFT FOR TYPE %d ' % (theType)
print('%s %s' % (label, which))
label += headerInfo
if len(localDict) > 0:
histo, shortVersion = Histogram.histoTheData(label, localDict, 1, 1, outFile)
# print('%s\n' % (histo))
outFile.write('%s\n' % (histo))
# shortStuff.append([type, shortVersion])
else:
# print('NO DELETIONS FOR %s\n' % (label))
outFile.write('NO DELETIONS FOR %s\n' % (label))
def histoParagraphCounts(theData, theType, which, outFile):
localDict = defaultdict(int)
for key, sent in sorted(theData.items()):
if (0 != theType) and (theType != sent.getType()): continue
if which != sent.getWhich(): continue
name = sent.getName()
if 'DRAFT' == which:
paraNum = int(sent.getLeftParaSub())
else:
paraNum = int(sent.getRightParaSub())
if paraNum > localDict[name]:
localDict[name] = paraNum
localDict2 = defaultdict(int)
for name, maxValue in sorted(localDict.items()):
# print('%6s %5d' % (name, maxValue))
localDict2[maxValue] += 1
# for numParas, freq in sorted(localDict2.items()):
# print('%5d %5d' % (numParas, freq))
headerInfo = '\nInput from file: %s\n' % (inFileName)
headerInfo += 'Column 1: # of paragraphs ' + which + '\n'
headerInfo += 'Column 2: percent of total\n'
headerInfo += 'Column 3: raw numbers of total\n'
headerInfo += 'Column 4: the histogram\n'
label = 'PARAGRAPH COUNTS FOR TYPE %d ' % (theType)
print('%s %s' % (label, which))
label += headerInfo
if len(localDict2) > 0:
histo, shortVersion = Histogram.histoTheData(label, localDict2, 1, 1, outFile)
# print('%s\n' % (histo))
outFile.write('%s\n' % (histo))
# shortStuff.append([type, shortVersion])
else:
# print('NO PARAGRAPHS FOR %s\n' % (label))
outFile.write('NO PARAGRAPHS FOR %s\n' % (label))
def histoAlignmentLevels(theData, theType, which, outFile):
localDict = defaultdict(int)
for key, sent in sorted(theData.items()):
if (0 != theType) and (theType != sent.getType()):
continue
if which != sent.getWhich():
continue
name = sent.getName()
level = int(sent.getAlignmentLevel())
if level > localDict[name]:
localDict[name] = level
localDict2 = defaultdict(int)
for name, maxValue in sorted(localDict.items()):
# print('%6s %5d' % (name, maxValue))
localDict2[maxValue] += 1
# for levels, freqs in sorted(localDict2.items()):
# print('%5d %5d' % (levels, freqs))
headerInfo = "\nInput from file: %s\n" % (inFileName)
headerInfo += "Column 1: last alignment with changes\n"
headerInfo += "Column 2: percent of total\n"
headerInfo += "Column 3: raw numbers of total\n"
headerInfo += "Column 4: the histogram\n"
label = "LAST ALIGNMENT FOR TYPE %d " % (theType)
print("%s %s" % (label, which))
label += headerInfo
if len(localDict2) > 0:
histo, shortVersion = Histogram.histoTheData(label, localDict2, 1, 1, outFile)
# print('%s\n' % (histo))
outFile.write("%s\n" % (histo))
# shortStuff.append([type, shortVersion])
else:
# print('NO ALIGNMENTS FOR %s\n' % (label))
outFile.write("NO ALIGNMENTS FOR %s\n" % (label))
def histoEditDistance(theData, theType, which, outFile):
localDict = defaultdict(int)
for key, sent in sorted(theData.items()):
if (0 != theType) and (theType != sent.getType()): continue
if which != sent.getWhich(): continue
distRounded = int(round(sent.getEditDistFracOfWorst() * 100.0))
if distRounded < 0: distRounded = 999
localDict[distRounded] += 1
# for dist, freq in sorted(localDict.items()):
# print('%5d %5d' % (dist, freq))
headerInfo = '\nInput from file: %s\n' % (inFileName)
headerInfo += 'Column 1: % change in aligned sentences from draft to final\n'
headerInfo += " 'UNALIGN' means deletions from draft, insertions into final\n"
headerInfo += 'Column 2: % of sentences with that change\n'
headerInfo += 'Column 3: raw numbers of sentences with that change\n'
headerInfo += 'Column 4: the histogram\n'
label = 'EDIT DISTANCE COMPARISONS FOR ' + which + ' OF TYPE %d ' % (theType)
print('%s %s' % (label, which))
label += headerInfo
if len(localDict) > 0:
histo, shortVersion = Histogram.histoTheData(label, localDict, 4, 4, outFile)
# print('%s\n' % (histo))
outFile.write('%s\n' % (histo))
# shortStuff.append([type, shortVersion])
else:
# print('NO EDIT DIST FRACS FOR %s\n' % (label))
outFile.write('NO EDIT DIST FRACS FOR %s\n' % (label))
if 0 != theType: return
localTypeListDict = defaultdict(list)
for key in range(0, 5):
localTypeListDict[key] = [0]
for key, sent in sorted(theData.items()):
if which != sent.getWhich(): continue
thisType = sent.getType()
distRounded = int(round(sent.getEditDistFracOfWorst() * 100.0))
if distRounded < 0: distRounded = -5
# do the actual type
thisList = localTypeListDict[thisType]
thisList.append(distRounded)
localTypeListDict[thisType] = thisList
# do the "all" type, which is type 0
thisList = localTypeListDict[0]
thisList.append(distRounded)
localTypeListDict[0] = thisList
multiset = []
for key, value in sorted(localTypeListDict.items()):
multiset.append(value)
# print('TYPE %d' % (key))
# print('type %d %s' % (key, value))
fig = plt.figure()
ax = fig.add_subplot(111)
numBins = 50
numBins = 25
ax.hist(multiset, numBins, color=['green','red','blue','lime','orange'], \
label = ['0', '1', '2', '3', '4'], alpha=0.8)
ax.legend(prop={'size': 10})
ax.set_title('Edit Distance Histograms ' + which)
# plt.show()
plt.savefig('EditDistHistograms' + which)
def histoDiversityByWordCount(theData, theType, whichLeft, whichRight, outFile):
allDict = defaultdict(int)
nonstopDict = defaultdict(int)
######################################################################
## REMEMBER that general diversity is all and nonstopwords
## REMEMBER that student diversity is all for draft and final
ALLDIVERSITY = 6
NONSTOPDIVERSITY = 9
#ALLDIVERSITY = 5
#NONSTOPDIVERSITY = 8
allRoundedList = []
nonstopRoundedList = []
for line in theData:
name = line[1]
name = name.replace('.txt', '')
if (0 != theType) and (int(theType) != int(name[-1])): continue
allDiversity = float(line[ALLDIVERSITY])
nonstopDiversity = float(line[NONSTOPDIVERSITY])
allRounded = round(100 * allDiversity)
allRoundedList.append(allRounded)
nonstopRounded = round(100 * nonstopDiversity)
nonstopRoundedList.append(nonstopRounded)
allDict[int(allRounded)] += 1
nonstopDict[int(nonstopRounded)] += 1
# print('%s %s %s %3d %3d' % \
# (name, name[-1], theType, int(draftRounded), (finalRounded) ))
# which = 'ALL'
headerInfo = '\nInput from file: %s\n' % (inFileName)
headerInfo = 'Diversity is quotient of #uniq words and #words for ' + whichLeft + '\n'
headerInfo += 'Column 1: This quotient times 100\n'
headerInfo += 'Column 2: Percent of papers with this quotient\n'
headerInfo += 'Column 3: Raw number of papers with this quotient\n'
headerInfo += 'Column 4: The histogram\n'
label = 'DIVERSITY FOR TYPE %d\n' % (theType)
label += headerInfo
if len(allDict) > 0:
histo, shortVersion = Histogram.histoTheData(label, allDict, 1, 1, outFile)
outFile.write('%s\n' % (histo))
else:
# print('NO PAPERS FOR %s\n' % (label))
outFile.write('NO PAPERS FOR %s\n' % (label))
# which = 'NONSTOP'
headerInfo = '\nInput from file: %s\n' % (inFileName)
headerInfo = 'Diversity is quotient of #uniq words and #words for ' + whichRight + '\n'
headerInfo += 'Column 1: This quotient times 100\n'
headerInfo += 'Column 2: Percent of papers with this quotient\n'
headerInfo += 'Column 3: Raw number of papers with this quotient\n'
headerInfo += 'Column 4: The histogram\n'
label = 'DIVERSITY FOR TYPE %d\n' % (theType)
label += headerInfo
if len(nonstopDict) > 0:
histo, shortVersion = Histogram.histoTheData(label, nonstopDict, 1, 1, outFile)
outFile.write('%s\n' % (histo))
else:
# print('NO PAPERS FOR %s\n' % (label))
outFile.write('NO PAPERS FOR %s\n' % (label))
print(allRoundedList)
if len(allRoundedList) > 0:
fig = plt.figure()
ax = fig.add_subplot(111)
numBins = 50
# ax.hist(allRoundedList,numBins,color='green',alpha=0.8)
# ax.hist(nonstopRoundedList,numBins,color='red',alpha=0.8)
# plt.show()
multiset = [allRoundedList, nonstopRoundedList]
ax.hist(multiset, numBins, color=['green','red'],label = ['All', 'Nonstop'], alpha=0.8)
ax.legend(prop={'size': 10})
ax.set_title('Allword versus Nonstopword diversity')
plt.show()
