def log_likelihood (data, model, query):
'''
Return the log-likelihood of the given data W according to the model
and the parameters inferred for the entries in W stored in the
queryState object.
This deliberately excludes links
'''
return lda.log_likelihood(data, model.ldaModel, query.ldaQuery)
def testCrossValPerplexityOnRealDataWithLdaInc(self):
ActiveFolds = 3
dtype = np.float64 # DTYPE
rd.seed(0xBADB055)
data = DataSet.from_files(words_file=AclWordPath, links_file=AclCitePath)
data.convert_to_dtype(dtype)
data.prune_and_shuffle(min_doc_len=MinDocLen, min_link_count=MinLinkCount)
# Initialise the model
trainPlan = lda.newTrainPlan(iterations=800, logFrequency=10, fastButInaccurate=False, debug=False)
queryPlan = lda.newTrainPlan(iterations=50, logFrequency=5, fastButInaccurate=False, debug=False)
topicCounts = [30, 35, 40, 45, 50] # [5, 10, 15, 20, 25, 30, 35, 40, 45, 50]
for K in topicCounts:
trainPerps = []
queryPerps = []
for fold in range(ActiveFolds): # range(NumFolds):
trainData, queryData = data.cross_valid_split(fold, NumFolds)
model = lda.newModelAtRandom(trainData, K, dtype=dtype)
query = lda.newQueryState(trainData, model)
# Train the model, and the immediately save the result to a file for subsequent inspection
model, trainResult, (_, _, _) = lda.train (trainData, model, query, trainPlan)
like = lda.log_likelihood(trainData, model, trainResult)
perp = perplexity_from_like(like, trainData.word_count)
trainPerps.append(perp)
estData, evalData = queryData.doc_completion_split()
query = lda.newQueryState(estData, model)
model, queryResult = lda.query(estData, model, query, queryPlan)
like = lda.log_likelihood(evalData, model, queryResult)
perp = perplexity_from_like(like, evalData.word_count)
queryPerps.append(perp)
trainPerps.append(sum(trainPerps) / ActiveFolds)
queryPerps.append(sum(queryPerps) / ActiveFolds)
print("K=%d,Segment=Train,%s" % (K, ",".join([str(p) for p in trainPerps])))
print("K=%d,Segment=Query,%s" % (K, ",".join([str(p) for p in queryPerps])))
def testPerplexityOnRealDataWithLdaInc(self):
dtype = np.float64 # DTYPE
rd.seed(0xBADB055)
data = DataSet.from_files(words_file=AclWordPath, links_file=AclCitePath)
with open(AclDictPath, "rb") as f:
d = pkl.load(f)
data.convert_to_dtype(dtype)
data.prune_and_shuffle(min_doc_len=MinDocLen, min_link_count=MinLinkCount)
# IDF frequency for when we print out the vocab later
freq = np.squeeze(np.asarray(data.words.sum(axis=0)))
scale = np.reciprocal(1 + freq)
# Initialise the model
topicCounts = [5, 10, 15, 20, 25, 30, 35, 40, 45, 50]
perps = []
for K in topicCounts:
model = lda.newModelAtRandom(data, K, dtype=dtype)
queryState = lda.newQueryState(data, model)
trainPlan = lda.newTrainPlan(iterations=800, logFrequency=10, fastButInaccurate=False, debug=False)
# Train the model, and the immediately save the result to a file for subsequent inspection
model, query, (bndItrs, bndVals, bndLikes) = lda.train (data, model, queryState, trainPlan)
# with open(newModelFileFromModel(model), "wb") as f:
# pkl.dump ((model, query, (bndItrs, bndVals, bndLikes)), f)
# Print out the most likely topic words
# scale = np.reciprocal(1 + np.squeeze(np.array(data.words.sum(axis=0))))
# vocab = lda.wordDists(model)
# topWordCount = 10
# kTopWordInds = [self.topWordInds(vocab[k,:], topWordCount) for k in range(K)]
like = lda.log_likelihood(data, model, query)
perp = perplexity_from_like(like, data.word_count)
perps.append(perp)
print ("K = %2d : Perplexity = %f\n\n" % (K, perp))
#
# for k in range(model.K):
# print("\nTopic %d\n=============================" % k)
# print("\n".join("%-20s\t%0.4f" % (d[kTopWordInds[k][c]], vocab[k][kTopWordInds[k][c]]) for c in range(topWordCount)))
# Plot the evolution of the bound during training.
fig, ax1 = plt.subplots()
ax1.plot(topicCounts, perps, 'b-')
ax1.set_xlabel('Topic Count')
ax1.set_ylabel('Perplexity', color='b')
fig.show()
plt.show()
def log_likelihood (data, model, query):
'''
Return the log-likelihood of the given data W according to the model
and the parameters inferred for the entries in W stored in the
queryState object.
This deliberately excludes links
'''
if model.method == TF_IDF:
return 0
elif model.method == LDA:
return lda.log_likelihood(data, model.ldaModel, query=None, topicDistOverride=query.reps ** 2)
else:
raise ValueError ("Unknown method %s " % model.method)
