def add_metric(database_id,
dataset,
analysis,
force_import=False,
*args,
**kwargs):
analysis = Analysis.objects.using(database_id).get(dataset__name=dataset,
name=analysis)
try:
metric = PairwiseTopicMetric.objects.using(database_id).get(
name=metric_name, analysis=analysis)
if not force_import:
raise RuntimeError("%s is already in the database for this"
" analysis" % metric_name)
except PairwiseTopicMetric.DoesNotExist:
metric = PairwiseTopicMetric(name=metric_name, analysis=analysis)
metric.save()
conn = sqlite3.connect(kwargs['counts'])
c = conn.cursor()
c.execute('PRAGMA temp_store=MEMORY')
c.execute('PRAGMA synchronous=OFF')
c.execute('PRAGMA cache_size=2000000')
c.execute('PRAGMA journal_mode=OFF')
c.execute('PRAGMA locking_mode=EXCLUSIVE')
c.execute("select words from total_counts")
for row in c:
total_words = float(row[0])
c.execute("select cooccurrences from total_counts")
for row in c:
total_cooccurrences = float(row[0])
topics = list(analysis.topics.all().order_by('number'))
num_words = 10
topicwords = []
wordset = set()
for topic in topics:
words = topic_words(topic, num_words)
topicwords.append(words)
for w in words:
wordset.add(w)
for i, topic1 in enumerate(topics):
print topic1
topic1_words = topicwords[i]
for j, topic2 in enumerate(topics):
print ' ', topic2
topic2_words = topicwords[j]
coherence = pairwise_coherence(topic1_words, topic2_words, c,
total_words, total_cooccurrences)
PairwiseTopicMetricValue.objects.using(database_id).create(
topic1=topic1, topic2=topic2, metric=metric, value=coherence)
