def make_sim():
# Get similarity from the expanded version of ConceptNet.
# This is not blended with anything yet.
conceptnet = divisi2.load('conceptnet_big.pickle') #.normalize_all()
U, S, V = conceptnet.svd(k=100)
sim = divisi2.reconstruct_similarity(U, S, offset=0.1)
return sim
def make_blend(thefile):
conceptnet = divisi2.network.conceptnet_matrix('en')
thegame = divisi2.load(thefile).normalize_all()
blended_matrix = blend(conceptnet, thegame)
u,s,v = blended_matrix().svd()
similarity = divisi2.reconstruct_similarity(u,s)
pd[thefile.split('.')[0]] = similarity
return similarity
def make_blend(thefile):
conceptnet = divisi2.network.conceptnet_matrix('en').normalize_all()
thegame = divisi2.load(thefile).normalize_all()
blended_matrix = blend([conceptnet, thegame], [0.9, 0.1])
u,s,v = blended_matrix.svd()
similarity = divisi2.reconstruct_similarity(u, s) # offset=1.5)
pd.mkdir(thefile.split('.')[0])
pd[thefile.split('.')[0]]['blend'] = similarity
return similarity
def getSimilarity(self, post_normalize=False, pre_normalize=False):
if post_normalize:
if 'sim_post' in self.cache:
return self.cache['sim_post']
U,S,V = self.getSVD()
sim_post = divisi2.reconstruct_similarity(U, S, post_normalize=True)
self.cache['sim_post'] = sim_post
return sim_post
if pre_normalize:
if 'sim_pre' in self.cache:
return self.cache['sim_pre']
U,S,V = self.getSVD(normalized=True)
sim_pre = divisi2.reconstruct_similarity(U, S, post_normalize=False)
self.cache['sim_pre'] = sim_pre
return sim_pre
else:
if 'sim' in self.cache:
return self.cache['sim']
U,S,V = self.getSVD()
sim = divisi2.reconstruct_similarity(U, S, post_normalize=False)
self.cache['sim'] = sim
return sim
def analyze(self):
# TODO: make it possible to blend multiple directories
self._documents_matrix = None
docs, projections, Sigma = self.get_eigenstuff()
magnitudes = np.sqrt(np.sum(np.asarray(projections*projections), axis=1))
if self.is_associative():
spectral = divisi2.reconstruct_activation(projections, Sigma, post_normalize=True, offset=0.0001)
else:
spectral = divisi2.reconstruct_similarity(projections, Sigma,
post_normalize=True, offset=0.0001)
self._step('Calculating stats...')
stats = self.compute_stats(docs, spectral)
results = StudyResults(self, docs, spectral.left, spectral, magnitudes, stats)
return results
def analyze(self):
# TODO: make it possible to blend multiple directories
self._documents_matrix = None
docs, projections, Sigma = self.get_eigenstuff()
magnitudes = np.sqrt(
np.sum(np.asarray(projections * projections), axis=1))
if self.is_associative():
spectral = divisi2.reconstruct_activation(projections,
Sigma,
post_normalize=True,
offset=0.0001)
else:
spectral = divisi2.reconstruct_similarity(projections,
Sigma,
post_normalize=True,
offset=0.0001)
self._step('Calculating stats...')
stats = self.compute_stats(docs, spectral)
results = StudyResults(self, docs, spectral.left, spectral, magnitudes,
stats)
return results
from csc import divisi2
matrix = divisi2.network.conceptnet_matrix('en')
# run SVD
U, S, V = matrix.normalize_all().svd(k = 100)
# get similar concepts
sim = divisi2.reconstruct_similarity(U, S, post_normalize = False)
sim.row_named('teach').top_items(10)
sim.entry_named('horse', 'cow')
# related concepts, spreading activation
assoc = divisi2.network.conceptnet_assoc('en')
U, S, _ = assoc.svd(k = 100)
spread = divisi2.reconstruct_activation(U, S)
spread.entry_named('cat', 'dog')
spread.row_named('think').top_items()
# separating out interests that don't exist in Divisi
allInts = json.loads(open('allInterests.json','r').read())
topInts = {}
for interest in allInts:
if allInts[interest] > 2:
topInts[interest] = allInts[interest]
# fix has words that need adjusting to find in Divisi
fix = []
found = []
for interest in topInts:
try: