def expand_study(study_name):
study = StudyDirectory(study_name).get_study()
theblend, concepts = study.get_assoc_blend()
U, S, V = theblend.normalize_all().svd(k=50)
doc_rows = divisi2.aligned_matrix_multiply(study.get_documents_matrix(), U)
projections = U.extend(doc_rows)
spectral = divisi2.reconstruct_activation(projections, S, post_normalize=True)
divisi2.save(spectral, study_name+'/Results/expanded.rmat')
def expand_study(study_name):
study = StudyDirectory(study_name).get_study()
theblend, concepts = study.get_assoc_blend()
U, S, V = theblend.normalize_all().svd(k=50)
doc_rows = divisi2.aligned_matrix_multiply(study.get_documents_matrix(), U)
projections = U.extend(doc_rows)
spectral = divisi2.reconstruct_activation(projections,
S,
post_normalize=True)
divisi2.save(spectral, study_name + '/Results/expanded.rmat')
def analyze(self):
# TODO: make it possible to blend multiple directories
self._documents_matrix = None
docs, projections, Sigma = self.get_eigenstuff()
magnitudes = np.sqrt(np.sum(np.asarray(projections*projections), axis=1))
if self.is_associative():
spectral = divisi2.reconstruct_activation(projections, Sigma, post_normalize=True, offset=0.0001)
else:
spectral = divisi2.reconstruct_similarity(projections, Sigma,
post_normalize=True, offset=0.0001)
self._step('Calculating stats...')
stats = self.compute_stats(docs, spectral)
results = StudyResults(self, docs, spectral.left, spectral, magnitudes, stats)
return results
def analyze(self):
# TODO: make it possible to blend multiple directories
self._documents_matrix = None
docs, projections, Sigma = self.get_eigenstuff()
magnitudes = np.sqrt(
np.sum(np.asarray(projections * projections), axis=1))
if self.is_associative():
spectral = divisi2.reconstruct_activation(projections,
Sigma,
post_normalize=True,
offset=0.0001)
else:
spectral = divisi2.reconstruct_similarity(projections,
Sigma,
post_normalize=True,
offset=0.0001)
self._step('Calculating stats...')
stats = self.compute_stats(docs, spectral)
results = StudyResults(self, docs, spectral.left, spectral, magnitudes,
stats)
return results
def make_svd(self):
matrix = divisi2.make_sparse(self.data).normalize_all()
self.U,self.s,self.V = matrix.svd(k=14)
self.predictions = divisi2.reconstruct_activation(self.V, self.s)
del self.data
print "init end"
from csc import divisi2
matrix = divisi2.network.conceptnet_matrix('en')
# run SVD
U, S, V = matrix.normalize_all().svd(k = 100)
# get similar concepts
sim = divisi2.reconstruct_similarity(U, S, post_normalize = False)
sim.row_named('teach').top_items(10)
sim.entry_named('horse', 'cow')
# related concepts, spreading activation
assoc = divisi2.network.conceptnet_assoc('en')
U, S, _ = assoc.svd(k = 100)
spread = divisi2.reconstruct_activation(U, S)
spread.entry_named('cat', 'dog')
spread.row_named('think').top_items()
# separating out interests that don't exist in Divisi
allInts = json.loads(open('allInterests.json','r').read())
topInts = {}
for interest in allInts:
if allInts[interest] > 2:
topInts[interest] = allInts[interest]
# fix has words that need adjusting to find in Divisi
fix = []
found = []
for interest in topInts:
try: