#drop rare words

freqs = (data > 0).sum(axis=0)

mask = []

v = []

for i in range(data.shape[1]):

if freqs[0,i] > minwords:

mask.append(i)

v.append(vocab[i])

cdata = data[:,mask]

stops = set()

#remove every substance-specific stopword

for k in customstops.keys():

for word in customstops[k]:

stops.add(word)

#also remove the names of substances, because some might not have been curated yet

for s in substance_count.keys():

stops.add(s)

bool = [word not in stops for word in v]

mask = []

v2 = []

for i in range(len(bool)):

if bool[i]:

mask.append(i)

v2.append(v[i])

cdata = cdata[:,mask]

if type(lst)==str:

lst = [lst]

print type(lst)

import webbrowser

webbrowser.open_new('file:///'+path+'data/xml/'+lst[0])

#webbrowser.open_new('file://'+path+'data/xml/' + lst[0])

from time import sleep

for item in lst[1:]:

sleep(1)

import webbrowser

data = data.toarray()

t = np.argsort(data[:,n].tolist())

exps = [experiences[i] for i in t[-1:-ndocs-1:-1]]

et = 1.0/len(vocab)

eta = np.linspace(et,et,num_topics*len(vocab))

eta = eta.reshape((num_topics, len(vocab)))

#apply weights

for n,l in enumerate(lst):

for word in l:

if word in vocab:

idx = vocab.index(word)

eta[n,idx] = 100.0*et

else:

print "note: skipped " + word

all_subs = sorted(substance_count.keys())

summs = np.zeros((len(all_subs),data.shape[1]))

for y,tag in enumerate(all_subs):

print "working on " + tag

mask = []

for e, exp in enumerate(all_index):

if tag in exp:

mask.append(e)

tdata = data[mask, :]

summ = tdata.mean(axis=0)

summs[y] = summ

global jsontree_tally

if id == 2*clusters.shape[0]:

jsontree_tally = 0

#if this is a leaf node

if id <= links.shape[0]:

jsontree_tally+=1

return {"name" : names[id], "size" : counts[id]}

#otherwise, this is a branch node

node = {}

#this size variable is actually the number of leaves, not the true size

left, right, dist, size = tuple(links[int(id-links.shape[0]-1)])

node["d"] = dist

#if we prune at this step, flatten the remaining branches

if dist < min_d or size < min_s: #wait a second...the min size thing does not actually work

jsontree_tally+=1

nodes = flatjson(links, names, counts, id)

#choose method of representing the aggregated nodes

if flatten=="full":

#flatten the children but do not prune leaves

node["name"] = str(int(id))

node['children'] = [{"name": n['name'], "size": counts[names.index(n['name'])]} for n in nodes]

elif flatten=="flat":

#flatten the hierarchy into a string

node["name"] = str([n['name'] for n in nodes])

node["size"] = sum([counts[names.index(n['name'])] for n in nodes])

elif flatten=="lists":

#flatten the children into a list

node["name"] = [n['name'] for n in nodes]

node["size"] = sum([counts[names.index(n['name'])] for n in nodes])

elif flatten=="hidden":

#flatten the children into a value, use the largest child as main name

# better would be to sort by size

# change title to word wrap?

best = np.argmax([counts[names.index(n['name'])] for n in nodes])

node["name"] = nodes[best]['name']+"*"

node["size"] = sum([counts[names.index(n['name'])] for n in nodes])

node["value"] = [n['name'] for n in nodes]

else:

#dflatten the hierarchy, count the nodes, and drop the name

node["name"] = str(int(id))

node["size"] = len(nodes)

#otherwise, recurse down the branches

else:

node["name"] = str(int(id))

node["children"] = [jsontree(links, int(left), names, counts, min_d, min_s,flatten=flatten),jsontree(links, int(right), names, counts, min_d, min_s,flatten=flatten)]

if id == 2*clusters.shape[0]:

print "created a total of " + str(jsontree_tally) + " clusters."

#if this is a leaf node, return the name in a list

if id <= links.shape[0]:

return [{'name': names[int(id)], 'size': counts[int(id)]}]

#return [names[int(id)]]

#otherwise, return the concatenation of two lists

left, right, dist, size = tuple(links[int(id-links.shape[0]-1)])

list1 = flatjson(links, names, counts, left)

list2 = flatjson(links, names, counts, right)

#does this always fully flatten the list?

return list1 + list2

def word_chisq( key,

reference=None,

data=ndata,

vocab=vocab,

n=10,

minwords=0,

maxwords=1000,

stops=[]):

from sklearn.feature_selection import chi2

if type(key) == str:

key = [key]

if stops == "custom":

collect = []

for k in key:

collect += customstops[k]

stops = collect

if reference == None:

refdata = data

refs = [True for row in all_index]

refs = np.array(refs)

else:

if type(reference) == str:

reference = [reference]

refs = [(True in [(k in row) for k in reference]) for row in all_index]

refs = np.array(refs)

refdata = data[refs,:]

#okay...so we're gettin' there...refdata now works correctly, but labels is wrong

labels = [(True in [(k in row) for k in key]) for row in all_index]

labels = np.array(labels)

labels = labels[refs]

chisq, p = chi2(refdata, labels)

ranking = np.argsort(chisq)[::-1]

values = []

freqs = (refdata > 0)[labels,:].sum(axis=0)

i = 0

for rank in ranking:

if i >= n:

break

if not np.isnan(chisq[rank]) and not freqs[:,rank]<minwords and not freqs[:,rank]>maxwords and vocab[rank] not in stops:

values.append((chisq[rank],vocab[rank],p[rank],freqs[:,rank][0,0]))

i+=1

return values[0:n]

y,n,x = "y","n","x"

def curate(keys, values, stops = customstops, jsonize=False):

if jsonize:

with open(path+"data/customstops.json","rb") as f:

customstops = json.loads(f.read())

if type(keys) == str:

keys = [keys]

j = 0

for value in values:

response = input("Use " + str(value) + "? (" + str(j) + " words so far) (y/n/x)")

if response == "n":

for key in keys:

if key not in stops:

stops[key] = []

if value[1] not in stops[key]:

stops[key].append(value[1])

print "Added " + value[1] + " to the custom stopword list entry for " + key + "."

j+=1

elif response == "x":

break

if jsonize:

dumpstops(stops=stops)

else:

print "Finished adding stopwords"

def dumpstops(stops=customstops):

with open(path+'data/customstops.json', 'w') as outfile:

import pprint

json.dump(stops, outfile)

cross = False

if len(rows)==len(columns):

cross = True

for n, row in enumerate(rows):

if rows[n] != columns[n]:

cross = False

import matplotlib.pyplot as plt

from sklearn.feature_selection import chi2

column_labels = columns

row_labels = rows

data = np.zeros((len(rows),len(columns)))

for y,col in enumerate(columns):

labels = [(col in row) for row in all_index]

mat = np.zeros((len(experiences),len(rows)))

for i,exp in enumerate(experiences):

for j,row in enumerate(rows):

if row in all_index[i]:

mat[i,j] = True

else:

mat[i,j] = False

chisq, p = chi2(mat, labels)

for x,cs in enumerate(chisq):

data[x,y] = cs

print "finished " + col

if cross==True:

for y,col in enumerate(columns):

for x,row in enumerate(rows):

if x <= y:

data[x,y] = None

with open(path + '/gh-pages/' + c + '.csv','wb') as f:

import csv

writer = csv.writer(f)

writer.writerow(["rowname"] + columns)

for i in range(len(rows)):

cross = False

if len(rows)==len(columns):

cross = True

for n, row in enumerate(rows):

if rows[n] != columns[n]:

cross = False

import matplotlib.pyplot as plt

from sklearn.feature_selection import chi2

column_labels = columns

row_labels = rows

data = np.zeros((len(rows),len(columns)))

for y,col in enumerate(columns):

labels = [(col in row) for row in all_index]

mat = np.zeros((len(experiences),len(rows)))

for i,exp in enumerate(experiences):

for j,row in enumerate(rows):

if row in all_index[i]:

mat[i,j] = True

else:

mat[i,j] = False

chisq, p = chi2(mat, labels)

for x,cs in enumerate(chisq):

data[x,y] = np.log(cs)

print "finished " + col

if cross==True:

blank = data.min()

for y,col in enumerate(columns):

for x,row in enumerate(rows):

if x <= y:

data[x,y] = blank

fig, ax = plt.subplots()

heatmap = ax.pcolor(data, cmap=plt.cm.Reds)

# put the major ticks at the middle of each cell

ax.set_xticks(np.arange(data.shape[1])+0.5, minor=False)

ax.set_yticks(np.arange(data.shape[0])+0.5, minor=False)

# want a more natural, table-like display

ax.invert_yaxis()

if not cross:

ax.xaxis.tick_top()

ax.set_xticklabels(column_labels, minor=False, rotation=90)

ax.set_yticklabels(row_labels, minor=False)

words = [row[1] for row in results]

import webbrowser

url = "http://infiniteperplexity.github.io/ineffable/wordclouds.html"

param = ",".join(words)

sub, _, _, _, subexps = rowslice(lst)

wdata = sub[:,vocab==word].toarray()

ranking = np.argsort(wdata[:,0])[::-1]

exps = []

for rank in ranking:

if wdata[rank,0] > 0:

exps.append(subexps[rank])

if sort==False:

exps = sorted(exps, key=lambda *args: random.random())

files = examples(word, lst, sort, n)

print "Word occurs in " + str(len(files)) + " files. (note this is wrong!)"

for name in files:

dummy = raw_input("*************Next report?**************")

with open(path+"data/xml/"+name) as f:

txt = f.read()

txt = txt.replace("*","")

txt = txt.replace(word,"***"+word+"***")

txt = txt.replace(word.capitalize(),"***"+word.capitalize()+"***")

from sklearn.feature_selection import chi2

if key in all_tags:

labels = [(key in row) for row in all_index]

else:

raise ValueError('Not a valid tag or substance')

mat = np.zeros((len(experiences),len(all_tags)))

for i,row in enumerate(experiences):

for j,tag in enumerate(all_tags):

if tag in all_index[i]:

mat[i,j] = True

else:

mat[i,j] = False

chisq, p = chi2(mat, labels)

ranking = np.argsort(chisq)[::-1]

values = []

for rank in ranking:

values.append((chisq[rank],all_tags[rank],p[rank]))