def loadTexts():
path = '../texts/math_learning_2letters_full/'
texts = sorted([os.path.join(path, fn) for fn in os.listdir(path)])
for text in texts:
if '.DS_Store' in text:
texts.remove(text)
delimiter = ' '
return texts, getDocuments(texts, delimiter)
def loadData(paths):
documents, texts = [{"AD":[],"TD":[]} for _ in range(2)]
for group in groups:
for ds in range(len(paths['AD'])):
currTexts = sorted([os.path.join(paths[group][ds], fn) for fn in os.listdir(paths[group][ds])])
for text in currTexts:
if '.DS_Store' in text:
currTexts.remove(text)
texts[group].append(currTexts)
texts[group] = sum(texts[group], [])
documents[group] = getDocuments(texts[group], 'none', False, texts[group])
return texts, documents
def runBag(path):
textNames = sorted([os.path.join(path, fn) for fn in os.listdir(path)])
scripts = sorted([os.path.join(path, fn) for fn in os.listdir(path)])
for script in scripts:
if '.DS_Store' in script:
scripts.remove(script)
documents = getDocuments(scripts, 'none', False, textNames)
data, newVocab, featureNames = bagOfWords(scripts, documents, True, 0, False, False)
return data, featureNames
def runBag(path):
textNames = sorted([os.path.join(path, fn) for fn in os.listdir(path)])
scripts = sorted([os.path.join(path, fn) for fn in os.listdir(path)])
for script in scripts:
if '.DS_Store' in script:
scripts.remove(script)
documents = getDocuments(scripts, 'none', False, textNames)
data, newVocab, featureNames = bagOfWords(scripts, documents, True, 0,
False, False)
return data, featureNames
def loadData(paths):
documents, texts = [{"AD": [], "TD": []} for _ in range(2)]
for group in groups:
for ds in range(len(paths['AD'])):
currTexts = sorted([
os.path.join(paths[group][ds], fn)
for fn in os.listdir(paths[group][ds])
])
for text in currTexts:
if '.DS_Store' in text:
currTexts.remove(text)
texts[group].append(currTexts)
texts[group] = sum(texts[group], [])
documents[group] = getDocuments(texts[group], 'none', False,
texts[group])
return texts, documents
let_acc = [0] * 14 #14
feat_info = {} #[[]] * 3
for let in range(3, 6): #5 #16 (2,16)
path = pre_path + '%s_word' % (let)
textNames = sorted([os.path.join(path, fn) for fn in os.listdir(path)])
scripts = textNames
new_texts = []
for script in scripts:
if '.DS_Store' not in script:
new_texts.append(script)
documents = getDocuments(new_texts, delimiter, isCorpus, textNames)
data, newVocab, featureNames = bagOfWords(new_texts, documents,
nGramsinCorpus, mincount,
windowGrams, gramsOnly)
rfACC = [0] * nModels
importances = [[]] * nModels
top_importances = [[]] * nModels
top_stds = [[]] * nModels
top_featNames = [[]] * nModels
stds = [[]] * nModels
for i in range(nModels):
print "Running RF with %i estimators...\n" % (nEstimators)
rfACC[i], importances[i], stds[i] = rfModel(
# for classification
nLabelOne = 40 #70#90#40#70#30 #number of TDs
nLabelTwo = 40 #70#90#40#70#30 #number of ADs
labels = np.asarray([0] * nLabelOne + [1] * nLabelTwo)
nFolds = len(labels) #leave-one-out
nEstimators = 1000 #1000 #number of estimators for random forest classifier
runClassification = True # run classification on topic probabilities
#runClassification = False # OPTION ONLY FOR HDP ...run classification on document similarities
################################################
############## PROCESS DATA ####################
################################################
# Create dictionary with list of processed words for each document key
documents = getDocuments(scripts, delimiter, isCorpus, textNames)
#plot_Networkletters(documents, title, letters)
if network_wise:
new_docs = {}
for doc in documents:
new_docs[doc] = []
for word in documents[doc]:
#documents[doc][documents[doc].index(word)] = 'DMN_' + word[0] + ' SN_' + word[1] + ' LECN_' + word[2] + ' RECN_' + word[3]
new_docs[doc].append(' dmn_' + word[0])
new_docs[doc].append(' sn_' + word[1])
new_docs[doc].append(' lecn_' + word[2])
new_docs[doc].append(' recn_' + word[3])
documents = new_docs
AD_train = []
TD_train = []
AD_test = []
TD_test = []
for path in paths:
textNames = sorted([os.path.join(path, fn) for fn in os.listdir(path)])
texts = sorted([os.path.join(path, fn) for fn in os.listdir(path)])
for text in texts: #removing .DS_Store
if '.DS_Store' in text:
texts.remove(text)
docs = getDocuments(texts, 'none', False, textNames)
doc_list = listsfromDict(docs)
bigram_docs = []
for doc in doc_list:
bigramList = []
for item in ngrams(doc, 2):
bigramList.append("_".join(item))
bigram_docs.append(bigramList)
doc_list = bigram_docs
if 'AD_' in path:
if path in train_path:
AD_train.append(doc_list)
elif path in test_path:
AD_test.append(doc_list)
#trigrams
# ad_voc = loadCSV('NYU_AD_2letter_trigrams_BoW')
# td_voc = loadCSV('NYU_TD_2letter_trigrams_BoW')
# ad_voc = ad_voc[1]
# td_voc = td_voc[1]
# vocab = [w for w in td_voc if w in set(ad_voc)]
# print vocab
vocab_dict = {"AD":dict((word,[]) for word in vocab),"TD":dict((word,[]) for word in vocab)}
for group in groups:
texts[group] = sorted([os.path.join(paths[group], fn) for fn in os.listdir(paths[group])])
if len(texts[group]) > 80:
texts[group] = texts[group][1:len(texts[group])]
documents[group] = getDocuments(texts[group], 'none', False, texts[group])
#### BAG OF WORDS ####
for text in texts[group]:
if '.DS_Store' not in text:
doc_dict = {text:documents[group][text]} #making single dict to get BoW code to work on individual text
counts[group][text], reducedDocuments, words[group][text] = bagOfWords([text], doc_dict, True, 0, False, False)
counts[group][text] = counts[group][text][0] #flattening
count_list = counts[group][text]
word_list = words[group][text] # for code simplification
no_occurrence = [w for w in vocab if w not in set(word_list)]
occurrence = [w for w in vocab if w in set(word_list)]
for word in occurrence:
count_ind = word_list.index(word)
return newList
AD_train = []
TD_train = []
AD_test = []
TD_test = []
for path in paths:
textNames = sorted([os.path.join(path, fn) for fn in os.listdir(path)])
texts = sorted([os.path.join(path, fn) for fn in os.listdir(path)])
for text in texts: #removing .DS_Store
if '.DS_Store' in text:
texts.remove(text)
docs = getDocuments(texts,'none',False,textNames)
doc_list = listsfromDict(docs)
bigram_docs = []
for doc in doc_list:
bigramList = []
for item in ngrams(doc,2):
bigramList.append("_".join(item))
bigram_docs.append(bigramList)
doc_list = bigram_docs
if 'AD_' in path:
if path in train_path:
AD_train.append(doc_list)
elif path in test_path:
AD_test.append(doc_list)
'''
#### DEFINE GROUPS AND GROUP VARIABLES ####
groups = ["AD","TD"]
paths = {"AD":'texts/ADHD_various_half/AD_4',"TD":'texts/ADHD_various_half/TD_4'}
documents, texts = [{"AD":[],"TD":[]} for _ in range(2)]
counts, words, DMN_inverse, tasknet_activation = [{"AD":{},"TD":{}} for _ in range(4)]
networks = ["DMN","SN","LECN","RECN"]
#### PROCESS DOCUMENTS ####
for group in groups:
texts[group] = sorted([os.path.join(paths[group], fn) for fn in os.listdir(paths[group])])
if len(texts[group]) > 80:
texts[group] = texts[group][1:len(texts[group])]
documents[group] = getDocuments(texts[group], 'none', False, texts[group])
#### BAG OF WORDS ####
for text in texts[group]:
DMN_inverse[group][text]= {"SN":0,"LECN":0,"RECN":0}
tasknet_activation[group][text] = {"SN_RECN":0,"SN_LECN":0,"LECN_RECN":0,"SN_LECN_RECN":0}
doc_dict = {text:documents[group][text]} #making single dict to get BoW code to work on individual text
counts[group][text], reducedDocuments, words[group][text] = bagOfWords([text], doc_dict, False, 0, False, False)
counts[group][text] = counts[group][text][0] #flattening
count_list = counts[group][text]
word_list = words[group][text] # for code simplification
#### SORT BY ACTIVITY ####
for word in word_list:
# inverse activity with DMN
let_acc = [0] * 14#14
feat_info = {}#[[]] * 3
for let in range(3,6): #5 #16 (2,16)
path = pre_path + '%s_word'%(let)
textNames = sorted([os.path.join(path, fn) for fn in os.listdir(path)])
scripts = textNames
new_texts = []
for script in scripts:
if '.DS_Store' not in script:
new_texts.append(script)
documents = getDocuments(new_texts, delimiter, isCorpus, textNames)
data, newVocab, featureNames = bagOfWords(new_texts, documents, nGramsinCorpus, mincount, windowGrams, gramsOnly)
rfACC = [0] * nModels
importances = [[]] * nModels
top_importances = [[]] * nModels
top_stds = [[]] * nModels
top_featNames = [[]] * nModels
stds = [[]] * nModels
for i in range(nModels):
print "Running RF with %i estimators...\n" %(nEstimators)
rfACC[i], importances[i], stds[i] = rfModel(data, labels, nFolds, nEstimators)
idx = (-importances[i]).argsort()[:10]
# for classification
nLabelOne = 40#70#90#40#70#30 #number of TDs
nLabelTwo = 40#70#90#40#70#30 #number of ADs
labels = np.asarray([0] * nLabelOne + [1] * nLabelTwo)
nFolds = len(labels) #leave-one-out
nEstimators = 1000 #1000 #number of estimators for random forest classifier
runClassification = True # run classification on topic probabilities
#runClassification = False # OPTION ONLY FOR HDP ...run classification on document similarities
################################################
############## PROCESS DATA ####################
################################################
# Create dictionary with list of processed words for each document key
documents = getDocuments(scripts, delimiter, isCorpus, textNames)
#plot_Networkletters(documents, title, letters)
if network_wise:
new_docs = {}
for doc in documents:
new_docs[doc] = []
for word in documents[doc]:
#documents[doc][documents[doc].index(word)] = 'DMN_' + word[0] + ' SN_' + word[1] + ' LECN_' + word[2] + ' RECN_' + word[3]
new_docs[doc].append(' dmn_' + word[0])
new_docs[doc].append(' sn_' + word[1])
new_docs[doc].append(' lecn_' + word[2])
new_docs[doc].append(' recn_' + word[3])
documents = new_docs