def load_documents(self,path):
docs = CategorizedPlaintextCorpusReader(path,r'.*/.*',cat_pattern=r'(.*)/.*')
for cat in docs.categories():
self.cat_gram_freq[cat] = {}
self.cat_word_freq[cat]={}
return ((category,list(docs.words(fileid)))
for category in docs.categories()
for fileid in docs.fileids(category))
def load_documents(self,path): docs = CategorizedPlaintextCorpusReader(path,r'.*/.*',cat_pattern=r'(.*)/.*') print docs.categories() documents = [(list(docs.words(fileid)), category) for category in docs.categories() for fileid in docs.fileids(category) ] random.shuffle(documents) return documents
import nltk
from nltk.corpus.reader.plaintext import CategorizedPlaintextCorpusReader
DOC_PATTERN = r'[\w_\s]+/[\w\s\d\-]+\.TXT'
CAT_PATTERN = r'([\w_\s]+)/.*'
corpus = CategorizedPlaintextCorpusReader('ENGLISH',
DOC_PATTERN,
cat_pattern=CAT_PATTERN)
print(corpus.categories())
print(corpus.fileids()[100:110])
print(corpus.words())
#Load Libraries
import os
import random
from nltk.corpus.reader.plaintext import CategorizedPlaintextCorpusReader
#Read the dataset into the categorized corpus
# Directory of the corpus
corpusdir = 'corpus/'
review_corpus = CategorizedPlaintextCorpusReader(corpusdir, r'.*\.txt', cat_pattern=r'\d+_(\w+)\.txt')
# list of documents(fileid) and category (pos/neg)
documents = [(list(review_corpus.words(fileid)), category)
for category in review_corpus.categories()
for fileid in review_corpus.fileids(category)]
random.shuffle(documents)
for category in review_corpus.categories():
print(category)
type(review_corpus)
len(documents)
#Compute word frequency
import nltk
all_words = nltk.FreqDist(w.lower() for w in review_corpus.words())
word_features = list(all_words)[:200]
print 'Loading corpus...',
t = time()
train_path = 'data/task1_train'
cr = CategorizedPlaintextCorpusReader(train_path, '.*', cat_pattern='(\w*)')
t = time() - t
print str(t) + 's'
# Test generation of CFD
print 'Creating CFD...',
sys.stdout.flush()
t = time()
cat = cr.categories()[0]
n = 3
cfd = ConditionalFreqDist()
prefix = ('',) * (n - 1)
for ngram in ingrams(chain(prefix, cr.words(categories=[cat])), n):
context = tuple(ngram[:-1])
token = ngram[-1]
cfd[context].inc(token)
t = time() - t
print str(t) + 's'
label=target_name)
plt.legend(loc='best', shadow=False, scatterpoints=1)
plt.title('PCA of BULATS dataset')
plt.show()
return model
if __name__ == "__main__":
PATH = "model.pickle"
# Loading speech features
speech = pd.read_csv("/ExamplePath.csv")
if not os.path.exists(PATH):
nli = CategorizedPlaintextCorpusReader(CORPUS,
DOC_PATTERN,
cat_pattern=CAT_PATTERN)
# since `nli` already has all the information (text and ids)
# you don't need to iterate over it multiple times so
# construct `X` and `y` in one go.
X = []
y = []
for fileid in nli.fileids():
X.append({
'text': nli.raw(fileid),
'id': fileid.split('/')[-1].split('.')[0]
})
y.append(nli.categories(fileid)[0])
clf = PCA(n_components=2)
model = build_and_evaluate(X, y, clf, speech)
return 1 in [c in str for c in set]
def is_number(s):
try:
float(s)
return True
except ValueError:
return False
doc_lowercase = [w.lower() for w in doc]
return lemma.lemmatize([w for w in doc_lowercase
if not (is_number(w)) and len(w) > 1 and contains_any(w, wordchars)
and not contains_any(w, exclude) and w not in stop])
doc_dict = {fid: clean(corpus.words(fid)) for cat in corpus.categories() for fid in corpus.fileids(cat)} # XXX
docs = doc_dict.values()
dictionary = gensim.corpora.Dictionary(docs)
doc_ids = [k for k in doc_dict.keys()]
doc_term_matrix = [dictionary.doc2bow(doc) for doc in docs]
bow_array = np.array(doc_term_matrix)
def find_best_lda_model(texts, bow, id2word, min_n=min_topics, max_n=max_topics):
best_model = None
max_coherence = -1
for n in range(min_n, max_n + 1):
ctm = CtmModel(
bow, id2word=id2word, num_topics=n)
from itertools import chain
# from nltk import trigrams, word_tokenize, sent_tokenize, FreqDist
from nltk.corpus.reader.plaintext import CategorizedPlaintextCorpusReader
from nltk.util import ingrams
n = 3
train_path = "data/task1_train"
print "Loading categorized corpus in", train_path, "..."
cr = CategorizedPlaintextCorpusReader(train_path, ".*", cat_pattern="(\w*)")
# Get categories
print "%d categories: %s" % (len(cr.categories()), ", ".join(cr.categories()))
for c in [cr.categories()[0]]:
print c + "..."
sys.stdout.flush()
ngrams = {}
for i in range(n, 0, -1):
print str(i) + "-grams..."
ngrams[i] = {}
prefix = ("",) * (i - 1)
for ngram in ingrams(chain(prefix, cr.words(categories=[c])), n):
if not ngram in ngrams[i]:
ngrams[i][ngram] = 0
ngrams[i][ngram] += 1
r'.*\.txt',
cat_file="../textcats.prn")
"""
fileid="nytimes-2017.txt"
raw = corpus.raw(fileid)
raw = raw.replace("N.H.S.", "NHS")
words = word_tokenize(raw)
words = corpus.words(fileid)
clean0 = [word for word in words if word not in stoplist]
"""
bloblist = corpus.fileids()
#bloblist = corpus.fileids(categories='2016')
M=len(bloblist)
# Look at the categories
corpus.categories()
# for each file in the corpus
for fileid in bloblist:
raw = corpus.raw(fileid)
raw = raw.replace("N.H.S.", "NHS")
raw = raw.replace("per cent", "%")
raw = raw.replace("votes", "vote")
raw = raw.replace("voted", "vote")
words = word_tokenize(raw)
# Bring in the default English NLTK stop words
stoplist = stopwords.words('english')
# Define additional stopwords in a string this will preserve the word image (without capital) mid sentence
additional_stopwords = """also one The Media playback is unsupported on your device caption Image Images copyright Reuters AP Getty EPA said BBC"""
