def datasets():
#select random number for random selection of data sets
start = random.random()
if start > 0.9:
start = 0.9
end = start + 0.1
#generate list of tagged data from corpus
words = cs.tagged_words(tagset='universal')
startn = round(len(words) * start)
endn = round(len(words) * end) #segment words into two data sets
training = words[:startn] + words[endn:]
testing = words[startn + 1:endn - 1]
#return data sets
return [training, testing]
方法的性能和易用性。
"""
# 间隙定义有自己模式：}<tag>{
"""
7. 用任何你之前已经开发的分块器执行下列评估任务。（请注意，大多数分块语料库包
含一些内部的不一致，以至于任何合理的基于规则的方法都将产生错误。）
a. 在来自分块语料库的 100个句子上评估你的分块器， 报告精度、 召回率和F量度。
b. 使用 chunkscore.missed()和 chunkscore.incorrect()方法识别你的分块器的
错误，并讨论它。
"""
# 分块器评估的学习,以及chunksore的使用
from nltk.corpus import conll2000
cp = nltk.RegexpParser("")
test_sents = conll2000.chunked_sents('test.txt', chunk_types=['NP'])
data = conll2000.tagged_words('test.txt')
print(cp.evaluate(test_sents))
grammer = r"NP: {<[CDJNP].*>+}"
cp = nltk.RegexpParser(grammer)
chunkscore = nltk.chunk.ChunkScore()
guess = cp.parse(data)
chunkscore.score(test_sents, guess)
chunkscore.missed()
chunkscore.incorrect()
"""
10. bigram分块器的准确性得分约为 90％。研究它的错误，并试图找出它为什么不能获
得 100％的准确率。实验trigram分块。你能够在提高性能吗？
"""
# 考察第二种分块器：词袋分块器
return tag
# Return the POS of a rule (used for list sorting)
def get_key(rule):
return rule.split()[1]
if __name__ == '__main__':
# Get allowed words
allowed_words_file = open('../../allowed_words.txt', 'r')
allowed_words = allowed_words_file.read().split('\n')
# Tagged words from corpora
treebank_tagged_words = list(set(treebank.tagged_words()))
conll2000_tagged_words = list(set(conll2000.tagged_words()))
brown_tagged_words = list(set(brown.tagged_words()))
nps_tagged_words = list(set(nps_chat.tagged_words()))
vocab_rules = []
unvocabbed_words = []
# Find tags that occur with allowed words in the corpora
for word in allowed_words:
curr_tags = get_tags_linear(word, treebank_tagged_words)
if not curr_tags:
curr_tags = get_tags_linear(word, conll2000_tagged_words)
if not curr_tags:
curr_tags = get_tags_linear(word, brown_tagged_words)
result = word_vectors.most_similar(positive=['child'],negative=['person'])
print("Most similar to 'child' but dissimilar to 'person':\n",result[:3])
#%%
from nltk.corpus import conll2000
from gensim.models import Word2Vec
import tensorflow as tf
from keras.layers import Dense, Embedding, Activation, Flatten
from keras import Sequential
from keras.utils import to_categorical
import numpy as np
import collections
#%%
train_words = conll2000.tagged_words("train.txt")
test_words = conll2000.tagged_words("test.txt")
print(train_words[:10])
#%%
def get_tag_vocabulary(tagged_words):
tag2id={}
for item in tagged_words:
tag = item[1]
tag2id.setdefault(tag,len(tag2id))
return tag2id
word2id = {k: v.index for k,v in word_vectors.vocab.items()}
tag2id = get_tag_vocabulary(train_words)
#%%
def get_int_data(tagged_words, word2id, tag2id):
X,Y = [],[]
# tagged corpora
print(brown.words())
print(brown.tagged_words())
print(brown.sents()) # doctest: +ELLIPSIS
print(brown.tagged_sents()) # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
print(brown.paras(
categories='reviews')) # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
print(brown.tagged_paras(
categories='reviews')) # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
# nltk.download('indian')
print(indian.words()) # doctest: +SKIP
print(indian.tagged_words()) # doctest: +SKIP
# nltk.download('universal_tagset')
print(brown.tagged_sents(
tagset='universal')) # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
print(conll2000.tagged_words(
tagset='universal')) # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
# chunked corpora
print(conll2000.sents()) # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
for tree in conll2000.chunked_sents()[:2]:
print(tree) # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
# nltk.download('conll2002')
print(conll2002.sents()) # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
for tree in conll2002.chunked_sents()[:2]:
print(tree) # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
# nltk.download('semcor')
print(semcor.words())
print(semcor.chunks())
print(semcor.sents()) # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
print(semcor.chunk_sents()) # doctest: +ELLIPSIS +NORMALIZE_WHITESPACE
print(list(map(str, semcor.tagged_chunks(tag='both')[:3])))
print([[str(c) for c in s] for s in semcor.tagged_sents(tag='both')[:2]])
# init colorama
colorama.init()
# global constants
CONST_tagset = 'universal'
# global list of gold corpora
# C:\Users\admin\AppData\Roaming\nltk_data\corpora\
corp_names = [
"brown", "nps_chat", "conll2000", "treebank", "twitter", "nhtsa_0",
"nhtsa_1", "nhtsa_2", "nhtsa_3", "nhtsa_4", "nhtsa_5", "nhtsa_6"
]
corp_words_tagged = [
brown.tagged_words(tagset=CONST_tagset),
nps_chat.tagged_words(tagset=CONST_tagset),
conll2000.tagged_words(tagset=CONST_tagset),
treebank.tagged_words(tagset=CONST_tagset)
]
corp_words_untagged = [
brown.words(),
nps_chat.words(),
conll2000.words(),
treebank.words()
]
corp_sents_tagged = [
brown.tagged_sents(tagset=CONST_tagset),
nps_chat.tagged_posts(tagset=CONST_tagset),
conll2000.tagged_sents(tagset=CONST_tagset),
treebank.tagged_sents(tagset=CONST_tagset)
]
corp_sents_untagged = [