def ask (string) :
with builtin_db() as session :
words = session.access.words(string)
scored = Scored(nn.predict(words)).sorted()
print('\n'.join(repr(score) for score in list(scored)[:12]))
def __demo__ (ut) :
from nlplib.data import builtin_db
from nlplib.core.model import Word, NeuralNetwork
from nlplib.exterior.train import usable
from nlplib.exterior.util import plot
top = 10
with builtin_db() as session :
top_words = list(session.access.most_common(Word, top=top)) + [None]
nn = NeuralNetwork(top_words, int(len(top_words)*2), top_words)
with builtin_db() as session :
patterns = usable(nn, list(session.access.all_documents())[:], gram_size=2)
errors = [nn.train(input_words, output_words, rate=0.1) for input_words, output_words in patterns]
from nlplib.general.math import avg
print(avg(errors[:]))
plot(errors,
sample_size=100)
def ask (string) :
with builtin_db() as session :
words = session.access.words(string)
scored = Scored(nn.predict(words)).sorted()
print('\n'.join(repr(score) for score in list(scored)[:12]))
return ask
from nlplib.exterior.scrape.wikipedia import gather_documents
from nlplib.core.process.index import Indexed
from nlplib.general.iterate import chunked
from nlplib.general import timing
@timing
def make_db(db, amount=100):
total = 0
for chunk in chunked(enumerate(gather_documents(amount), total + 1), 10, trail=True):
with db as session:
indexed = Indexed(session)
for total, document in chunk:
if len(document):
session.add(document)
indexed.add(document)
print(total, ":", repr(document))
return total
if __name__ == "__main__":
from nlplib.data import builtin_db
make_db(builtin_db())