def print_cost_estimates(commons_path, corpora_path):
from corpora import Corpora
train = Corpora(corpora_path, commons_path, gold=True)
actions = Actions()
for document in train:
for action in document.gold:
actions.add(action)
train.rewind()
cascades = [cascade_class(actions) for cascade_class in \
[FlatCascade, ShiftCascade, ShiftMarkCascade, ShiftPropbankEvokeCascade]]
costs = [0] * len(cascades)
counts = [[0] * cascade.size() for cascade in cascades]
for document in train:
gold = document.gold
for index, cascade in enumerate(cascades):
cascade_gold_sequence = cascade.translate(gold)
delegate = 0
cost = 0
for cascade_gold in cascade_gold_sequence:
cost += cascade.delegates[delegate].size()
counts[index][delegate] += 1
if cascade_gold.is_cascade():
delegate = cascade_gold.delegate
else:
delegate = 0
costs[index] += cost
for c, cost, cascade in zip(counts, costs, cascades):
print "\n", cascade.__class__.__name__, "cost =", cost, "\n", \
"Delegate invocations:", c, "\n", cascade
def initialize_corpora(data_path, data_prefix, dict_path, iterator, **kwargs):
corpora_params = dict(data_path=data_path, prefix=data_prefix)
if os.path.exists(dict_path):
corpora = Corpora(dictionary=dict_path, **corpora_params)
else:
corpora = Corpora(**corpora_params).build()
corpora.dictionary.save_as_text(dict_path)
if len(corpora) == 0:
raise ValueError(
f'Did not find any documents from path: {data_path} for given prefix {data_prefix}'
)
return corpora
def dev_accuracy(commons_path, dev_path, tmp_folder, caspar):
dev = Corpora(dev_path, caspar.spec.commons)
print "Annotating dev documents", now(), mem()
test_path = os.path.join(tmp_folder, "dev.annotated.rec")
writer = sling.RecordWriter(test_path)
count = 0
start_time = time.time()
cascade = caspar.spec.cascade
dev_total = [0] * cascade.size()
dev_disallowed = [0] * cascade.size()
for document in dev:
state, disallowed, total, trace = \
caspar.forward(document, train=False, debug=True)
state.write()
trace.write()
writer.write(str(count), state.encoded())
count += 1
if count % 100 == 0:
print " Annotated", count, "documents", now(), mem()
for i, c in enumerate(disallowed):
dev_total[i] += total[i]
dev_disallowed[i] += c
writer.close()
end_time = time.time()
print "Annotated", count, "documents in", "%.1f" % (end_time - start_time), \
"seconds", now(), mem()
print "Disallowed/Total leaf actions for", cascade.__class__.__name__
for i, c in enumerate(dev_disallowed):
print "Delegate", i, "disallowed", c, "out of", dev_total[i]
return utils.frame_evaluation(gold_corpus_path=dev_path, \
test_corpus_path=test_path, \
commons_path=commons_path)
def train(args):
check_present(
args,
["train_corpus", "output_folder", "dev_corpus", "train_shuffle_seed"])
train_corpus_path = args.train_corpus
if args.train_shuffle_seed > 0:
reader = sling.RecordReader(args.train_corpus)
items = [(key, value) for key, value in reader]
reader.close()
r = random.Random(args.train_shuffle_seed)
r.shuffle(items)
train_corpus_path = os.path.join(args.output_folder,
"train_shuffled.rec")
writer = sling.RecordWriter(train_corpus_path)
for key, value in items:
writer.write(key, value)
writer.close()
print("Wrote shuffled train corpus to %s using seed %d" % \
(train_corpus_path, args.train_shuffle_seed))
# Setting an explicit seed for the sake of determinism.
torch.manual_seed(1)
# Make commons store if needed.
if args.commons == '' or not os.path.exists(args.commons):
if args.commons == '':
fname = os.path.join(args.output_folder, "commons")
print("Will create a commons store at", fname)
args.commons = fname
else:
print("No commons found at", args.commons, ", creating it...")
_, symbols = commons_builder.build(
[train_corpus_path, args.dev_corpus], args.commons)
print("Commons created at", args.commons, "with", len(symbols), \
"symbols besides the usual ones.")
# Make the training spec.
spec = Spec()
spec.build(args.commons, train_corpus_path)
# Initialize the model with the spec and any word embeddings.
caspar = Caspar(spec)
embeddings_file = args.word_embeddings
if embeddings_file == '': embeddings_file = None
caspar.initialize(embeddings_file)
tmp_folder = os.path.join(args.output_folder, "tmp")
if not os.path.exists(tmp_folder):
os.makedirs(tmp_folder)
evaluator = partial(dev_accuracy, args.dev_corpus, tmp_folder)
output_file_prefix = os.path.join(args.output_folder, "caspar")
hyperparams = Hyperparams(args)
print("Using hyperparameters:", hyperparams)
trainer = Trainer(caspar, hyperparams, evaluator, output_file_prefix)
train = Corpora(train_corpus_path, spec.commons, gold=True)
trainer.train(train)
def update_query(self):
results = [r["Description"] + r['Title'] for r in self.results]
try:
corpora = Corpora(self.query, results, self.selectedIDs)
print corpora
except:
print "NLTK Corpora not install, query expansion requires NLKT Corpuses"
print "Augmenting query..."
# filter - choose words that are not in the query already
candidates = [(w, s) for w, s in corpora.getUpdatedQuery() \
if w not in set(self.query.split())]
(w1, s1), (w2, s2) = candidates[0], candidates[1]
newQueryWords = [w1, w2] if s1 == s2 else [w1]
# build new query
self.query = " ".join([self.query] + newQueryWords)
print "Restarting search with query: ", self.query
self.start()
def update_query(self):
results = [r["Description"] + r['Title'] for r in self.results]
try:
corpora = Corpora(self.query, results, self.selectedIDs)
print corpora
except:
print "NLTK Corpora not install, query expansion requires NLKT Corpuses"
print "Augmenting query..."
# filter - choose words that are not in the query already
candidates = [(w, s) for w, s in corpora.getUpdatedQuery() \
if w not in set(self.query.split())]
(w1, s1), (w2, s2) = candidates[0], candidates[1]
newQueryWords = [w1, w2] if s1 == s2 else [w1]
# build new query
self.query = " ".join([self.query] + newQueryWords)
print "Restarting search with query: ", self.query
self.start()
def train(args):
check_present(args, ["train_corpus", "output_folder", "dev_corpus"])
# Setting an explicit seed for the sake of determinism.
torch.manual_seed(1)
# Make commons store if needed.
if args.commons == '' or not os.path.exists(args.commons):
if args.commons == '':
fname = os.path.join(args.output_folder, "commons")
print "Will create a commons store at", fname
args.commons = fname
else:
print "No commons found at", args.commons, ", creating it..."
_, symbols = commons_builder.build(
[args.train_corpus, args.dev_corpus], args.commons)
print "Commons created at", args.commons, "with", len(symbols), \
"symbols besides the usual ones."
# Make the training spec.
spec = Spec()
spec.build(args.commons, args.train_corpus)
# Initialize the model with the spec and any word embeddings.
caspar = Caspar(spec)
embeddings_file = args.word_embeddings
if embeddings_file == '': embeddings_file = None
caspar.initialize(embeddings_file)
tmp_folder = os.path.join(args.output_folder, "tmp")
if not os.path.exists(tmp_folder):
os.makedirs(tmp_folder)
evaluator = partial(dev_accuracy,
args.commons,
args.dev_corpus,
tmp_folder)
output_file_prefix = os.path.join(args.output_folder, "caspar")
hyperparams = Hyperparams(args)
print "Using hyperparameters:", hyperparams
trainer = Trainer(caspar, hyperparams, evaluator, output_file_prefix)
train = Corpora(args.train_corpus, spec.commons, gold=True)
trainer.train(train)
def run(args):
check_present(args, ["input", "parser", "output"])
assert os.path.exists(args.input), args.input
assert os.path.exists(args.parser), args.parser
# Read parser flow.
flow = Flow()
flow.load(args.parser)
# Initialize the spec from the flow.
spec = Spec()
spec.from_flow(flow)
# Initialize the model from the flow.
caspar = Caspar(spec)
caspar.from_flow(flow)
corpus = Corpora(args.input, caspar.spec.commons)
writer = sling.RecordWriter(args.output)
count = 0
for document in corpus:
state, _, _, trace = caspar.forward(document,
train=False,
debug=args.trace)
state.write()
if trace:
trace.write()
writer.write(str(count), state.encoded())
count += 1
if count % 100 == 0:
print "Annotated", count, "documents", now(), mem()
writer.close()
print "Annotated", count, "documents", now(), mem()
print "Wrote annotated documents to", args.output
if args.evaluate:
f = tempfile.NamedTemporaryFile(delete=False)
fname = f.name
caspar.spec.commons.save(fname, binary=True)
f.close()
eval_result = frame_evaluation(gold_corpus_path=args.input, \
test_corpus_path=args.output, commons=caspar.spec.commons)
os.unlink(fname)
return eval_result
def build(self, commons_path, corpora_path):
# Prepare lexical dictionaries.
self.words = Lexicon(self.words_normalize_digits)
self.suffix = Lexicon(self.words_normalize_digits, oov_item=None)
# Initialize training corpus.
corpora = Corpora(corpora_path, commons_path)
# Collect word and affix lexicons.
for document in corpora:
for token in document.tokens:
word = token.word
self.words.add(word)
for s in self.get_suffixes(word):
assert type(s) is str
self.suffix.add(s)
print "Words:", self.words.size(), "items in lexicon, including OOV"
print "Suffix:", self.suffix.size(), "items in lexicon"
# Load common store, but not freeze it yet. We will add the action table
# and cascade specification to it.
self.commons_path = commons_path
self.commons = sling.Store()
self.commons.load(commons_path)
schema = sling.DocumentSchema(self.commons)
# Prepare action table and cascade.
self._build_action_table(corpora)
self.cascade = cascade.ShiftMarkCascade(self.actions)
print self.cascade
# Save cascade specification in commons.
_ = self.cascade.as_frame(self.commons,
delegate_cell_prefix="delegate")
# Freeze the common store.
self.commons.freeze()
# Add feature specs.
self._specify_features()
def beer_tax():
return Corpora(TAX_SYSTEM_IN_US)
def getty():
return Corpora(GETTYSBURG)
def main(model, alpha, gamma, kappa, n_topics, data_path, data_prefix,
result_path, dictionary_path, stopwords, vectors_path, batch_size,
iterations, passes, n_words, shuffle):
if not os.path.exists(result_path):
raise OSError(f'Provided path {result_path} does not exist.')
corpora_params = dict(data_path=data_path,
prefix=data_prefix,
iterator='bow',
stopwords=stopwords)
if os.path.exists(dictionary_path):
corpora = Corpora(dictionary=dictionary_path, **corpora_params)
else:
corpora = Corpora(**corpora_params).build()
corpora.dictionary.save_as_text(dictionary_path)
if len(corpora) == 0:
raise ValueError(
f'Did not find any documents from path: {data_path} for given prefix {data_prefix}'
)
MAP = dict(lda=(LDAWrapper,
dict(n_topics=n_topics,
alpha=alpha,
iterations=iterations,
passes=passes,
batch_size=batch_size,
id2word=corpora.dictionary)),
shdp=(SHDPWrapper,
dict(n_topics=n_topics,
alpha=alpha,
gamma=gamma,
passes=passes,
batch_size=batch_size,
batch_shuffle=shuffle,
vector_map=load_vectors(
vectors_path, dictionary=corpora.dictionary),
num_docs=len(corpora))))
model_class, params = MAP[model]
topic_model = model_class(**params)
data = [doc for doc, _ in corpora
] # Gather the data since BOW reprecentation is lightweight.
for i, seq in enumerate(data):
if len(seq) < 1:
raise AssertionError(f'Empty seq at index {i}')
if shuffle:
np.random.shuffle(data)
topic_model.fit(data)
model_name = str(topic_model) + f'_{data_prefix}'
years = corpora.years
if years:
model_name += f'{years[0]}-{years[-1]}'
if shuffle:
model_name += f'_shuffled'
path_dir = os.path.join(result_path, model_name)
if not os.path.exists(path_dir):
os.mkdir(path_dir)
if hasattr(topic_model, 'save'):
topic_model.save(os.path.join(path_dir, model_name))
topic_df = document_topics(topic_model, corpora)
dictionary = None
if model == 'shdp':
dictionary = corpora.dictionary
words_df = model_words(topic_model, n=n_words, dictionary=dictionary)
topics_path = os.path.join(path_dir, 'topics.csv')
words_path = os.path.join(path_dir, 'words.csv')
topic_df.to_csv(topics_path, index=False)
words_df.to_csv(words_path, index=False)
import numpy as np
import matplotlib.pyplot as plt
from discord.ext import commands
# last one sitting universe
#TODO: Add channel ID HERE
channels = [0000000]
VC = ""
bot = commands.Bot(command_prefix=">")
from corpora import Corpora
Corpora(bot)
"""
------------------------------
Members
------------------------------
"""
@bot.command("docs")
async def hello(ctx):
with open("help.txt") as afile:
docs = afile.read()
await ctx.send(docs)
"""
from sklearn.decomposition import PCA
from sklearn.cluster import DBSCAN
from bert_embedding import BertEmbedding
sys.path.append('..')
style.use('ggplot')
from corpora import Corpora
data_path = 'M:/Projects/KeyTopicDetection/parsed'
dict_path = '../../data/cvpr_13-18_DICT.txt'
if os.path.exists(dict_path):
corpora = Corpora(data_path=data_path,
prefix='CVPR',
iterator='bow',
dictionary=dict_path)
else:
corpora = Corpora(data_path=data_path,
prefix='CVPR',
iterator='bow',
word_up_limit=0.75,
word_low_limit=20).build()
corpora.dictionary.save_as_text(dict_path)
ctx = mx.gpu(0)
bert = BertEmbedding(ctx=ctx)
def visualize_clusters(tw, data):
db = DBSCAN(eps=0.5, min_samples=50).fit(data)