def load(self, model_dir): self._model_dir = model_dir self._paramsfile = os.path.join(self._model_dir, 'params.py') self._params = utils.load_param_file(self._paramsfile) ckpt = os.path.join(utils.get_dict_value(self._params,'output_location'), utils.get_dict_value(self._params, 'model_name') + '.ckpt') self._e = Evaluator.load2(ckpt)
def main(argv):
try:
argv = FLAGS(argv) # parse flags
except gflags.FlagsError as e:
print('%s\\nUsage: %s ARGS\\n%s' % (e, sys.argv[0], FLAGS))
sys.exit(1)
params = utils.load_param_file(FLAGS.paramsfile)
model_dirname = os.path.dirname(FLAGS.paramsfile)
src_dir = os.path.abspath(os.path.join(model_dirname, '..'))
target_dir = FLAGS.target_dir
model_name = params['model_name']
release_dir_name = FLAGS.release_dir_name
if release_dir_name == "":
release_dir_name = model_name
release_num, dir_name, target_model_dir = copy2repo(
src_dir, target_dir, model_name, release_dir_name)
if release_num == -1:
print("source dir %s doesn't exist" % dir_name)
elif release_num == -2:
print("release source dir %s doesn't exist" % dir_name)
elif release_num == -3:
print("target dir %s doesn't exist" % dir_name)
else:
add_release_num_to_json(
os.path.join(target_model_dir, release_dir_name, 'params.json'),
release_num)
add_release_num_to_json(
os.path.join(target_model_dir,
'%s.%s' % (release_dir_name, release_num),
'params.json'), release_num)
print("SUCCESSFULLY COPY MODEL %s.%s TO REPO" %
(model_name, release_num))
def main(argv):
try:
argv = FLAGS(argv) # parse flags
except gflags.FlagsError as e:
print('%s\\nUsage: %s ARGS\\n%s' % (e, sys.argv[0], FLAGS))
sys.exit(1)
print(FLAGS.paramsfile)
params = utils.load_param_file(FLAGS.paramsfile)
eval(params,
save_accuracy_file=False,
batch_size=1,
num_batches=200,
topn=5)
def load(self, model_dir): self._model_dir = model_dir self._paramsfile = os.path.join(self._model_dir, 'params.py') self._params = utils.load_param_file(self._paramsfile) self._num_before = utils.get_dict_value(self._params, "num_words_before") self._num_after = utils.get_dict_value(self._params, "num_words_after") ckpt = os.path.join(utils.get_dict_value(self._params,'output_location'), utils.get_dict_value(self._params, 'model_name') + '.ckpt') vocab_file = os.path.join(utils.get_dict_value(self._params, 'output_location'), 'vocab.pkl') self._e = Evaluator.load2(ckpt) self._i = TextIndexer.from_file(vocab_file) self._keywords = self._params['keywords'] self._id_to_word = self._params['id_to_keyword']
def params2json(params_file, json_filename):
params = utils.load_param_file(params_file)
with open(json_filename, 'w') as fo:
ignore_list = []
for f in params:
if not (isinstance(params[f], list) or isinstance(params[f], dict) or
isinstance(params[f], int) or isinstance(params[f], str)
or isinstance(params[f], float)
):
ignore_list.append(f);
for f in ignore_list:
print("IGNORING %s" % f)
del params[f]
json.dump(params, fo)
def main(argv):
params_files_list = [
'output/tellmeV12/params.py', 'output/tellmeV15/params.py',
'output/tellmeV17/params.py', 'output/tellmeV18/params.py',
'output/tellmeV19/params.py'
]
params = []
data = []
for paramsfile in params_files_list:
params.append(utils.load_param_file(paramsfile))
data.append(load_results(params[-1]))
fig, ax = plt.subplots()
for cdata in data:
ax.plot([(x * 8192) / 1000000 for x in cdata[1]], cdata[8])
ax.set(xlabel='Million Records Seen',
ylabel='Accuracy @ 1',
title=','.join([x['model_name'] for x in params]))
ax.grid()
max_value = .82 #np.max(data[8])
#plt.ylim((.75,math.ceil(max_value*10)/10))
plt.ylim((.75, .81))
#plt.ylim((.75,1))
fig.savefig("accuracy_compare.png")
plt.show(block=False)
fig, ax = plt.subplots()
for cdata in data:
ax.plot([(x * 8192) / 1000000 for x in cdata[1]], cdata[3])
ax.set(xlabel='Million Records Seen',
ylabel='Loss',
title=','.join([x['model_name'] for x in params]))
ax.grid()
min_value = 0 #np.min(data[3])
plt.ylim((math.floor(min_value * 10) / 10, 1))
#plt.ylim((.75,1))
fig.savefig("loss_compare.png")
plt.show(block=False)
input("Press enter to exit...")
def load(self, model_dir): self._model_dir = model_dir self._paramsfile = os.path.join(self._model_dir, 'params.py') self._params = utils.load_param_file(self._paramsfile) self._num_before = utils.get_dict_value(self._params, "num_words_before") self._num_after = utils.get_dict_value(self._params, "num_words_after") ckpt = os.path.join(utils.get_dict_value(self._params,'output_location'), utils.get_dict_value(self._params, 'model_name') + '.ckpt') vocab_file = os.path.join(utils.get_dict_value(self._params, 'output_location'), 'vocab.pkl') self._e = Evaluator.load2(ckpt) self._i = TextIndexer.from_file(vocab_file) with open(os.path.join( utils.get_dict_value(self._params, 'output_location'), 'keywords.pkl'), 'rb') as f: keywords = pickle.load(f) self._params['keywords'] = keywords self._keywords = self._params['keywords'] self._keyword_map, self._keyword_list = gen_keywords(self._params)
def main(argv):
try:
argv = FLAGS(argv) # parse flags
except gflags.FlagsError as e:
print('%s\\nUsage: %s ARGS\\n%s' % (e, sys.argv[0], FLAGS))
sys.exit(1)
print(FLAGS.paramsfile)
params = utils.load_param_file(FLAGS.paramsfile)
data = load_results(params)
fig, ax = plt.subplots()
ax.plot([(x * 8192) / 1000000 for x in data[1]], data[8])
ax.set(xlabel='Million Records Seen',
ylabel='Accuracy @ 1',
title=params['model_name'])
ax.grid()
max_value = np.max(data[8])
# plt.ylim((.75,math.ceil(max_value*10)/10))
#plt.ylim((.75,1))
fig.savefig(
os.path.join(utils.get_dict_value(params, 'output_location'),
"accuracy.png"))
plt.show(block=False)
fig, ax = plt.subplots()
ax.plot([(x * 8192) / 1000000 for x in data[1]], data[3])
ax.set(xlabel='Million Records Seen',
ylabel='Loss',
title=params['model_name'])
ax.grid()
min_value = np.min(data[3])
# plt.ylim((math.floor(min_value*10)/10,1))
#plt.ylim((.75,1))
fig.savefig(
os.path.join(utils.get_dict_value(params, 'output_location'),
"loss.png"))
plt.show(block=False)
input("Press enter to exit...")
from framework.utils.data.text_indexer import TextIndexer
from word_classifier.data import ClassifierData
import framework.subgraph.losses as losses
import framework.utils.common as utils
import data
from framework.trainer import Trainer, _default_train_iteration_done
from time import time
import pickle
import model
import os
import shutil
import copy
import numpy as np
param_file = 'params.py'
params = utils.load_param_file(param_file)
params['num_classes'] = len(params['keywords'])+1
indexer = TextIndexer.from_txt_file(utils.get_dict_value(params, 'vocab_file'), max_size=utils.get_dict_value(params,'max_vocab_size',-1))
indexer.add_token('<pad>')
indexer.add_token('unk')
output_indexer = copy.deepcopy(indexer)
output_indexer.add_token('<blank>')
os.makedirs(utils.get_dict_value(params,'output_location'), exist_ok=True)
indexer.save_vocab_as_pkl(os.path.join(utils.get_dict_value(params,'output_location'), 'vocab.pkl'))
files_to_copy = [param_file]
for file in files_to_copy:
shutil.copyfile(file,os.path.join(utils.get_dict_value(params,'output_location'), file))
params['vocab_size'] = indexer.vocab_size()
from framework.utils.data.text_indexer import TextIndexer
from framework.evaluator import Evaluator
import framework.utils.common as utils
import os
import numpy as np
run_server = False
params = utils.load_param_file('params.py')
vocab_file = os.path.join(utils.get_dict_value(params, 'output_location'),
'vocab.pkl')
ckpt = os.path.join(utils.get_dict_value(params, 'output_location'),
utils.get_dict_value(params, 'model_name') + '.ckpt')
sentences = ['The apple , which is rotten is not edible']
e = Evaluator.load2(ckpt)
i = TextIndexer.from_file(vocab_file)
num_before = utils.get_dict_value(params, "num_words_before")
num_after = utils.get_dict_value(params, "num_words_after")
pad_tok = utils.get_dict_value(params, "pad_tok", '<pad>')
sentence = "In simple terms , high precision means that an algorithm " \
"returned substantially more relevant results than irrelevant ones , while" \
" high recall means that an algorithm returned most of the relevant results ."
sentence = "<S> In simple terms , high precision means that algorithm " \
"returned substantially more relevant results than irrelevant ones , while" \
" high recall means that algorithm returned most of relevant results ."
#sentence = "<S> Precision can be seen as measure of exactness or quality , "\
locations = []
probs = []
for candidate_idx, candidate in enumerate(candidates):
_, indexed, _, _ = i.index_wordlist(candidate)
r = e.eval({'sentence': [indexed]}, {'sm_decision'})
print('%s %s' % (candidate[int(len(candidate) / 2)], r[0][0][1]))
# print(candidate)
# print(r)
if r[0][0][1] > thres:
locations.append(candidate_idx)
probs.append(r[0][0][1])
print(probs)
return locations
params = utils.load_param_file('params.10_10.py')
vocab_file = os.path.join(utils.get_dict_value(params, 'output_location'),
'vocab.pkl')
ckpt = os.path.join(utils.get_dict_value(params, 'output_location'),
utils.get_dict_value(params, 'model_name') + '.ckpt')
sentences = [
'To evaluate if the Transformer can generalize to other tasks we performed experiments on English constituency parsing',
'We also trained it in a semi-supervised setting , using the larger high-confidence and BerkleyParser corpora from with approximately 17M sentences',
'We used a vocabulary of 16K tokens for the WSJ only setting and a vocabulary of 32K tokens for the semi-supervised setting',
'Our results in Table 4 show that despite the lack of task-specific tuning our model performs surprisingly well , yielding '
'better results than all previously reported models with the exception of the Recurrrent Neural Network Grammar',
'In this work we presented the Transformer , the first sequence transduction model based entirely on'
' attention replacing the recurrent layers most commonly used in encoder-decoder architectures with multi-headed self-attention',
'We are excited about the future of attention-based models and plan to apply them to other tasks',
from framework.utils.data.text_indexer import TextIndexer
from word_classifier.data import ClassifierData
from framework.evaluator import Evaluator
from shell_command import shell_call
import framework.utils.common as utils
from time import time
import numpy as np
import os
import sys
params = utils.load_param_file(sys.argv[1])
params['num_classes'] = len(params['keywords']) + 1
release_cmd = 'python3 ../tools/release_model.py %s' % sys.argv[1]
shell_call(release_cmd)
vocab_file = os.path.join(utils.get_dict_value(params, 'output_location'),
'vocab.pkl')
release_dir = os.path.join(utils.get_dict_value(params, 'output_location'),
params['model_name'])
graphdef_file = os.path.join(release_dir, params['model_name'] + '.graphdef')
ckpt = os.path.join(utils.get_dict_value(params, 'output_location'),
utils.get_dict_value(params, 'model_name') + '.ckpt')
e = Evaluator.load_graphdef(graphdef_file)
e.dump_variable_sizes()
i = TextIndexer.from_file(vocab_file)
test_data = ClassifierData.get_data_from_dirs(
['/mnt/work/training_data/statmt.tokenized/valid'], params=params)
#test_data = ClassifierData.get_data(params=params)
model_results = []
from framework.utils.data.text_indexer import TextIndexer
from word_classifier.data import ClassifierData
from framework.evaluator import Evaluator
import framework.utils.common as utils
from time import time
import numpy as np
import os
params = utils.load_param_file('output/determinerV3/params.py')
vocab_file = os.path.join(utils.get_dict_value(params, 'output_location'),
'vocab.pkl')
ckpt = os.path.join(utils.get_dict_value(params, 'output_location'),
utils.get_dict_value(params, 'model_name') + '.ckpt')
e = Evaluator.load2(ckpt)
i = TextIndexer.from_file(vocab_file)
test_data = ClassifierData.get_monolingual_test(params=params)
model_results = []
timestr = str(int(time()))
f = open(
os.path.join(utils.get_dict_value(params, 'output_location'),
'heldout_%s.txt' % timestr), 'w')
fe = open(
os.path.join(utils.get_dict_value(params, 'output_location'),
'heldout_%s_err.txt' % timestr), 'w')
fip = open(
os.path.join(utils.get_dict_value(params, 'output_location'),
'heldout_%s_err2.txt' % timestr), 'w')
import os
import numpy as np
import time
import pickle
import math
import sys
from tokenex.tokenizer import Tokenizer
run_server = False
rtime = time.time()
prelim = False
infile = None
out_file = None
paramsfile = sys.argv[1]
params = utils.load_param_file(paramsfile) #'output/rnnlmV8/par# ams.py')
dir_path = os.path.dirname(paramsfile)
if not prelim:
infile = '../we_test/devtest_sent_list.pkl'
out_file = '../we_test/devtest_sent_prob.pkl'
#infile = '../we_test/valid_sent_list.pkl'
#out_file= '../we_test/valid_sent_prob.pkl'
cmd = 'tar -czvf ' + str(rtime) + '.tgz ' + dir_path
print(cmd)
os.system(cmd)
vocab_file = os.path.join(utils.get_dict_value(params, 'output_location'),
'vocab.pkl')
ckpt = os.path.join(utils.get_dict_value(params, 'output_location'),
utils.get_dict_value(params, 'model_name') + '.ckpt')
from framework.evaluator import Evaluator
import framework.utils.common as utils
from tellme.data import TellmeData
import numpy as np
from time import time
import gflags
import os
import sys
params = utils.load_param_file('./output/tellmeV8/params.py')
ckpt = os.path.join(utils.get_dict_value(params, 'output_location'),
utils.get_dict_value(params, 'model_name') + '.ckpt')
accuracy_file = os.path.join(utils.get_dict_value(params, 'output_location'),
'accuracy.txt')
e = Evaluator.load2(ckpt)
e.dump_graph()
e.save_graph_as_pbtxt('tellmev8.pbtxt')
params=params,
gen_data_from_file_fcn=gen_data_from_file_fcn,
gen_data_fcn=gen_data_fcn)
#result = gen_data_from_file('/mnt/work/1-billion-word-language-modeling-benchmark/training-monolingual.tokenized.shuffled/news.en-00001-of-00100')
#for x in result:
# print(x)
if __name__ == "__main__":
from framework.utils.data.text_indexer import TextIndexer
# x = "We went to the store , and I bought some fruits".split()
# print(x)
# y = gen_data(x, [','])
# for yy in y:
# print(yy)
params = utils.load_param_file('../determiner.ccnn/params.py')
indexer = TextIndexer.from_txt_file(
utils.get_dict_value(params, 'vocab_file'),
max_size=utils.get_dict_value(params, 'max_vocab_size', -1))
indexer.add_token('<pad>')
indexer.add_token('unk')
indexer = None
d = ClassifierData.get_data('/mnt/work/data_gen_test',
indexer=indexer,
params=params)
a = d.next_batch(batch_size=20)
for x, y in zip(a['y'], a['sentence']):
print("%s:%s" % (y, x))
# print(training_data.next_batch(batch_size=16))
# x = ['fraud', 'or', 'wrongdoing', 'have', 'contributed', 'to', 'the', 'current', 'problems', ';', 'authorities', 'need', 'to', ',', 'and', 'are', 'prosecuting', 'them', '.', '<pad>']
# a,b = merge_tokens_for_text(x)