def bench(args):
config_dir = '{0}/{1}'.format(args.dir, args.bench_name)
brname = args.bench_name + '-br'
ip = IPRoute()
ctn_intfs = flatten((l.get_attr('IFLA_IFNAME') for l in ip.get_links() if l.get_attr('IFLA_MASTER') == br) for br in ip.link_lookup(ifname=brname))
if not args.repeat:
# currently ctn name is same as ctn intf
# TODO support proper mapping between ctn name and intf name
for ctn in ctn_intfs:
dckr.remove_container(ctn, force=True) if ctn_exists(ctn) else None
if os.path.exists(config_dir):
shutil.rmtree(config_dir)
else:
for ctn in ctn_intfs:
if ctn != 'tester':
dckr.remove_container(ctn, force=True) if ctn_exists(ctn) else None
if args.file:
with open(args.file) as f:
conf = yaml.load(Template(f.read()).render())
else:
conf = gen_conf(args)
if not os.path.exists(config_dir):
os.makedirs(config_dir)
with open('{0}/scenario.yaml'.format(config_dir), 'w') as f:
f.write(conf)
conf = yaml.load(Template(conf).render())
num_tester = sum(len(t.get('tester', [])) for t in conf.get('testers', []))
if num_tester > gc_thresh3():
print 'gc_thresh3({0}) is lower than the number of peer({1})'.format(gc_thresh3(), num_tester)
print 'type next to increase the value'
print '$ echo 16384 | sudo tee /proc/sys/net/ipv4/neigh/default/gc_thresh3'
if args.target == 'gobgp':
target = GoBGP
elif args.target == 'bird':
target = BIRD
elif args.target == 'quagga':
target = Quagga
is_remote = True if 'remote' in conf['target'] and conf['target']['remote'] else False
if is_remote:
r = ip.get_routes(dst=conf['target']['local-address'].split('/')[0], family=AF_INET)
if len(r) == 0:
print 'no route to remote target {0}'.format(conf['target']['local-address'])
sys.exit(1)
idx = [t[1] for t in r[0]['attrs'] if t[0] == 'RTA_OIF'][0]
intf = ip.get_links(idx)[0]
if intf.get_attr('IFLA_MASTER') not in ip.link_lookup(ifname=brname):
br = ip.link_lookup(ifname=brname)
if len(br) == 0:
ip.link_create(ifname=brname, kind='bridge')
br = ip.link_lookup(ifname=brname)
br = br[0]
ip.link('set', index=idx, master=br)
else:
print 'run', args.target
if args.image:
target = target(args.target, '{0}/{1}'.format(config_dir, args.target), image=args.image)
else:
target = target(args.target, '{0}/{1}'.format(config_dir, args.target))
target.run(conf, brname)
print 'run monitor'
m = Monitor('monitor', config_dir+'/monitor')
m.run(conf, brname)
time.sleep(1)
print 'waiting bgp connection between {0} and monitor'.format(args.target)
m.wait_established(conf['target']['local-address'].split('/')[0])
if not args.repeat:
print 'run tester'
for idx, tester in enumerate(conf['testers']):
if 'name' not in tester:
name = 'tester{0}'.format(idx)
else:
name = tester['name']
if 'type' not in tester:
tester_type = 'normal'
else:
tester_type = tester['type']
if tester_type == 'normal':
t = Tester(name, config_dir+'/'+name)
elif tester_type == 'mrt':
t = MRTTester(name, config_dir+'/'+name)
else:
print 'invalid tester type:', tester_type
sys.exit(1)
t.run(tester, conf['target'], brname)
start = datetime.datetime.now()
q = Queue()
m.stats(q)
if not is_remote:
target.stats(q)
def mem_human(v):
if v > 1000 * 1000 * 1000:
return '{0:.2f}GB'.format(float(v) / (1000 * 1000 * 1000))
elif v > 1000 * 1000:
return '{0:.2f}MB'.format(float(v) / (1000 * 1000))
elif v > 1000:
return '{0:.2f}KB'.format(float(v) / 1000)
else:
return '{0:.2f}B'.format(float(v))
f = open(args.output, 'w') if args.output else None
cpu = 0
mem = 0
cooling = -1
while True:
info = q.get()
if not is_remote and info['who'] == target.name:
cpu = info['cpu']
mem = info['mem']
if info['who'] == m.name:
now = datetime.datetime.now()
elapsed = now - start
recved = info['state']['adj-table']['accepted'] if 'accepted' in info['state']['adj-table'] else 0
if elapsed.seconds > 0:
rm_line()
print 'elapsed: {0}sec, cpu: {1:>4.2f}%, mem: {2}, recved: {3}'.format(elapsed.seconds, cpu, mem_human(mem), recved)
f.write('{0}, {1}, {2}, {3}\n'.format(elapsed.seconds, cpu, mem, recved)) if f else None
f.flush() if f else None
if cooling == args.cooling:
f.close() if f else None
return
if cooling >= 0:
cooling += 1
if info['checked']:
cooling = 0
import os
import sys
sys.path.append(os.getcwd())
import numpy as np
import tensorflow as tf
from tester import Tester
ts = Tester()
### Forward pass
def matmul(lhs, rhs):
return np.matmul(lhs, rhs)
X = np.random.randn(13, 7)
Y = np.random.randn(7, 29)
out_tf = tf.matmul(X, Y).numpy()
out_np = matmul(X, Y)
ts.check_tensors('matmul', out_tf, out_np)
### Backward pass
def tf_matmul_grad(lhs, rhs, dout):
v_lhs = tf.Variable(lhs)
v_rhs = tf.Variable(rhs)
if args.train:
trainer = Trainer(agent, config)
loop.run_until_complete(trainer.train())
loop.run_forever()
elif args.retrain:
if args.retrain_model is None:
print('please add the retrain model path:', '--retrain_model xxxx')
exit(0)
ep, step = agent.load_checkpoint(args.retrain_model)
trainer = Trainer(agent, config)
loop.run_until_complete(trainer.train())
loop.run_forever()
elif args.test:
if args.model_path is None:
print('please add the model path:', '--model_path xxxx')
exit(0)
# record
# if args.record:
# os.makedirs('video', exist_ok=True)
# filepath = 'video/' + args.env + '-' + time_seq()
# env = wrappers.Monitor(env, filepath, video_callable=lambda episode_id: episode_id % 25 == 0)
tester = Tester(agent, env, model_path=args.model_path)
tester.test()
else:
print('choose train or test:', '--train or --test')
# edges = False turns off the top current loop
ext_kernel = GaussianKernel(mask[sl].shape, params, edges=False)
g_scale = ext_kernel.applyM(g)[window].ptp()
true_g = g / g_scale
g_ext = g_ext / g_scale
# True flux data
phi = ext_kernel.applyM(true_g)[window]
ext_flux = ext_kernel.applyM(g_ext)
tester = Tester(true_g[window],
phi,
kernel,
sigma,
g_ext=g_ext[window].ravel(),
phi_ext=ext_flux[window].ravel())
mirror_tester = Tester(true_g[window], phi, mirror_kernel, sigma)
control_tester = Tester(true_g[window], phi, kernel, sigma)
admm_kwargs = {
'iprint': 1,
'eps_rel': 1e-8,
'eps_abs': 1e-8,
'itnlim': 300,
'rho': 1e-4
}
L_factor = 0.8
TV_factor = 0.9
def train(config):
self_summary_strs = [] # summary string to print out for later
# first, read both data and filter stuff, to get the word2vec idx,
train_data = read_data(config,'train',config.load)
val_data = read_data(config,'val',True) # dev should always load model shared data(word2idx etc.) from train
# now that the dataset is loaded , we get the max_word_size from the dataset
# then adjust the max based on the threshold as well
# also get the vocab size
config_vars = vars(config)
str_ = "threshold setting--\n" + "\t"+ " ,".join(["%s:%s"%(key,config_vars[key]) for key in config.thresmeta])
print (str_)
self_summary_strs.append(str_)
# cap the numbers
# max sentence word count etc.
update_config(config,[train_data,val_data],showMeta=True) # all word num is <= max_thres
str_ = "renewed ----\n"+"\t" + " ,".join(["%s:%s"%(key,config_vars[key]) for key in config.maxmeta])
print (str_)
self_summary_strs.append(str_)
# now we initialize the matrix for word embedding for word not in glove
word2vec_dict = train_data.shared['word2vec']
word2idx_dict = train_data.shared['word2idx'] # this is the word not in word2vec
# we are not fine tuning , so this should be empty
idx2vec_dict = {word2idx_dict[word]:vec for word,vec in word2vec_dict.items() if word in word2idx_dict}
#print len(idx2vec_dict) # 0
# config.word_vocab_size = len(train_data.shared['word2idx']) # the word not in word2vec
# so the emb_mat should all be a random vector
# np.random.multivariate_normal gets mean of zero and co of 1 for each dim, like
#>>> np.random.multivariate_normal(np.zeros(5),np.eye(5))
#array([-0.73663652, -1.16417783, -0.74083293, -0.80016731, 0.060182 ])
# random initial embedding matrix for new words
config.emb_mat = np.array([idx2vec_dict[idx] if idx in idx2vec_dict else np.random.multivariate_normal(np.zeros(config.word_emb_size), np.eye(config.word_emb_size)) for idx in range(config.word_vocab_size)],dtype="float32")
model = get_model(config) # construct model under gpu0
trainer = Trainer(model,config)
tester = Tester(model,config)
saver = tf.train.Saver(max_to_keep=5) # how many model to keep
bestsaver = tf.train.Saver(max_to_keep=5) # just for saving the best model
save_period = config.save_period # also the eval period
# start training!
# allow_soft_placement : tf will auto select other device if the tf.device(*) not available
tfconfig = tf.ConfigProto(allow_soft_placement=True)
tfconfig.gpu_options.allow_growth = True # this way it will only allocate nessasary gpu, not take all
# or you can set hard limit
#tfconfig.gpu_options.per_process_gpu_memory_fraction = 0.4
with tf.Session(config=tfconfig) as sess:
# calculate total parameters
totalParam = cal_total_param()
str_ = "total parameters: %s"%(totalParam)
print (str_)
self_summary_strs.append(str_)
initialize(load=config.load,load_best=config.load_best,model=model,config=config,sess=sess)
# the total step (iteration) the model will run
last_time = time.time()
# total / batchSize * epoch
num_steps = int(math.ceil(train_data.num_examples/float(config.batch_size)))*config.num_epochs
# get_batches is a generator, run on the fly
# there will be num_steps batch
str_ = " batch_size:%s, epoch:%s,total step:%s,eval/save every %s steps"%(config.batch_size,config.num_epochs,num_steps,config.save_period)
print (str_)
self_summary_strs.append(str_)
best = {"acc":0.0,"step":-1} # remember the best eval acc during training
finalAcc = None
isStart = True
for batch in tqdm(train_data.get_batches(config.batch_size,num_steps=num_steps),total=num_steps):
# each batch has (batch_idxs,Dataset(batch_data, full_shared))
# batch_data has {"q":,"y":..."pidx2feat",.."photo_idxs"..}
global_step = sess.run(model.global_step) + 1 # start from 0
# if load from existing model, save if first
if config.load and isStart:
tqdm.write("saving original model...")
tqdm.write("\tsaving model...")
saver.save(sess,config.save_dir_model,global_step=global_step)
tqdm.write("\tdone")
isStart=False
id2predanswers = {}
id2realanswers = {}
for evalbatch in val_data.get_batches(config.batch_size,num_steps=config.val_num_batches,shuffle=False,cap=True):
yp = tester.step(sess,evalbatch) # [N,4] # id2realanswersprob for each answer
pred,gt = getAnswers(yp,evalbatch) # from here we get the qid:yindx,
id2predanswers.update(pred)
id2realanswers.update(gt)
evalAcc = getEvalScore(id2predanswers,id2realanswers)
tqdm.write("\teval on validation %s batches Acc:%s, (best:%s at step %s) "%(config.val_num_batches,evalAcc,best['acc'],best['step']))
# remember the best acc
if(evalAcc > best['acc']):
best['acc'] = evalAcc
best['step'] = global_step
# save the best model
tqdm.write("\t saving best model...")
bestsaver.save(sess,config.save_dir_best_model,global_step=global_step)
tqdm.write("\t done.")
finalAcc = evalAcc
loss,train_op = trainer.step(sess,batch)
if global_step % save_period == 0: # time to save model
duration = time.time() - last_time # in seconds
sec_per_step = duration/float(save_period)
last_time = time.time()
#use tqdm to print
tqdm.write("step:%s/%s (epoch:%.3f), took %s, loss:%s, estimate remaining:%s"%(global_step,num_steps,(config.num_epochs*global_step/float(num_steps)),sec2time(duration),loss,sec2time((num_steps - global_step)*sec_per_step)))
tqdm.write("\tsaving model...")
saver.save(sess,config.save_dir_model,global_step=global_step)
tqdm.write("\tdone")
id2predanswers = {}
id2realanswers = {}
for evalbatch in val_data.get_batches(config.batch_size,num_steps=config.val_num_batches,shuffle=False,cap=True):
yp = tester.step(sess,evalbatch) # [N,4] # id2realanswersprob for each answer
pred,gt = getAnswers(yp,evalbatch) # from here we get the qid:yindx,
id2predanswers.update(pred)
id2realanswers.update(gt)
evalAcc = getEvalScore(id2predanswers,id2realanswers)
tqdm.write("\teval on validation %s batches Acc:%s, (best:%s at step %s) "%(config.val_num_batches,evalAcc,best['acc'],best['step']))
# remember the best acc
if(evalAcc > best['acc']):
best['acc'] = evalAcc
best['step'] = global_step
# save the best model
tqdm.write("\t saving best model...")
bestsaver.save(sess,config.save_dir_best_model,global_step=global_step)
tqdm.write("\t done.")
finalAcc = evalAcc
#exit()
if global_step % save_period != 0: # time to save model
saver.save(sess,config.save_dir_model,global_step=global_step)
str_ = "best eval on val Accurucy: %s at %s step, final step %s Acc is %s"%(best['acc'],best['step'], global_step,finalAcc)
print (str_)
self_summary_strs.append(str_)
if config.write_self_sum:
f = open(config.self_summary_path,"w")
f.writelines("%s"%("\n".join(self_summary_strs)))
f.close()
def main():
f = open('try_3.txt', 'w')
g = open('accs.txt', 'w')
g.close()
task = MarioTask("testbed", initMarioMode=2)
task.env.initMarioMode = 2
task.env.levelDifficulty = 1
results = []
names = []
with open('type.txt', 'w') as f:
f.write('dt')
iterations = 20
rounds = 30
learning_samples = 33
eval_samples = 10
# iterations = 5
# rounds = 2
# learning_samples = 3
# eval_samples = 2
agent = Dagger(IT, useKMM=False)
if args['linear']:
agent.learner.linear = True
prefix = 'svc-dagger-change-'
else:
agent.learner.linear = False
prefix = 'dt-dagger-change-'
exp = EpisodicExperiment(task, agent)
T = Tester(agent, exp)
dagger_data, _, acc, loss, js, test_acc = T.test(
rounds=rounds,
iterations=iterations,
learning_samples=learning_samples,
eval_samples=eval_samples,
prefix=prefix)
np.save('./data/' + prefix + 'dagger_data.npy', dagger_data)
np.save('./data/' + prefix + 'acc.npy', acc)
np.save('./data/' + prefix + 'loss.npy', loss)
np.save('./data/' + prefix + 'js.npy', js)
np.save('./data/' + prefix + 'test_acc.npy', test_acc)
analysis = Analysis()
analysis.get_perf(dagger_data, range(iterations))
analysis.plot(names=['DAgger'],
label='Reward',
filename='./results/' + prefix + 'return_plots.eps')
acc_a = Analysis()
acc_a.get_perf(acc, range(iterations))
acc_a.plot(names=['DAgger Acc.'],
label='Accuracy',
filename='./results/' + prefix + 'acc_plots.eps',
ylims=[0, 1])
test_acc_a = Analysis()
test_acc_a.get_perf(test_acc, range(iterations))
test_acc_a.plot(names=['DAgger Acc.'],
label='Test Accuracy',
filename='./results/' + prefix + 'test_acc_plots.eps',
ylims=[0, 1])
loss_a = Analysis()
loss_a.get_perf(loss, range(iterations))
loss_a.plot(names=['DAgger Loss'],
label='Loss',
filename='./results/' + prefix + 'loss_plots.eps',
ylims=[0, 1])
js_a = Analysis()
js_a.get_perf(js, range(iterations))
js_a.plot(names=['DAgger'],
label='J()',
filename='./results/' + prefix + '-js_plots.eps')
print "finished"
"--config",
nargs="?",
type=str,
default="syndoc.yml",
help="Config file name")
parser.add_argument("-s",
"--seed",
nargs="?",
type=int,
default=4321,
help="Seed number")
parser.add_argument('-wt',
'--with_test',
action='store_true',
help='Whether to run corresponding Tester')
args = parser.parse_args()
config = coerce_to_path_and_check_exist(CONFIGS_PATH / args.config)
run_dir = MODELS_PATH / args.tag
trainer = Trainer(config, run_dir, seed=args.seed)
trainer.run(seed=args.seed)
if args.with_test:
dataset_name = trainer.dataset_name
output_dir = run_dir / "test_{}".format(dataset_name)
tester = Tester(output_dir, run_dir / MODEL_FILE, dataset_name,
trainer.dataset_kwargs)
tester.run()
def _benchmark(self):
return Tester(self).benchmark
def fib_matrix(n):
if n < 2:
return n
n -= 1
res = np.eye(2, dtype='object')
base = np.ones((2, 2), dtype='object')
base[1, 1] = 0
while n:
if n % 2 == 1:
# res = res @ base
# being more explicit and taking advantage of res[0, 1] == res[1, 0] and res[0, 0] == res[1, 1] + res[0, 1]
res[1, 1] = res[1, :] @ base[:, 1]
res[[0, 1], [1, 0]] = res[0, :] @ base[:, 1]
res[0, 0] = res[1, 1] + res[0, 1]
base = base @ base
n >>= 1
return res[0, 0]
if __name__ == '__main__':
from tester import Tester
algos = [(fib_recursive, 'recursive'), (fib_iterative, 'iterative'),
(fib_golden_ratio, 'golden-ratio-based'),
(fib_matrix, 'matrix-based')]
for algo, desc in algos:
print(f'\nResults for {desc} algorithm\n')
tester = Tester(algo, '4.Fibo')
tester.run_tests()
def run_testing(
path_to_data: str = '/Users/yaeger/Documents/Porphyria',
path_to_models:
str = '/Users/yaeger/Documents/Modules/Porphyria/models',
save_directory:
str = '/Users/yaeger/Documents/Modules/Porphyria/results/testing',
metrics: list = [roc_auc_score],
results_file_name: str = 'test_set_results.csv'):
"""Wrapper method for running the tester on all of the models in a directory
for an input list of evaluation metrics. Writes a .csv to a file named
according to the results_file_name argument with columns specifying the name
of each model and each metric. Each row contains the results for each metric
for a different model.
INPUTS:
path_to_data: where the data lives.
path_to_models: where the models live.
save_directory: the directory where the results should be saved. If this
directory does not exist, it will be created.
metrics: a list of metrics accepting y_true and y_pred (or y_score). These
metrics should NOT accept a classifier as an argument.
results_file_name: the name to use to save the results file.
RETURNS:
None
"""
# Make a directory to store results
if not isinstance(save_directory, Path):
save_directory = Path(save_directory)
if not save_directory.is_dir():
save_directory.mkdir()
save_path = save_directory.joinpath(results_file_name)
# Create a csv file to store results
metric_names = list(
map(lambda x: re.findall(r'function (.*) at', str(x))[0], metrics))
metric_names.insert(0, 'fraction_correctly_ranked')
metric_names.insert(0, 'model_name')
with save_path.open('w') as fh:
writer = csv.writer(fh)
writer.writerow(metric_names)
# Create a dictionary to store parameter values
param_dict = {}
# Instantiate Tester object
tester = Tester(path_to_data=path_to_data, metrics=metrics)
# Convert path_to_models to a Path and iterate over it to test all models
# in directory
if not isinstance(path_to_models, Path):
path_to_models = Path(path_to_models)
for model_path in path_to_models.iterdir():
if model_path.name == '.DS_Store':
continue
try:
results_dict, parameters = tester.evaluate_model(model_path)
param_dict = update_param_dict(parameters, param_dict)
with save_path.open('a') as fh:
writer = csv.writer(fh)
writer.writerow([results_dict[name] for name in metric_names])
except:
continue
# Put parameters into a dataframe
param_directory = save_directory.joinpath(
results_file_name.split('.')[0] + '_model_parameters.csv')
params_df = pd.DataFrame.from_dict(param_dict)
# Reorder so that model name is first column
new_columns = list(params_df.columns)
new_columns.pop(new_columns.index('model_name'))
new_columns.insert(0, 'model_name')
params_df = params_df.reindex(columns=new_columns)
params_df.to_csv(param_directory, index=False)
def _verify(self):
return Tester(self).verify
#!/usr/bin/python
# -*- coding: utf8 -*-
from tester import Tester
from multiprocessing.dummy import Pool as ThreadPool
import threading
import time
import sys
import logging
import traceback
w = Tester()
pool = ThreadPool(w.maxThreads)
loop = True
result = None
def main():
global loop
global result
th = threading.Thread(target=main_loop)
th.start()
try:
while True:
time.sleep(1)
if not th.is_alive():
logging.critical("Main loop thread is dead, exiting")
return
except KeyboardInterrupt:
logging.warning("Interrupt signal received, exiting")
loop = False
if result is not None:
import matplotlib.pyplot as plt
from nltk.tag.stanford import NERTagger
TRAINING_DATA_PATH = os.path.dirname(os.path.abspath(__file__)) + \
'/weps2007_data_1.1/training/'
NER_CLASSIFIER = os.path.dirname(os.path.abspath(__file__)) + \
'/stanford-ner-2015-04-20/classifiers/english.all.3class.distsim.crf.ser.gz'
NER_JAR = os.path.dirname(os.path.abspath(__file__)) + \
'/stanford-ner-2015-04-20/stanford-ner.jar'
corpus = Corpus(TRAINING_DATA_PATH)
corpus.read_person_searches()
t = Tester(TRAINING_DATA_PATH)
classifier = NERTagger(NER_CLASSIFIER, NER_JAR)
results = [0, 0]
results2 = [0, 0]
for ps in corpus.person_searches:
#tfs = features.TokenFeatureSet(ps.name, pd)
psf = features.PersonSearchFeatures(ps)
print ps.name
# print len(psf.parsed_docs)
#print psf.distance_matrix
Z = linkage(psf.distance_matrix, method='complete')
# print psf.distance_matrix
_trainer.train_loop()
elif args.test:
del train
del val
models_dir = list(
filter(lambda x: "_" in str(x), list(Path("models/").iterdir())))
for model_file in models_dir:
model = load_model(model_file)
name = str(model_file).split("/")[-1].split(".")[0]
tsteps = int(name.split("_")[1])
_tester = Tester(test_set=test[7, 3000:10000, :],
show_set=test[0, :, :],
window=tsteps,
model=model,
model_name=name,
store_dir="final_run1/")
_tester.run_tests()
# print_model_debug(model)
# _tester.error_test()
# tsteps = [2, 5, 10, 20, 50, 100, 200]
#
# for tstep in tsteps:
# ewm_ = EWM_Model(dataset=test[7, 3000:10000, :], timesteps=tstep)
# ewm_.do_tests()
def scheduler_tester(self, cycle=TESTER_CYCLE):
tester = Tester()
while True:
print("测试器开始运行")
tester.run()
time.sleep(cycle)
def main():
f = open('try_3.txt', 'w')
g = open('accs.txt', 'w')
g.close()
task = MarioTask("testbed", initMarioMode=2)
task.env.initMarioMode = 2
task.env.levelDifficulty = 1
results = []
names = []
iterations = 50
rounds = 15
learning_samples = 2
eval_samples = 3
agent = Supervise(IT, useKMM=False)
exp = EpisodicExperiment(task, agent)
T = Tester(agent, exp)
sl_data, sup_data, acc = T.test(rounds=rounds, iterations=iterations)
np.save('./data/sup_data.npy', sup_data)
np.save('./data/sl_data.npy', sl_data)
np.save('./data/acc.npy', acc)
IPython.embed()
analysis = Analysis()
analysis.get_perf(sup_data, range(iterations))
analysis.get_perf(sl_data, range(iterations))
analysis.plot(names=['Supervisor', 'Supervised Learning'],
label='Reward',
filename='./results/return_plots.eps') #, ylims=[0, 1600])
acc_a = Analysis()
acc_a.get_perf(acc, range(iterations))
acc_a.plot(names=['Supervised Learning Acc.'],
label='Accuracy',
filename='./results/acc_plots.eps')
"""
agent = Dagger(IT,useKMM = False)
exp = EpisodicExperiment(task, agent)
T = Tester(agent,exp)
dagger_data, _, acc = T.test(rounds = rounds, iterations = iterations)
np.save('./data/dagger_data.npy', dagger_data)
np.save('./data/acc.npy', acc)
IPython.embed()
analysis = Analysis()
analysis.get_perf(dagger_data, range(iterations))
analysis.plot(names=['DAgger'], label='Reward', filename='./results/return_plots.eps')
acc_a = Analysis()
acc_a.get_perf(acc, range(iterations))
acc_a.plot(names=['DAgger Acc.'], label='Accuracy', filename='./results/acc_plots.eps')
"""
#agent = Supervise(IT,useKMM = False)
#exp = EpisodicExperiment(task, agent)
#T = Tester(agent,exp)
#supervise_results = T.test(rounds = rounds, iterations = iterations)
#supervise_data = supervise_results[-1]
#supervise_results = supervise_results[:-1]
#results.append(supervise_results)
#names.append('supervise')
#pickle.dump(results,open('results.p','wb'))
#IPython.embed()
#analysis = Analysis()
#analysis.get_perf(supervise_data, results[1][5])
#analysis.get_perf(dagger_data, results[0][5])
#analysis.plot(names=['Supervise', 'DAgger'], label='Reward', filename='./return_plot.eps')#, ylims=[-1, 0])
# agent = Sheath(IT,useKMM = False,sigma = 1.0)
# exp = EpisodicExperiment(task, agent)
# T = Tester(agent,exp)
# dagger_results = T.test(rounds = 10,iterations = 35)
# results.append(dagger_results)
# names.append('sheath_1')
# pickle.dump(results,open('results.p','wb'))
# agent = Sheath(IT,useKMM = False,sigma = 1e-1)
# exp = EpisodicExperiment(task, agent)
# T = Tester(agent,exp)
# dagger_results = T.test(rounds = 10,iterations = 35)
# results.append(dagger_results)
# names.append('sheath_1')
# pickle.dump(results,open('results.p','wb'))
# agent = Sheath(IT,useKMM = False,sigma = 0.5)
# exp = EpisodicExperiment(task, agent)
# T = Tester(agent,exp)
# dagger_results = T.test(rounds = 10,iterations = 35)
# results.append(dagger_results)
# names.append('sheath_1')
# pickle.dump(results,open('results.p','wb'))
# agent = Sheath(IT,useKMM = False,sigma = 1e-1)
# exp = EpisodicExperiment(task, agent)
# T = Tester(agent,exp)
# dagger_results = T.test(rounds = 4,iterations = 35)
# results.append(dagger_results)
# names.append('sheath_1')
# agent = Sheath(IT,useKMM = False,sigma = 1e-2)
# exp = EpisodicExperiment(task, agent)
# T = Tester(agent,exp)
# dagger_results = T.test(rounds = 4,iterations = 35)
# results.append(dagger_results)
# names.append('sheath_1')
# # # # # #test big ahude
# agent = Ahude(IT,f,gamma = 1e-2,labelState = True, useKMM = True)
# exp = EpisodicExperiment(task, agent)
# T = Tester(agent,exp)
# ahude_big_results = T.test(rounds = 3)
# results.append(ahude_big_results)
# names.append('ahude_1e-1')
# pickle.dump(results,open('results.p','wb'))
# # # # # #test med ahude
# agent = Ahude(IT,f,gamma = 1e-2,labelState = False,useKMM = True)
# exp = EpisodicExperiment(task, agent)
# T = Tester(agent,exp)
# ahude_med_results = T.test(rounds = 3)
# results.append(ahude_med_results)
# names.append('ahude_1e-2')
# # #
# # # # # # #test small ahude
# agent = Ahude(IT,f,gamma = 1e-3)
# exp = EpisodicExperiment(task, agent)
# T = Tester(agent,exp)
# ahude_small_results = T.test()
# results.append(ahude_small_results)
# names.append('ahude_1e-3')
# pickle.dump(results,open('results.p','wb'))
#plt.figure(1)
#for i in range(len(results)):
# plt.plot(results[i][5],results[i][1])
#plt.legend(names,loc='upper left')
# plt.figure(2)
# for i in range(len(results)):
# plt.plot(results[i][0])
# plt.legend(names,loc='upper left')
# plt.figure(3)
# for i in range(0,len(results)):
# plt.plot(results[i][3])
# plt.legend(names,loc='upper left')
plt.show()
IPython.embed()
f.close()
#agent.saveModel()
print "finished"
def train_and_eval(self):
print("Training the %s model..." % self.model_name)
print("Number of training data points: %d" %
len(self.dataset.data["train"]))
if (self.model_name == "MDistMult"):
model = MDistMult(self.dataset, self.emb_dim,
**self.kwargs).to(self.device)
elif (self.model_name == "MCP"):
model = MCP(self.dataset, self.emb_dim,
**self.kwargs).to(self.device)
elif (self.model_name == "HSimplE"):
model = HSimplE(self.dataset, self.emb_dim,
**self.kwargs).to(self.device)
elif (self.model_name == "HypE"):
model = HypE(self.dataset, self.emb_dim,
**self.kwargs).to(self.device)
elif (self.model_name == "MTransH"):
model = MTransH(self.dataset, self.emb_dim,
**self.kwargs).to(self.device)
model.init()
opt = torch.optim.Adagrad(model.parameters(), lr=self.learning_rate)
loss_layer = torch.nn.CrossEntropyLoss()
print("Starting training...")
best_mrr = 0
for it in range(1, self.num_iterations + 1):
last_batch = False
model.train()
losses = 0
while not last_batch:
r, e1, e2, e3, e4, e5, e6, targets, ms, bs = self.dataset.next_batch(
self.batch_size,
neg_ratio=self.neg_ratio,
device=self.device)
last_batch = self.dataset.was_last_batch()
opt.zero_grad()
number_of_positive = len(np.where(targets > 0)[0])
if (self.model_name == "HypE" or self.model_name == "HypE_DM"):
predictions = model.forward(r, e1, e2, e3, e4, e5, e6, ms,
bs)
elif (self.model_name == "MTransH"):
predictions = model.forward(r, e1, e2, e3, e4, e5, e6, ms)
else:
predictions = model.forward(r, e1, e2, e3, e4, e5, e6)
predictions = self.padd_and_decompose(
targets, predictions, self.neg_ratio * self.max_arity)
targets = torch.zeros(number_of_positive).long().to(
self.device)
loss = loss_layer(predictions, targets)
loss.backward()
opt.step()
losses += loss.item()
print("iteration#: " + str(it) + ", loss: " + str(losses))
if (it % 100 == 0):
model.eval()
with torch.no_grad():
print("validation:")
tester = Tester(self.dataset, model, "valid",
self.model_name)
mrr = tester.test()
if (mrr > best_mrr):
best_mrr = mrr
best_model = model
best_itr = it
best_model.eval()
with torch.no_grad():
print("test in iteration " + str(best_itr) + ":")
tester = Tester(self.dataset, best_model, "test", self.model_name)
tester.test()
def train(config):
self_summary_strs = [] # summary string to print out for later
val_perf = [] # summary of validation performance
# first, read both data and filter stuff, to get the word2vec idx,
train_data = read_data(config, 'train', config.load)
config.imgfeat_dim = train_data.imgfeat_dim
val_data = read_data(
config, 'val', True, subset=False
) # dev should always load model shared data(word2idx etc.) from train
# now that the dataset is loaded , we get the max_word_size from the dataset
# then adjust the max based on the threshold as well
# also get the vocab size
config_vars = vars(config)
str_ = "threshold setting--\n" + "\t" + " ,".join(
["%s:%s" % (key, config_vars[key]) for key in config.thresmeta])
print str_
self_summary_strs.append(str_)
# cap the numbers
# max sentence word count etc.
update_config(config, [train_data, val_data],
showMeta=True) # all word num is <= max_thres
str_ = "renewed ----\n" + "\t" + " ,".join(
["%s:%s" % (key, config_vars[key]) for key in config.maxmeta])
print str_
self_summary_strs.append(str_)
# now we initialize the matrix for word embedding for word not in glove
word2vec_dict = train_data.shared[
'word2vec'] # empty if not use pre-train vector
word2idx_dict = train_data.shared[
'word2idx'] # this is the word not in word2vec # for finetuning or not using word2vec then it is all the word
# we are not fine tuning , so this should be empty; empty if not use pretrain vector
# if finetuning , this will have glove
idx2vec_dict = {
word2idx_dict[word]: vec
for word, vec in word2vec_dict.items() if word in word2idx_dict
}
print "len(idx2vec_dict):%s,word_vocab_size:%s" % (len(idx2vec_dict),
config.word_vocab_size)
# config.word_vocab_size = len(train_data.shared['word2idx']) # the word not in word2vec
# so the emb_mat should all be a random vector
# np.random.multivariate_normal gets mean of zero and co of 1 for each dim, like
#>>> np.random.multivariate_normal(np.zeros(5),np.eye(5))
#array([-0.73663652, -1.16417783, -0.74083293, -0.80016731, 0.060182 ])
# random initial embedding matrix for new words
# this will take a long time when vocab_size is 6k
if not config.no_wordvec:
config.emb_mat = np.array([
idx2vec_dict[idx]
if idx2vec_dict.has_key(idx) else np.random.multivariate_normal(
np.zeros(config.word_emb_size), np.eye(config.word_emb_size))
for idx in xrange(config.word_vocab_size)
],
dtype="float32")
model = get_model(config) # construct model under gpu0
#for var in tf.trainable_variables():
# print var.name,var.get_shape()
#sys.exit()
trainer = Trainer(model, config)
tester = Tester(model, config)
saver = tf.train.Saver(max_to_keep=5) # how many model to keep
bestsaver = tf.train.Saver(max_to_keep=5) # just for saving the best model
save_period = config.save_period # also the eval period
# start training!
# allow_soft_placement : tf will auto select other device if the tf.device(*) not available
tfconfig = tf.ConfigProto(allow_soft_placement=True)
tfconfig.gpu_options.allow_growth = True # this way it will only allocate nessasary gpu, not take all
# or you can set hard limit
#tfconfig.gpu_options.per_process_gpu_memory_fraction = 0.4
with tf.Session(config=tfconfig) as sess:
# calculate total parameters
totalParam = cal_total_param()
str_ = "total parameters: %s" % (totalParam)
print str_
self_summary_strs.append(str_)
initialize(load=config.load,
load_best=config.load_best,
model=model,
config=config,
sess=sess)
# the total step (iteration) the model will run
last_time = time.time()
# total / batchSize * epoch
num_steps = int(
math.ceil(train_data.num_examples /
float(config.batch_size))) * config.num_epochs
# get_batches is a generator, run on the fly
# there will be num_steps batch
str_ = " batch_size:%s, epoch:%s, %s step every epoch, total step:%s,eval/save every %s steps" % (
config.batch_size, config.num_epochs,
math.ceil(train_data.num_examples / float(config.batch_size)),
num_steps, config.save_period)
print str_
self_summary_strs.append(str_)
metric = "t2i_r@5" # TODO: change this?
best = {
metric: 0.0,
"step": -1
} # remember the best eval acc during training
finalperf = None
isStart = True
for batch in tqdm(train_data.get_batches(config.batch_size,
no_img_feat=True,
num_steps=num_steps),
total=num_steps,
ascii=True):
# each batch has (batch_idxs,Dataset(batch_data, full_shared))
# batch_data has {"q":,"y":..."pidx2feat",.."photo_idxs"..}
global_step = sess.run(model.global_step) + 1 # start from 0
# if load from existing model, save if first
if (global_step % save_period
== 0) or (config.load_best
and isStart) or (config.load and isStart and
(config.ignore_vars is None)):
isStart = False
tqdm.write("\tsaving model %s..." % global_step)
saver.save(sess,
config.save_dir_model,
global_step=global_step)
tqdm.write("\tdone")
evalperf = evaluate(val_data, config, sess, tester)
tqdm.write(
"\teval on validation:%s, (best %s:%s at step %s) " %
(evalperf, metric, best[metric], best['step']))
# remember the best acc
if (evalperf[metric] > best[metric]):
best[metric] = evalperf[metric]
best['step'] = global_step
# save the best model
tqdm.write("\t saving best model...")
bestsaver.save(sess,
config.save_dir_best_model,
global_step=global_step)
tqdm.write("\t done.")
finalperf = evalperf
val_perf.append(evalperf)
batchIdx, batch_data = batch
batch_data = batch_data.data
# each batch is ['pos'],['neg']
# hard negative mining
# pair each pos image with max_similarity(pos_img,neg_sent)
# and pos sent with max_similarity(pos_sent,neg_img)
assert len(batch_data['pos']) == len(batch_data['neg'])
# TODO: select hn_num of negative here, save computation?
alldata = batch_data['pos'] + batch_data[
'neg'] # (imgid,sent,sent_c)
# 1. get all pos and neg's image and sentence embeddding
all_imgs = list(set([one[0] for one in alldata]))
imgid2idx = {}
for imgid in all_imgs:
imgid2idx[imgid] = len(imgid2idx.keys())
# load the actual image feature matrix
image_feats = load_feats(imgid2idx, train_data.shared, config)
mining_batch = {}
mining_batch['imgs'] = [one[0] for one in alldata]
mining_batch['imgid2idx'] = imgid2idx
mining_batch['imgidx2feat'] = image_feats
mining_batch['data'] = [(one[1], one[2]) for one in alldata
] # a list of (sent,sent_c)
# mining_batch, N_pos+N_neg
z_u, z_v = tester.step(sess,
(batchIdx,
Dataset(mining_batch,
"test",
shared=train_data.shared,
is_train=False,
imgfeat_dim=config.feat_dim)))
assert len(z_u) == len(z_v), (len(z_u), len(z_v))
z_u_pos = z_u[:len(batch_data['pos'])]
z_v_pos = z_v[:len(batch_data['pos'])]
z_u_neg = z_u[len(batch_data['pos']):]
z_v_neg = z_v[len(batch_data['pos']):]
assert len(z_u_pos) == len(z_v_pos) == len(z_u_neg) == len(
z_v_neg) == len(batch_data['pos']), (len(z_u_pos),
len(z_v_pos),
len(z_u_neg),
len(z_v_neg),
len(batch_data['pos']))
# 2. ensemble (pos_img,pos_sent,neg_img,neg_sent) batch
# 2.1 for each pos_img ,find the best neg_sent,
posimg2negsentIdxs = {
} # pos idx :0 -> N-1, ====> neg idx :0 ->N-1
possent2negimgIdxs = {}
check_num = config.hn_num
neg_idxs = range(len(z_u_neg))
for i in xrange(len(batch_data['pos'])):
"""
pos_img_vec = z_v_pos[i] # [hop+1,d]
check_neg_idxs = random.sample(neg_idxs,check_num)
simis = np.zeros(check_num,dtype="float")
for j,check_neg_idx in enumerate(check_neg_idxs):
neg_sent_vec = z_u_neg[check_neg_idx]
simis[j] = np.sum(pos_img_vec*neg_sent_vec)
posimg2negsentIdxs[i] = check_neg_idxs[np.argmax(simis)]
pos_sent_vec = z_v_pos[i] # [hop+1,d]
check_neg_idxs = random.sample(neg_idxs,check_num)
simis = np.zeros(check_num,dtype="float")
for j,check_neg_idx in enumerate(check_neg_idxs):
neg_img_vec = z_v_neg[check_neg_idx]
simis[j] = np.sum(pos_sent_vec*neg_img_vec)
possent2negimgIdxs[i] = check_neg_idxs[np.argmax(simis)]
"""
pos_img_vec = z_v_pos[i] # [hop+1,d]
pos_sent_vec = z_v_pos[i] # [hop+1,d]
check_neg_idxs = random.sample(neg_idxs, check_num)
simis = np.zeros((2, check_num), dtype="float")
for j, check_neg_idx in enumerate(check_neg_idxs):
neg_sent_vec = z_u_neg[check_neg_idx]
neg_img_vec = z_v_neg[check_neg_idx]
simis[0, j] = np.sum(pos_img_vec * neg_sent_vec)
simis[1, j] = np.sum(pos_sent_vec * neg_img_vec)
posimg2negsentIdxs[i] = check_neg_idxs[np.argmax(simis[0])]
possent2negimgIdxs[i] = check_neg_idxs[np.argmax(simis[1])]
new_batch = {"data": []}
imgids = {}
for i in xrange(len(batch_data['pos'])):
thisPos = batch_data['pos'][i]
negImg = batch_data['neg'][possent2negimgIdxs[i]][0]
negSent = batch_data['neg'][posimg2negsentIdxs[i]]
thisNeg = (negImg, negSent[1], negSent[2])
new_batch['data'].append((thisPos, thisNeg))
# get all imageid
imgids[thisPos[0]] = 1
imgids[thisNeg[0]] = 1
# no need to get feature again
#imgid2idx = {}
#for imgid in imgids:
# imgid2idx[imgid] = len(imgid2idx.keys())
#image_feats = load_feats(imgid2idx,train_data.shared,config)
new_batch['imgid2idx'] = imgid2idx
new_batch['imgidx2feat'] = image_feats
batch = batchIdx, Dataset(new_batch,
"train",
shared=train_data.shared,
is_train=True,
imgfeat_dim=config.feat_dim)
loss, train_op = trainer.step(sess, batch)
#print mcb1
#print "-"*40
if global_step % save_period != 0: # time to save model
saver.save(sess, config.save_dir_model, global_step=global_step)
str_ = "best eval on val %s: %s at %s step, final step %s %s is %s" % (
metric, best[metric], best['step'], global_step, metric,
finalperf[metric])
print str_
self_summary_strs.append(str_)
if config.write_self_sum:
f = open(config.self_summary_path, "w")
f.writelines("%s" % ("\n".join(self_summary_strs)))
f.close()
if config.record_val_perf:
pickle.dump(val_perf, open(config.val_perf_path, "wb"))
analysis = Analysis()
analysis.get_perf(sup_data, range(iterations))
analysis.get_perf(sl_data, range(iterations))
analysis.plot(names=['Supervisor', 'Supervised Learning'], label='Reward', filename='./results/return_plots.eps')#, ylims=[0, 1600])
acc_a = Analysis()
acc_a.get_perf(acc, range(iterations))
acc_a.plot(names=['Supervised Learning Acc.'], label='Accuracy', filename='./results/acc_plots.eps')
"""
agent = Dagger(IT,useKMM = False)
exp = EpisodicExperiment(task, agent)
T = Tester(agent,exp)
prefix = 'svc-dagger-change'
dagger_data, _, acc = T.test(rounds = rounds, iterations = iterations, prefix = prefix)
np.save('./data/svc-dagger-change-dagger_data.npy', dagger_data)
np.save('./data/svc-dagger-change-acc.npy', acc)
# IPython.embed()
analysis = Analysis()
analysis.get_perf(dagger_data, range(iterations))
analysis.plot(names=['DAgger'], label='Reward', filename='./results/svc-dagger-change-return_plots.eps')
acc_a = Analysis()
acc_a.get_perf(acc, range(iterations))
acc_a.plot(names=['DAgger Acc.'], label='Accuracy', filename='./results/svc-dagger-change-acc_plots.eps')
def test(config):
if config.is_test_on_val:
test_data = read_data(config, 'val', True, subset=False)
print "total val samples:%s" % test_data.num_examples
else:
test_data = read_data(
config, 'test', True, subset=False
) # here will load shared.p from config.outpath (outbase/modelname/runId/)
print "total test samples:%s" % test_data.num_examples
config.imgfeat_dim = test_data.imgfeat_dim
# get the max_sent_size and other stuff
print "threshold setting--"
config_vars = vars(config)
print "\t" + " ,".join(
["%s:%s" % (key, config_vars[key]) for key in config.thresmeta])
# cap the numbers
update_config(config, [test_data], showMeta=True)
print "renewed ----"
print "\t" + " ,".join(
["%s:%s" % (key, config_vars[key]) for key in config.maxmeta])
model = get_model(config)
tfconfig = tf.ConfigProto(allow_soft_placement=True)
tfconfig.gpu_options.allow_growth = True # this way it will only allocate nessasary gpu, not take all
# or you can set hard limit
#tfconfig.gpu_options.per_process_gpu_memory_fraction = 0.4
with tf.Session(config=tfconfig) as sess:
initialize(load=True,
load_best=config.load_best,
model=model,
config=config,
sess=sess)
if config.is_pack_model:
saver = tf.train.Saver()
global_step = model.global_step
# put input and output to a universal name for reference when in deployment
# find the nessary stuff in model.get_feed_dict
# multiple input
tf.add_to_collection("sents", model.sents)
tf.add_to_collection("sents_c", model.sents_c)
tf.add_to_collection("sents_mask", model.sents_mask)
tf.add_to_collection("pis", model.pis)
# image and text feature
tf.add_to_collection("image_emb_mat", model.image_emb_mat)
tf.add_to_collection("existing_emb_mat", model.existing_emb_mat)
# for getting the highest ranked photo
#tf.add_to_collection("att_logits",model.att_logits)
tf.add_to_collection(
"is_train", model.is_train) # TODO, change this to a constant
tf.add_to_collection("output", model.s)
# also save all the model config and note into the model
pack_model_note = tf.get_variable(
"model_note",
shape=[],
dtype=tf.string,
initializer=tf.constant_initializer(config.pack_model_note),
trainable=False)
full_config = tf.get_variable("model_config",
shape=[],
dtype=tf.string,
initializer=tf.constant_initializer(
json.dumps(vars(config))),
trainable=False)
print "saving packed model"
# the following wont save the var model_note, model_config that's not in the graph,
# TODO: fix this
"""
# put into one big file to save
input_graph_def = tf.get_default_graph().as_graph_def()
#print [n.name for n in input_graph_def.node]
# We use a built-in TF helper to export variables to constants
output_graph_def = tf.graph_util.convert_variables_to_constants(
sess, # The session is used to retrieve the weights
input_graph_def, # The graph_def is used to retrieve the nodes
[tf.get_collection("output")[0].name.split(":")[0]] # The output node names are used to select the usefull nodes
)
output_graph = os.path.join(config.pack_model_path,"final.pb")
# Finally we serialize and dump the output graph to the filesystem
with tf.gfile.GFile(output_graph, "wb") as f:
f.write(output_graph_def.SerializeToString())
print("%d ops in the final graph." % len(output_graph_def.node))
"""
# save it into a path with multiple files
saver.save(sess,
os.path.join(config.pack_model_path, "final"),
global_step=global_step)
print "model saved in %s" % (config.pack_model_path)
return
if config.is_save_weights:
weight_dict = {}
weight_sum = open(os.path.join(config.weights_path, "all.txt"),
"w")
for var in tf.trainable_variables():
shape = var.get_shape()
weight_sum.writelines("%s %s\n" % (var.name, shape))
var_val = sess.run(var)
weight_dict[var.name] = var_val
np.savez(os.path.join(config.weights_path, "weights.npz"),
**weight_dict)
weight_sum.close()
last_time = time.time()
# num_epoch should be 1
num_steps = int(
math.ceil(test_data.num_examples /
float(config.batch_size))) * config.num_epochs
# load the graph and variables
tester = Tester(model, config, sess)
perf = evaluate(test_data, config, sess, tester)
print "performance:"
print perf
metric = ["r@1", "r@5", "r@10", "mr"]
metrics = []
for one in metric:
metrics.append("%s_%s" % ("t2i", one))
metrics.append("%s_%s" % ("i2t", one))
print "\t".join([m for m in metrics])
print "\t".join(["%.4f" % perf[m] for m in metrics])
if config.save_answers:
pickle.dump(imgid2sents, open("%s/answers_i2t.p" % config.val_path,
"w"))
pickle.dump(sentId2imgs, open("%s/answers_t2i.p" % config.val_path,
"w"))
if __name__ == "__main__":
arg_parser = ArgParser()
data_base = DataBase("./data")
NET_PATH="./cifar_net.pth"
if arg_parser.is_train():
net = Net()
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.9)
trainer = Trainer().\
set_optimizer(optimizer).\
set_num_epochs(2).\
set_train_loader(data_base.train_loader()).\
set_net(net).\
set_criterion(criterion)
trainer.train()
torch.save(net.state_dict(), NET_PATH)
if arg_parser.is_test():
net = Net()
net.load_state_dict(torch.load(NET_PATH))
tester = Tester().\
set_test_loader(data_base.test_loader()).\
set_net(net)
tester.test()
#!/usr/bin/env python3
from tester import Tester, get_input, is_python_3
import sys
import os
import codecs
import webbrowser
import glob
# test if files exist
Tester().test()
# import urlopen
if is_python_3:
from urllib.request import urlopen
from urllib.error import URLError
else:
from urllib2 import urlopen
##########################################################################
__version__ = "1.1.0"
__help_string__ = "eBooker v" + __version__ + " Help\n==============" + \
("=" * len(__version__)) + "\nhelp - show this help\nexit - quit the session\nabout - read about this tool\nedit - edit/create a file\nclear -\
clear the screen\ndebug - give you a list of commonly occurring issues\nserve - open your book in a web browser for reviewing\n"
__about_string__ = "eBooker is a command-line tool which lets you create ebooks and other text files from command line with ease. \
You don't have to be a programming expert or a nerd to use this. Anyone with a basic knowledge in computers can use this tool very easily.\n"
##########################################################################
def install_markdown():
trainer.train()
print("~~~~ Select best epoch on validation set ~~~~")
epochs2test = [
str(int(args.save_each * (i + 1)))
for i in range(args.ne // args.save_each)
]
dataset = Dataset(args.dataset)
best_mrr = -1.0
best_epoch = "0"
for epoch in epochs2test:
start = time.time()
print(epoch)
model_path = "models/" + args.dataset + "/" + epoch + ".chkpnt"
tester = Tester(dataset, model_path, "valid")
mrr = tester.test()
if mrr > best_mrr:
best_mrr = mrr
best_epoch = epoch
print(time.time() - start)
print("Best epoch: " + best_epoch)
print("~~~~ scoring on the best epoch ~~~~")
best_model_path = "models/" + args.dataset + "/" + best_epoch + ".chkpnt"
myscorer = Scorer(dataset, best_model_path, "test")
path = args.path
train_data = prepare_data(myscorer, path + "/train.txt")
def process_tester(self):
'''测试器执行子进程'''
tester = Tester()
while True:
tester.run()
time.sleep(TESTER_CYCLE)
def test(config):
if config.is_test_on_val:
test_data = read_data(config,'val',True)
print ("total val samples:%s"%test_data.num_examples)
else:
test_data = read_data(config,'test',True) # here will load shared.p from config.outpath (outbase/modelname/runId/)
print ("total test samples:%s"%test_data.num_examples)
# get the max_sent_size and other stuff
print ("threshold setting--")
config_vars = vars(config)
print ("\t"+ " ,".join(["%s:%s"%(key,config_vars[key]) for key in config.thresmeta]))
# cap the numbers
update_config(config,[test_data],showMeta=True)
print ("renewed ----")
print ("\t" + " ,".join(["%s:%s"%(key,config_vars[key]) for key in config.maxmeta]))
model = get_model(config)
# update each batch forward into this dict
id2predanswers = {}
id2realanswers = {}
id2yp = {}
tfconfig = tf.ConfigProto(allow_soft_placement=True)
tfconfig.gpu_options.allow_growth = True # this way it will only allocate nessasary gpu, not take all
# or you can set hard limit
#tfconfig.gpu_options.per_process_gpu_memory_fraction = 0.4
with tf.Session(config=tfconfig) as sess:
initialize(load=True,load_best=config.load_best,model=model,config=config,sess=sess)
if config.is_save_weights:
weight_dict = {}
weight_sum = open(os.path.join(config.weights_path,"all.txt"),"w")
for var in tf.trainable_variables():
shape = var.get_shape()
weight_sum.writelines("%s %s\n"%(var.name,shape))
var_val = sess.run(var)
weight_dict[var.name] = var_val
np.savez(os.path.join(config.weights_path,"weights.npz"),**weight_dict)
weight_sum.close()
last_time = time.time()
# num_epoch should be 1
num_steps = int(math.ceil(test_data.num_examples/float(config.batch_size)))*config.num_epochs
# load the graph and variables
tester = Tester(model,config,sess)
count=0
print ("total step:%s"%num_steps)
for batch in tqdm(test_data.get_batches(config.batch_size,num_steps=num_steps,shuffle=False),total=num_steps):
count+=1
yp = tester.step(sess,batch) # [N,4] # id2realanswersprob for each answer
if config.get_yp:
pred,gt,yp = getAnswers_yp(yp,batch)
id2yp.update(yp)
else:
pred,gt = getAnswers(yp,batch) # from here we get the qid:yindx,
id2predanswers.update(pred)
id2realanswers.update(gt)
acc = getEvalScore(id2predanswers,id2realanswers)
print ("done, got %s answers, accuracy:%s"%(len(id2predanswers),acc))
from tester import Tester
if __name__ == '__main__':
tester = Tester()
tester.testUImpatience()
allAUC = []
for curIndexes in theIndexes:
print('[TRAIN ' + str(curIndexes[0] + 1) + ' VAL ' +
str(curIndexes[1] + 1) + ' TEST ' + str(curIndexes[2] + 1) + ']')
trainGenerator = dataGenerators[curIndexes[0]]
valGenerator = dataGenerators[curIndexes[1]]
testGenerator = dataGenerators[curIndexes[2]]
print(' * TRAINING ')
theModel = ModelWrapper(inputShape=(imgSize[0], imgSize[1], 3),
outputSize=outputSize)
theModel.create()
theModel.train(trainGenerator, valGenerator, curIndexes[3])
fileName = savePrefixFName + 'T' + str(curIndexes[0] + 1) + '_V' + str(
curIndexes[1] + 1) + '_EPOCHS' + str(curIndexes[3])
print(' * SAVING MODEL ' + fileName)
theModel.save(fileName)
print(' * TESTING MODEL')
theTester = Tester(theModel, dataSets[curIndexes[2]])
stuff, denseAUC = theTester.compute_hitratio_evolution(useCNN=False)
stuff, cnnAUC = theTester.compute_hitratio_evolution(useCNN=True)
print(' + DENSE AUC: ' + str(denseAUC))
print(' + CNN AUC: ' + str(cnnAUC))
allAUC.append([denseAUC, cnnAUC])
print(' * SAVING TEST RESULTS')
with open(savePrefixFName + 'AUC.pkl', 'wb') as aucFile:
dump(allAUC, aucFile)
print('[DONE]')
train_step = M3sdaTrainStep(
build_generator_lambda=build_generator_lambda,
build_classifier_lambda=build_classifier_lambda,
domains=DOMAINS,
n_moments=2,
n_frozen_layers=0,
learning_rate=0.0001,
loss_weight=0.005,
)
trainer = Trainer(
train_step=train_step,
n_iterations=12000,
n_log_iterations=100,
n_save_iterations=1000,
n_validate_iterations=2,
log_path=LOG_PATH,
restore_model_flag=True,
restore_optimizer_flag=False
)
trainer(train_dataset, validate_dataset)
test_dataset = iter(make_domain_dataset(
paths=target_paths,
labels=target_labels,
preprocessor=Preprocessor(CONFIG),
batch_size=BATCH_SIZE
))
test_step = M3sdaTestStep(build_generator_lambda, build_classifier_lambda, domains=DOMAINS)
tester = Tester(test_step=test_step, log_path=LOG_PATH)
tester(test_dataset)
def train_and_eval(self):
# If the number of iterations is the same as the current iteration, exit.
if (self.model.cur_itr.data >= self.num_iterations):
print("*************")
print(
"Number of iterations is the same as that in the pretrained model."
)
print("Nothing left to train. Exiting.")
print("*************")
# return
print("Training the {} model...".format(self.model_name))
print("Number of training data points: {}".format(
len(self.dataset.data["train"])))
loss_layer = torch.nn.CrossEntropyLoss()
W = None
print("Starting training at iteration ... {}".format(
self.model.cur_itr.data))
for it in range(self.model.cur_itr.data, self.num_iterations + 1):
last_batch = False
self.model.train()
self.model.cur_itr.data += 1
losses = 0
while not last_batch:
r, e1, e2, e3, e4, e5, e6, targets, ms, bs = self.dataset.next_batch(
self.batch_size,
neg_ratio=self.neg_ratio,
device=self.device)
last_batch = self.dataset.was_last_batch()
self.opt.zero_grad()
number_of_positive = len(np.where(targets > 0)[0])
if (self.model_name == "HypE"):
predictions = self.model.forward(r, e1, e2, e3, e4, e5, e6,
ms, bs)
elif (self.model_name == "MTransH"):
predictions = self.model.forward(r, e1, e2, e3, e4, e5, e6,
ms)
elif (self.model_name == "RealEv3"):
predictions = self.model.forward(r, e1, e2, e3, e4, e5,
e6) + self.model.loss()
else:
predictions = self.model.forward(r, e1, e2, e3, e4, e5, e6)
predictions = self.padd_and_decompose(
targets, predictions, self.neg_ratio * self.max_arity)
targets = torch.zeros(number_of_positive).long().to(
self.device)
loss = loss_layer(predictions, targets)
loss.backward()
self.opt.step()
losses += loss.item()
print("Iteration#: {}, loss: {}".format(it, losses))
# Evaluate the model every 100th iteration or if it is the last iteration
if (it % 100 == 0) or (it == self.num_iterations):
self.model.eval()
with torch.no_grad():
print("validation:")
tester = Tester(self.dataset, self.model, "valid",
self.model_name)
measure_valid, _ = tester.test(True, False, False, False)
mrr = measure_valid.mrr["fil"]
# This is the best model we have so far if
# no "best_model" exists yes, or if this MRR is better than what we had before
is_best_model = (self.best_model is
None) or (mrr > self.best_model.best_mrr)
if is_best_model:
self.best_model = self.model
# Update the best_mrr value
self.best_model.best_mrr.data = torch.from_numpy(
np.array([mrr]))
self.best_model.best_itr.data = torch.from_numpy(
np.array([it]))
# Save the model at checkpoint
self.save_model(it, "valid", is_best_model=is_best_model)
#if self.best_model is None:
# self.best_model = self.model
self.best_model.eval()
with torch.no_grad():
print("testing best model at iteration {} .... ".format(
self.best_model.best_itr))
tester = Tester(self.dataset, self.best_model, "test",
self.model_name)
self.measure, self.measure_by_arity = tester.test(
True, self.test_by_arity, self.test_by_op, self.test_by_deg)
help='the path of the data directory')
parser.add_argument('--test-csv',
type=str,
default='',
help='the path of the test csv')
parser.add_argument('--weight-dir',
type=str,
default='',
help='the path of trained model')
parser.add_argument('--batch-size',
type=int,
default=256,
help='the batch size when testing')
parser.add_argument('--no-cuda',
action='store_true',
default=False,
help='disables CUDA training')
parser.add_argument('--reshape-size',
type=int,
default=224,
help='the size of the input image')
parser.add_argument('--classes',
type=int,
default=156,
help='the #classes of target')
args = parser.parse_args()
args.cuda = not args.no_cuda and torch.cuda.is_available()
tester = Tester(args=args)
tester.test()
