def train():
train_dir, validation_dir = get_train_val_dirs()
train_dataset = get_dataset(train_dir)
val_dataset = get_dataset(validation_dir)
class_names = np.array(sorted([item.name for item in train_dir.glob('*') if item.name != "LICENSE.txt"]))
util.class_names = class_names
train_dataset = train_dataset.map(process_path, num_parallel_calls=AUTOTUNE)
val_dataset = val_dataset.map(process_path, num_parallel_calls=AUTOTUNE)
train_dataset = configure_for_performance(train_dataset)
val_dataset = configure_for_performance(val_dataset)
print(train_dataset)
model = util.get_model()
base_learning_rate = 0.0001
model.compile(optimizer=tf.keras.optimizers.Adam(lr=base_learning_rate),
loss=tf.keras.losses.BinaryCrossentropy(from_logits=True),
metrics=['accuracy'])
initial_epochs = 10
history = model.fit(train_dataset,
epochs=initial_epochs,
validation_data=val_dataset)
model.save('./tlmodel')
tar = tarfile.open("tlmodel.tar.gz", "w:gz")
tar.add("./tlmodel", arcname="tlmodel")
tar.close()
shutil.move("tlmodel.tar.gz", model_artifacts_path)
def main(cross_validate=False):
start = time()
print("loading data...")
credits_X, credits_y = get_dataset(
["credits/mad_max.mov", "credits/under_the_skin.mov"], 1,
process_frame, 10000
)
print("credits done:", credits_X.shape, credits_y.shape)
content_X, content_y = get_dataset(
glob("./content/*.mov"), 0,
process_frame, 10000
)
print("content done:", content_X.shape, content_y.shape)
print("took", time() - start, "sec")
X = np.vstack((credits_X, content_X))
y = np.vstack((credits_y, content_y)).ravel()
model = SVC()
if cross_validate:
print("training model with cross-validation...")
crossvalidate(model, X, y)
else:
print("training model...")
train_and_dump_model(model, X, y)
print("DONE")
print("took", time() - start, "sec")
def main():
start = time()
print("loading data...")
credits_X, credits_y = get_dataset(
["credits/mad_max.mov", "credits/under_the_skin.mov"], 1,
process_frame, 20000)
print("credits done:", credits_X.shape, credits_y.shape)
content_X, content_y = get_dataset(glob("./content/*.mov"), 0,
process_frame, 20000)
print("content done:", content_X.shape, content_y.shape)
X = np.vstack((credits_X, content_X))
y = np.vstack((credits_y, content_y)).ravel()
print("took", time() - start, "sec")
model = build_model()
train_and_dump_model(model, X, y)
print("testing on unseen data...")
credits_X, credits_y = get_dataset(["credits/john_wick.mov"], 1,
process_frame)
print("data loaded:", credits_X.shape, credits_y.shape)
credits_y = np_utils.to_categorical(credits_y, 2)
print("score:", model.evaluate(credits_X, credits_y, batch_size=128))
def main():
args = parse_args()
with open(args.config_path) as f:
config = json.load(f)
command_config = {'gpu': args.gpu}
config.update(command_config)
device_id = config['gpu']
batch_size = config['batch_size']
output_path = args.output
network_params = config['network']
for k, v in network_params.items():
net = util.create_network(v)
model_path = '{}.{}.model'.format(args.model, k)
serializers.load_npz(model_path, net)
if device_id >= 0:
chainer.cuda.get_device_from_id(device_id).use()
for net in nets.values():
net.to_gpu()
datasets = util.get_dataset(config.get('dataset', {}))
test = datasets['test']
test_iter = chainer.iterators.SerialIterator(test, batch_size,
repeat=False, shuffle=False)
y = predict_dataset(net, test_iter, device=device_id)
np.save(output_path, y)
def gparam(config):
with open(config.word_emb_file, "r") as fh:
word_mat = np.array(json.load(fh), dtype=np.float32)
with open(config.char_emb_file, "r") as fh:
char_mat = np.array(json.load(fh), dtype=np.float32)
with open(config.test_eval_file, "r") as fh:
eval_file = json.load(fh)
with open(config.test_meta, "r") as fh:
meta = json.load(fh)
total = meta["total"]
print("Loading model...")
test_batch = get_dataset(config.test_record_file, get_record_parser(
config, is_test=True), config).make_one_shot_iterator()
model = Model(config, test_batch, word_mat, char_mat, trainable=False)
all_weights = {v.name: v for v in tf.trainable_variables()}
total_size = 0
for v_name in sorted(list(all_weights)):
v = all_weights[v_name]
print("%s\tshape %s", v.name[:-2].ljust(80),
str(v.shape).ljust(20))
v_size = np.prod(np.array(v.shape.as_list())).tolist()
total_size += v_size
print("Total trainable variables size: %d", total_size)
def run(args):
train, test = util.get_dataset(args.dataset)
colors = [vz.colors.all_one_lg, vz.colors.linear_lg]
names = ['all-one', 'harmonic', 'linear', 'half_one']
colors = [
vz.colors.all_one_lg, vz.colors.harmonic_lg, vz.colors.linear_lg,
vz.colors.half_one_lg
]
models = [
MLP.MLP(10, cg.uniform),
MLP.MLP(10, cg.harmonic),
MLP.MLP(10, cg.linear),
MLP.MLP(10, cg.uniform_exp),
]
comp_ratios = np.linspace(0.1, 1.0, 20)
acc_dict = {}
ratios_dict = {}
for name, model in zip(names, models):
util.load_or_train_model(model, train, test, args)
acc_dict[name] = util.sweep_idp(model, test, comp_ratios, args)
ratios_dict[name] = [100. * cr for cr in comp_ratios]
filename = "MLP_coef_comparison_{}".format(args.dataset)
vz.plot(ratios_dict,
acc_dict,
names,
filename,
colors=colors,
folder=args.figure_path,
ext=args.ext,
xlabel='IDP (%)',
ylabel='Classification Accuracy (%)',
title='MLP (MNIST)',
legend_loc='lower right',
ylim=(85, 100))
def run(args):
train, test = util.get_dataset(args.dataset)
names = ['all-one (standard)', 'linear']
colors = [vz.colors.all_one_lg, vz.colors.linear_lg]
models = [
BinaryNet.BinaryConvNet(10, cg.uniform, 'all'),
BinaryNet.BinaryConvNet(10, cg.linear, 'slow_exp'),
]
comp_ratios = np.linspace(0.1, 1.0, 20)
acc_dict = {}
ratios_dict = {}
for name, model in zip(names, models):
util.load_or_train_model(model, train, test, args)
acc_dict[name] = util.sweep_idp(model, test, comp_ratios, args)
ratios_dict[name] = [100. * cr for cr in comp_ratios]
filename = "BinaryNet_{}".format(args.dataset)
vz.plot(ratios_dict,
acc_dict,
names,
filename,
colors=colors,
folder=args.figure_path,
ext=args.ext,
title='BinaryNet (MNIST)',
xlabel='IDP (%)',
ylabel='Classification Accuracy (%)',
ylim=(90, 100))
def train(inputs, outputs, pre_train=False):
tf.keras.backend.clear_session()
dataset, VOCAB_SIZE, _ = get_dataset(inputs, outputs)
if pre_train:
model = tf.keras.models.load_model(config.MODEL_PATH)
else:
model = transformer(vocab_size=VOCAB_SIZE,
num_layers=config.NUM_LAYERS,
units=config.UNITS,
d_model=config.D_MODEL,
num_heads=config.NUM_HEADS,
dropout=config.DROPOUT)
learning_rate = model.CustomSchedule(config.D_MODEL)
optimizer = tf.keras.optimizers.Adam(learning_rate,
beta_1=0.9,
beta_2=0.98,
epsilon=1e-9)
model.compile(optimizer=optimizer,
loss=model.loss_function,
metrics=[model.accuracy])
model.fit(dataset, epochs=config.EPOCHS)
model.save(config.MODEL_PATH)
def run(args):
train, test = util.get_dataset(args.dataset)
names = ['linear']
colors = [vz.colors.linear_sm, vz.colors.linear_lg]
models = [VGG.VGGMulti(10, cg.linear, profiles=[(0, 2), (2, 10)])]
comp_ratios = np.linspace(0.1, 1, 20)
acc_dict = {}
ratios_dict = {}
key_names = []
for name, model in zip(names, models):
util.train_model_profiles(model, train, test, args)
for profile in range(len(model.profiles)):
key = name + '-' + str(profile + 1)
key_names.append(key)
acc_dict[key] = util.sweep_idp(model,
test,
comp_ratios,
args,
profile=profile)
ratios_dict[key] = [100. * cr for cr in comp_ratios]
filename = "VGG_{}_multi".format(args.dataset)
vz.plot(ratios_dict,
acc_dict,
key_names,
filename,
colors=colors,
folder=args.figure_path,
ext=args.ext,
title='VGG-16 (CIFAR-10)',
xlabel='IDP (%)',
ylabel='Classification Accuracy (%)')
def train_with_hook(model):
# Create datasets
ds_train, _, ds_test, _ = get_dataset(args.data_dir,
test_size=0.2,
need_valid=False)
model.train_and_evaluate(ds_train, ds_test)
def test(config):
with open(config.word_emb_file, "r") as fh:
word_mat = np.array(json.load(fh), dtype=np.float32)
with open(config.char_emb_file, "r") as fh:
char_mat = np.array(json.load(fh), dtype=np.float32)
with open(config.test_eval_file, "r") as fh:
eval_file = json.load(fh)
with open(config.test_meta, "r") as fh:
meta = json.load(fh)
total = meta["total"]
print("Loading model...")
test_batch = get_dataset(config.test_record_file,
get_record_parser(config, is_test=True),
config).make_one_shot_iterator()
model = Model(config, test_batch, word_mat, char_mat, trainable=False)
graph_handler = GraphHandler(config, model)
sess_config = tf.ConfigProto(allow_soft_placement=True)
sess_config.gpu_options.allow_growth = True
with tf.Session(config=sess_config) as sess:
sess.run(tf.global_variables_initializer())
#saver = tf.train.Saver()
graph_handler.initialize(sess)
#saver.restore(sess, tf.train.latest_checkpoint(config.save_dir))
sess.run(tf.assign(model.is_train, tf.constant(False, dtype=tf.bool)))
losses = []
answer_dict = {}
remapped_dict = {}
ensember_dict = {}
for step in tqdm(range(total // config.batch_size + 1)):
start_logits, stop_logits, qa_id, loss, yp1, yp2 = sess.run([
model.start_logits, model.stop_logits, model.qa_id, model.loss,
model.yp1, model.yp2
])
answer_dict_, remapped_dict_ = convert_tokens(
eval_file, qa_id.tolist(), yp1.tolist(), yp2.tolist())
answer_dict.update(answer_dict_)
remapped_dict.update(remapped_dict_)
losses.append(loss)
start_logits.tolist()
stop_logits.tolist()
for id, start, stop in zip(qa_id, start_logits, stop_logits):
ensember_dict[str(id)] = {'yp1': start, 'yp2': stop}
loss = np.mean(losses)
metrics = evaluate(eval_file, answer_dict)
answer_path = config.answer_file + "_" + str(config.load_step)
with open(answer_path, "w") as fh:
json.dump(remapped_dict, fh)
print("Exact Match: {}, F1: {}".format(metrics['exact_match'],
metrics['f1']))
ensember_dict['loss'] = loss
ensember_dict['exact_math'] = metrics['exact_match']
ensember_dict['f1'] = metrics['f1']
file_name = config.model_name + '_' + config.run_id + '.pklz'
save_path = os.path.join(config.result_path, file_name)
with gzip.open(save_path, 'wb', compresslevel=3) as fh:
pickle.dump(ensember_dict, fh)
def test(config):
with open(config.word_emb_file, "r") as fh:
word_mat = np.array(json.load(fh), dtype=np.float32)
with open(config.char_emb_file, "r") as fh:
char_mat = np.array(json.load(fh), dtype=np.float32)
with open(config.test_eval_file, "r") as fh:
eval_file = json.load(fh)
with open(config.test_meta, "r") as fh:
meta = json.load(fh)
total = meta["total"]
graph = tf.Graph()
print("Loading model...")
with graph.as_default() as g:
test_batch = get_dataset(config.test_record_file,
get_record_parser(config, is_test=True),
config).make_one_shot_iterator()
model = Model(config,
test_batch,
word_mat,
char_mat,
trainable=False,
graph=g)
sess_config = tf.ConfigProto(allow_soft_placement=True)
sess_config.gpu_options.allow_growth = True
with tf.Session(config=sess_config) as sess:
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver()
saver.restore(sess, tf.train.latest_checkpoint(config.save_dir))
if config.decay < 1.0:
sess.run(model.assign_vars)
losses = []
answer_dict = {}
remapped_dict = {}
for step in tqdm(range(total // config.batch_size + 1)):
qa_id, loss, yp1, yp2 = sess.run(
[model.qa_id, model.loss, model.yp1, model.yp2])
answer_dict_, remapped_dict_ = convert_tokens(
eval_file, qa_id.tolist(), yp1.tolist(), yp2.tolist())
answer_dict.update(answer_dict_)
remapped_dict.update(remapped_dict_)
losses.append(loss)
loss = np.mean(losses)
#with open(config.answer_file, "w") as fh:
# json.dump(remapped_dict, fh)
'''
metrics = evaluate(eval_file, answer_dict)
print("Exact Match: {}, F1: {}".format(
metrics['exact_match'], metrics['f1']))
'''
with open(config.answer_csv, 'w') as f:
print('dumping ans file to : %s' % str(config.answer_csv))
s = csv.writer(f, delimiter=',', lineterminator='\n')
for i in sorted(remapped_dict):
s.writerow([remapped_dict[i]])
def test(config, dataset="test"):
with open(config.word_emb_file, "r") as fh:
word_mat = np.array(json.load(fh), dtype=np.float32)
with open(config.char_emb_file, "r") as fh:
char_mat = np.array(json.load(fh), dtype=np.float32)
if dataset == "test":
test_eval_file = config.test_eval_file
test_meta = config.test_meta
test_record_file = config.test_record_file
elif dataset == "addsent":
print('HELLO')
test_eval_file = config.addsent_eval_file
test_meta = config.addsent_meta
test_record_file = config.addsent_record_file
elif dataset == "addonesent":
test_eval_file = config.addonesent_eval_file
test_meta = config.addonesent_meta
test_record_file = config.addonesent_record_file
with open(test_eval_file, "r") as fh:
eval_file = json.load(fh)
with open(test_meta, "r") as fh:
meta = json.load(fh)
total = meta["total"]
print("Loading model...")
test_batch = get_dataset(test_record_file,
get_record_parser(config, is_test=True),
config).make_one_shot_iterator()
model = Model(config, test_batch, word_mat, char_mat, trainable=False)
sess_config = tf.ConfigProto(allow_soft_placement=True)
sess_config.gpu_options.allow_growth = True
with tf.Session(config=sess_config) as sess:
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver()
saver.restore(sess, tf.train.latest_checkpoint(config.save_dir))
sess.run(tf.assign(model.is_train, tf.constant(False, dtype=tf.bool)))
losses = []
answer_dict = {}
remapped_dict = {}
for step in tqdm(range(total // config.batch_size + 1)):
qa_id, loss, yp1, yp2 = sess.run(
[model.qa_id, model.loss, model.yp1, model.yp2])
answer_dict_, remapped_dict_ = convert_tokens(
eval_file, qa_id.tolist(), yp1.tolist(), yp2.tolist())
answer_dict.update(answer_dict_)
remapped_dict.update(remapped_dict_)
losses.append(loss)
loss = np.mean(losses)
metrics = evaluate(eval_file, answer_dict)
with open(config.answer_file, "w") as fh:
json.dump(remapped_dict, fh)
print("Exact Match: {}, F1: {}".format(metrics['exact_match'],
metrics['f1']))
def test(config):
gpu_options = tf.GPUOptions(visible_device_list="2")
sess_config = tf.ConfigProto(allow_soft_placement=True,
gpu_options=gpu_options)
sess_config.gpu_options.allow_growth = True
with open(config.word_emb_file, "r") as fh:
word_mat = np.array(json.load(fh), dtype=np.float32)
with open(config.char_emb_file, "r") as fh:
char_mat = np.array(json.load(fh), dtype=np.float32)
with open(config.test_eval_file, "r") as fh:
eval_file = json.load(fh)
with open(config.test_meta, "r") as fh:
meta = json.load(fh)
total = meta["total"]
print("Loading model...")
test_batch = get_dataset(config.test_record_file,
get_record_parser(config, is_test=True),
config).make_one_shot_iterator()
model = Model(config, test_batch, word_mat, char_mat, trainable=False)
with tf.Session(config=sess_config) as sess:
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver()
saver.restore(sess, tf.train.latest_checkpoint(config.save_dir))
sess.run(tf.assign(model.is_train, tf.constant(False, dtype=tf.bool)))
losses = []
answer_dict = {}
remapped_dict = {}
# tqdm
for step in tqdm(range(total // config.batch_size + 1)):
qa_id, loss, yp1, yp2 = sess.run(
[model.qa_id, model.loss, model.yp1, model.yp2])
answer_dict_, remapped_dict_, outlier = convert_tokens(
eval_file, qa_id.tolist(), yp1.tolist(), yp2.tolist())
answer_dict.update(answer_dict_)
remapped_dict.update(remapped_dict_)
losses.append(loss)
print("\n", loss)
if (loss > 50):
for i, j, k in zip(qa_id.tolist(), yp1.tolist(), yp2.tolist()):
print(answer_dict[str(i)], j, k)
#print("IDs: {} Losses: {} Yp1: {} Yp2: {}".format(qa_id.tolist(),\
# loss.tolist(), yp1.tolist(), yp2.tolist()))
loss = np.mean(losses)
# evaluate with answer_dict, but in evaluate-v1.1.py, evaluate with remapped_dict
# since only that is saved. Both dict are a little bit different, check evaluate-v1.1.py
metrics = evaluate(eval_file, answer_dict)
with open(config.answer_file, "w") as fh:
json.dump(remapped_dict, fh)
print("Exact Match: {}, F1: {} Rouge-l-f: {} Rouge-l-p: {} Rouge-l-r: {}".format(\
metrics['exact_match'], metrics['f1'], metrics['rouge-l-f'], metrics['rouge-l-p'],\
metrics['rouge-l-r']))
def train_without_hook(model):
# Create datasets
ds_train, ds_valid, ds_test, ds_train_size = get_dataset(args.data_dir,
test_size=0.2,
valid_size=0.1)
model.train_and_evaluate_without_hook(ds_train, ds_valid, ds_test,
ds_train_size)
def predict(config):
prepro_predict(config)
with open(config.word_emb_file, "r") as fh:
word_mat = np.array(json.load(fh), dtype=np.float32)
with open(config.char_emb_file, "r") as fh:
char_mat = np.array(json.load(fh), dtype=np.float32)
with open(config.bpe_emb_file, "r") as fh:
bpe_mat = np.array(json.load(fh), dtype=np.float32)
with open(config.pos_emb_file, "r") as fh:
pos_mat = np.array(json.load(fh), dtype=np.float32)
with open(config.predict_eval_file, "r") as fh:
predict_eval_file = json.load(fh)
with open(config.predict_meta, "r") as fh:
meta = json.load(fh)
total = meta["total"]
print("Loading model...")
test_batch = get_dataset(config.predict_record_file,
get_record_parser(config, is_test=True),
config).make_one_shot_iterator()
model = Model(config,
test_batch,
word_mat,
char_mat,
bpe_mat,
pos_mat,
trainable=False)
sess_config = tf.ConfigProto(allow_soft_placement=True)
sess_config.gpu_options.allow_growth = True
with tf.Session(config=sess_config) as sess:
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver()
# TODO: add restoring from best model or from model name
saver.restore(sess, tf.train.latest_checkpoint(config.save_dir))
print('Restoring from: {}'.format(
tf.train.latest_checkpoint(config.save_dir)))
sess.run(tf.assign(model.is_train, tf.constant(False, dtype=tf.bool)))
answer_dict = {}
remapped_dict = {}
for step in tqdm(range(total // config.batch_size + 1)):
qa_id, loss, yp1, yp2 = sess.run(
[model.qa_id, model.loss, model.yp1, model.yp2])
answer_dict_, remapped_dict_ = convert_tokens(
predict_eval_file, qa_id.tolist(), yp1.tolist(), yp2.tolist())
answer_dict.update(answer_dict_)
remapped_dict.update(remapped_dict_)
path_to_save_answer = config.predict_file + '_ans'
with open(path_to_save_answer, "w") as fh:
json.dump(remapped_dict, fh)
print("Answer dumped: {}".format(path_to_save_answer))
def check_slc_status(slc_id):
result = util.get_dataset(slc_id)
total = result['hits']['total']
if total > 0:
return True
return False
def parse_command_line_args(args):
global gal, num_pixels, path_to_data, max_set_len, traindata, valdata
num_pixels = args['num_pixels']
path_to_data = args['path_to_data']
max_set_len = args['max_ccds']
gal = args['gal_type']
traindata, valdata = get_dataset(num_pixels=num_pixels,
max_set_len=max_set_len,
gal=gal,
path_to_data=path_to_data)
def test(config):
os.environ["CUDA_VISIBLE_DEVICES"] = config.choose_gpu
with open(config.word_emb_file, "r") as fh:
word_mat = np.array(json.load(fh), dtype=np.float32)
with open(config.char_emb_file, "r") as fh:
char_mat = np.array(json.load(fh), dtype=np.float32)
with open(config.test_eval_file, "r") as fh:
eval_file = json.load(fh)
with open(config.test_meta, "r") as fh:
meta = json.load(fh)
total = meta["total"]
graph = tf.Graph()
print("Loading model...")
with graph.as_default() as g:
test_batch = get_dataset(config.test_record_file,
get_record_parser(config, is_test=True),
config).make_one_shot_iterator()
model = QANet(config,
test_batch,
word_mat,
char_mat,
trainable=False,
graph=g)
sess_config = tf.ConfigProto(allow_soft_placement=True)
sess_config.gpu_options.allow_growth = True
sess_config.gpu_options.per_process_gpu_memory_fraction = config.gpu_memory_fraction
with tf.Session(config=sess_config) as sess:
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver()
saver.restore(sess, tf.train.latest_checkpoint(config.save_dir))
if config.decay < 1.0:
sess.run(model.assign_vars)
losses = []
answer_dict = {}
remapped_dict = {}
for step in tqdm(range(total // config.batch_size + 1)):
qa_id, loss, yp1, yp2 = sess.run(
[model.qa_id, model.loss, model.yp1, model.yp2])
answer_dict_, remapped_dict_ = convert_tokens(
eval_file, qa_id.tolist(), yp1.tolist(), yp2.tolist())
answer_dict.update(answer_dict_)
remapped_dict.update(remapped_dict_)
losses.append(loss)
loss = np.mean(losses)
metrics = evaluate(eval_file, answer_dict)
with open(config.answer_file, "w") as fh:
json.dump(remapped_dict, fh)
print("Exact Match: {}, F1: {}".format(metrics['exact_match'],
metrics['f1']))
def process(config):
print("Building model...")
parser = get_record_parser(config)
test_dataset = get_dataset(config.test_record_file, parser, config)
handle = tf.placeholder(tf.string, shape=[])
iterator = tf.data.Iterator.from_string_handle(handle,
test_dataset.output_types,
test_dataset.output_shapes)
test_iterator = test_dataset.make_one_shot_iterator()
model = xDeepFMModel(config, iterator)
sess_config = tf.ConfigProto(allow_soft_placement=True)
sess_config.gpu_options.allow_growth = True
loss_save = 100.0
save_f1 = 0.1
patience = 0
lr = config.init_lr
test_total = load_data_total(config.test_meta)
with tf.Session(config=sess_config) as sess:
param_num = sum(
[np.prod(sess.run(tf.shape(v))) for v in model.all_params])
print('There are {} parameters in the model'.format(param_num))
writer = tf.summary.FileWriter(config.event_dir)
sess.run(tf.global_variables_initializer())
summary_writer = tf.summary.FileWriter('./log/log', sess.graph)
saver = tf.train.Saver(max_to_keep=10000)
test_handle = sess.run(test_iterator.string_handle())
sess.run(tf.assign(model.lr, tf.constant(lr, dtype=tf.float32)))
## 加载训练好的模型
if os.path.exists(config.save_dir + "/checkpoint"):
print("Restoring Variables from Checkpoint.", config.save_dir)
saver.restore(sess, tf.train.latest_checkpoint(config.save_dir))
sess.run(tf.assign(model.is_train, tf.constant(False, dtype=tf.bool)))
test_metrics = evaluate_batch(model, test_total,
test_total // config.batch_size + 1,
sess, handle, test_handle)
print(test_metrics)
def main():
# TODO: Implements your evaluation
args = parse_args()
output_dir = args.output_dir
if not os.path.exists(output_dir):
os.makedirs(output_dir)
datasets = util.get_dataset()
test = datasets['test']
test_label = [t for x, t in test[:]]
test_label = np.asarray(test_label, dtype=np.int32)
y = np.load(args.prediction_path)
predict_label = np.argmax(y, axis=1)
skplt.metrics.plot_confusion_matrix(test_label,
np.argmax(y, axis=1),
normalize=True)
plt.savefig(os.path.join(output_dir, 'confusion_matrix.png'))
log_path = os.path.join(output_dir, 'log.txt')
with open(log_path, 'w') as f:
accuracy = float(np.sum(test_label == predict_label)) / len(test_label)
f.write('Accuracy: {}\n'.format(accuracy))
f.write('\nCorrect Images:\n')
image_dir = os.path.join(output_dir, 'correct_image')
if not os.path.exists(image_dir):
os.makedirs(image_dir)
for i in six.moves.range(10):
rank = sort_prediction(y, test_label == i, predict_label == i)
for j in rank[:10]:
scores = ','.join(map(str, y[j]))
f.write('{0},{1},{1},{2}\n'.format(j, i, scores))
file_name = '{0}_{1:05d}.png'.format(i, j)
x = (test[j][0] * 255).astype(np.uint8).reshape((28, 28))
Image.fromarray(x).save(os.path.join(image_dir, file_name))
f.write('\nIncorrect Images:\n')
image_dir = os.path.join(output_dir, 'incorrect_image')
if not os.path.exists(image_dir):
os.makedirs(image_dir)
for i in six.moves.range(10):
rank = sort_prediction(y, test_label == i, predict_label != i)
for j in rank[:10]:
label = np.argmax(y[j])
scores = ','.join(map(str, y[j]))
f.write('{0},{1},{2},{3}\n'.format(j, i, label, scores))
file_name = '{0}_{1}_{2:05d}.png'.format(i, label, j)
x = (test[j][0] * 255).astype(np.uint8).reshape((28, 28))
Image.fromarray(x).save(os.path.join(image_dir, file_name))
def test(config):
with open(config.word_emb_file, "r", encoding="utf-8") as fh:
word_mat = np.array(json.load(fh), dtype=np.float32)
with open(config.test_eval_file, "r", encoding="utf-8") as fh:
eval_file = json.load(fh)
with open(config.test_meta, "r", encoding="utf-8") as fh:
meta = json.load(fh)
total = meta["total"]
print("Loading model...")
test_batch = get_dataset(config.test_record_file,
get_record_parser(config, is_test=True),
config).make_one_shot_iterator()
model = Model(config, test_batch, word_mat, trainable=False)
sess_config = tf.ConfigProto(allow_soft_placement=True)
sess_config.gpu_options.allow_growth = True
with tf.Session(config=sess_config) as sess:
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver()
saver.restore(sess, tf.train.latest_checkpoint(config.save_dir))
sess.run(tf.assign(model.is_train, tf.constant(False, dtype=tf.bool)))
answer_dict = {}
remapped_dict = {}
for _ in tqdm(range(total // config.batch_size + 1)):
qa_id, loss, yp = sess.run([model.qa_id, model.loss, model.yp])
remapped_dict_, answer_dict_ = convert_tokens(
eval_file, qa_id.tolist(), yp.tolist())
answer_dict.update(answer_dict_)
remapped_dict.update(remapped_dict_)
f = open(config.answer_file, "w", encoding="utf-8")
for key in answer_dict:
f.write(str(key) + "\t" + answer_dict[key] + "\n")
# 处理不合法（被丢弃）的测试样本
# 直接选第一个答案
ans_list = list(answer_dict.keys())
with open(config.test_file, "r", encoding="utf-8") as fh:
for line in fh:
sample = json.loads(line)
if sample["query_id"] not in ans_list:
f.write(
str(sample["query_id"]) + "\t" +
sample['alternatives'].split("|")[0] + "\n")
f.close()
def train_and_dump_model(model, X, y):
X_train, X_test, y_train, y_test = train_test_split(X, y)
model.fit(X_train, y_train)
print("score:", model.score(X_test, y_test))
print("dumping model...")
dump(model, open("models/svm.p", "wb"))
print("testing on unseen data...")
credits_X, credits_y = get_dataset(
["credits/john_wick.mov"], 1,
process_frame
)
print("data loaded:", credits_X.shape, credits_y.shape)
print("score:", model.score(credits_X, credits_y))
def test(config):
with open(config.word_emb_file, "r") as fh:
word_mat = np.array(json.load(fh), dtype=np.float32)
with open(config.char_emb_file, "r") as fh:
char_mat = np.array(json.load(fh), dtype=np.float32)
with open(config.test_eval_file, "r") as fh:
eval_file = json.load(fh)
with open(config.test_meta, "r") as fh:
meta = json.load(fh)
total = meta["total"]
graph = tf.Graph()
print("Loading model...")
with graph.as_default() as g:
test_batch = get_dataset(config.test_record_file, get_record_parser(
config, is_test=True), config).make_one_shot_iterator()
model = Model(config, test_batch, word_mat, char_mat, trainable=False, graph = g)
sess_config = tf.ConfigProto(allow_soft_placement=True)
sess_config.gpu_options.allow_growth = True
with tf.Session(config=sess_config) as sess:
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver()
saver.restore(sess, tf.train.latest_checkpoint(config.save_dir))
if config.decay < 1.0:
sess.run(model.assign_vars)
losses = []
answer_dict = {}
remapped_dict = {}
for step in tqdm(range(total // config.batch_size + 1)):
qa_id, loss, yp1, yp2 = sess.run(
[model.qa_id, model.loss, model.yp1, model.yp2])
answer_dict_, remapped_dict_ = convert_tokens(
eval_file, qa_id.tolist(), yp1.tolist(), yp2.tolist())
answer_dict.update(answer_dict_)
remapped_dict.update(remapped_dict_)
losses.append(loss)
loss = np.mean(losses)
metrics = evaluate(eval_file, answer_dict)
with open(config.answer_file, "w") as fh:
json.dump(remapped_dict, fh)
print("Exact Match: {}, F1: {}".format(
metrics['exact_match'], metrics['f1']))
def train_xgb():
train_df, test_df = get_dataset('lda')
train_y = train_df.label
test_y = test_df.label
train_df.drop(['label', 'id', 'tweet', 'verified', 'Unnamed: 0', 'tweet'], axis = 1, inplace = True)
test_df.drop(['label', 'id', 'tweet', 'verified', 'Unnamed: 0'], axis = 1, inplace = True)
print(list(train_df))
model = XGBClassifier()
model.fit(train_df, train_y)
predict = model.predict(test_df)
for name, imp in zip(list(test_df), model.feature_importances_):
print(name+":"+str(imp))
#print(model.feature_importances_)
## print
print(classification_report(test_y, predict))
def test(config):
with codecs.open(config.word_emb_file, "r", encoding="utf-8") as fh:
word_mat = np.array(json.load(fh), dtype=np.float32)
with codecs.open(config.test_eval_file, "r", encoding="utf-8") as fh:
eval_file = json.load(fh)
with codecs.open(config.test_meta, "r", encoding="utf-8") as fh:
meta = json.load(fh)
# with open(config.char_emb_file, "r") as fh:
# char_mat = np.array(json.load(fh), dtype=np.float32)
total = meta["total"]
print("Loading model...")
test_batch = get_dataset(config.test_record_file,
get_record_parser(config, is_test=True),
config).make_one_shot_iterator()
# model = Model(config, test_batch, word_mat,char_mat, trainable=False)
model = Model(config, test_batch, word_mat, False)
sess_config = tf.ConfigProto(allow_soft_placement=True)
sess_config.gpu_options.allow_growth = True
with tf.Session(config=sess_config) as sess:
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver()
saver.restore(sess, tf.train.latest_checkpoint(config.save_dir))
sess.run(tf.assign(model.is_train, tf.constant(False, dtype=tf.bool)))
answer_dict = {}
remapped_dict = {}
for step in tqdm(range(total // config.batch_size + 1)):
qa_id, loss, yp = sess.run([model.qa_id, model.loss, model.yp])
remapped_dict_, answer_dict_ = convert_tokens(
eval_file, qa_id.tolist(), yp.tolist())
answer_dict.update(answer_dict_)
remapped_dict.update(remapped_dict_)
with codecs.open(config.answer_file, "w", encoding="utf-8") as fh:
for key in answer_dict:
fh.write(str(key) + "\t" + answer_dict[key] + "\n")
print("test")
def run(args):
train, test = util.get_dataset(args.dataset)
# names = ['all-ones,exp', 'all-ones,all', 'linear,exp', 'linear,all']
names = ['linear']
colors = [vz.colors.linear_sm, vz.colors.linear_lg]
models = [
MLP.MLP(10, cg.linear, [(0, 3), (3, 10)], n_units=100),
]
comp_ratios = np.linspace(0.1, 1, 20)
acc_dict = {}
ratios_dict = {}
key_names = []
for name, model in zip(names, models):
util.train_model_profiles(model, train, test, args)
for profile in range(len(model.profiles)):
key = name + '-' + str(profile + 1)
key_names.append(key)
acc_dict[key] = util.sweep_idp(model,
test,
comp_ratios,
args,
profile=profile)
ratios_dict[key] = [100. * cr for cr in comp_ratios]
filename = "MLP_{}_multi".format(args.dataset)
vz.plot(ratios_dict,
acc_dict,
key_names,
filename,
colors=colors,
folder=args.figure_path,
ext=args.ext,
xlabel='IDP (%)',
ylabel='Classification Accuracy (%)',
title='MLP (MNIST)',
ylim=(85, 100))
def train(config):
with open(config.word_emb_file, "r") as fh:
word_mat = np.array(json.load(fh), dtype=np.float32)
with open(config.char_emb_file, "r") as fh:
char_mat = np.array(json.load(fh), dtype=np.float32)
with open(config.train_eval_file, "r") as fh:
train_eval_file = json.load(fh)
with open(config.dev_eval_file, "r") as fh:
dev_eval_file = json.load(fh)
with open(config.dev_meta, "r") as fh:
meta = json.load(fh)
# 验证集总数据量
dev_total = meta["total"]
print("Building model...")
parser = get_record_parser(config)
graph = tf.Graph()
with graph.as_default() as g:
# tf.data.TFRecordDataset类型
train_dataset = get_batch_dataset(config.train_record_file, parser,
config)
dev_dataset = get_dataset(config.dev_record_file, parser, config)
# 单次迭代器
train_iterator = train_dataset.make_one_shot_iterator()
dev_iterator = dev_dataset.make_one_shot_iterator()
# 迭代器句柄占位符
handle = tf.placeholder(tf.string, shape=[])
# 这个迭代器会根据handle不同而使用handle对应的迭代器
iterator = tf.data.Iterator.from_string_handle(
handle, train_dataset.output_types, train_dataset.output_shapes)
model = Model(config, iterator, word_mat, char_mat, graph=g)
# 可动态申请显存
sess_config = tf.ConfigProto(allow_soft_placement=True)
sess_config.gpu_options.allow_growth = True
loss_save = 100.0
patience = 0
best_f1 = 0.
best_em = 0.
with tf.Session(config=sess_config) as sess:
# 指定一个文件来保存图，可以调用add_summary()将训练过成数据保存在FileWriter指定的文件中
writer = tf.summary.FileWriter(config.log_dir)
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver()
# 各个数据集创建对应的handle
train_handle = sess.run(train_iterator.string_handle())
dev_handle = sess.run(dev_iterator.string_handle())
if os.path.exists(os.path.join(config.save_dir, "checkpoint")):
saver.restore(sess,
tf.train.latest_checkpoint(config.save_dir))
global_step = max(sess.run(model.global_step), 1)
for _ in tqdm(range(global_step, config.num_steps + 1)):
global_step = sess.run(model.global_step) + 1
loss, train_op = sess.run([model.loss, model.train_op],
feed_dict={
handle: train_handle,
model.dropout: config.dropout
})
# 每config.period轮记录一次损失batch_loss
if global_step % config.period == 0:
loss_sum = tf.Summary(value=[
tf.Summary.Value(tag="model/loss", simple_value=loss),
])
# 向logs写入模型误差
writer.add_summary(loss_sum, global_step)
# 每config.checkpoint轮评估和保存一次模型
if global_step % config.checkpoint == 0:
# 训练集效果
_, summ = evaluate_batch(model, config.val_num_batches,
train_eval_file, sess, "train",
handle, train_handle)
for s in summ:
# 写入训练效果
writer.add_summary(s, global_step)
# 验证集效果
metrics, summ = evaluate_batch(
model, dev_total // config.batch_size + 1,
dev_eval_file, sess, "dev", handle, dev_handle)
for s in summ:
# 写入验证效果
writer.add_summary(s, global_step)
dev_f1 = metrics["f1"]
dev_em = metrics["exact_match"]
# 如果大于early_stop轮没提升就停止训练
if dev_f1 < best_f1 and dev_em < best_em:
patience += 1
if patience > config.early_stop:
break
else:
patience = 0
best_em = max(best_em, dev_em)
best_f1 = max(best_f1, dev_f1)
# 保存logs
writer.flush()
# 保存模型
filename = os.path.join(
config.save_dir, "model_{}.ckpt".format(global_step))
saver.save(sess, filename)
def run_test(
dataset,
clf_type,
epochs,
true_rh1,
downsample_ratio,
ordered_models_keys,
list_of_images=range(10),
suppress_error=False,
verbose=False,
pi1=0.0,
one_vs_rest=True,
cv_n_folds=3,
early_stopping=True,
pulearning=None,
):
# Cast types to ensure consistency for 1 and 1.0, 0 and 0.0
true_rh1 = float(true_rh1)
downsample_ratio = float(downsample_ratio)
pi1 = float(pi1)
# Load MNIST or CIFAR data
(X_train_original,
y_train_original), (X_test_original,
y_test_original) = get_dataset(dataset=dataset)
X_train_original, y_train_original = downsample(X_train_original,
y_train_original,
downsample_ratio)
# Initialize models and result storage
metrics = {key: [] for key in ordered_models_keys}
data_all = {"metrics": metrics, "calculated": {}, "errors": {}}
start_time = dt.now()
# Run through the ten images class of 0, 1, ..., 9
for image in list_of_images:
if one_vs_rest:
# X_train and X_test will not be modified. All data will be used. Adjust pointers.
X_train = X_train_original
X_test = X_test_original
# Relabel the image data. Make label 1 only for given image.
y_train = np.array(y_train_original == image, dtype=int)
y_test = np.array(y_test_original == image, dtype=int)
else: # one_vs_other
# Reducing the dataset to just contain our image and image = 4
other_image = 4 if image != 4 else 7
X_train = X_train_original[(y_train_original == image) |
(y_train_original == other_image)]
y_train = y_train_original[(y_train_original == image) |
(y_train_original == other_image)]
X_test = X_test_original[(y_test_original == image) |
(y_test_original == other_image)]
y_test = y_test_original[(y_test_original == image) |
(y_test_original == other_image)]
# Relabel the data. Make label 1 only for given image.
y_train = np.array(y_train == image, dtype=int)
y_test = np.array(y_test == image, dtype=int)
print()
print("Evaluating image:", image)
print("Number of positives in y:", sum(y_train))
print()
sys.stdout.flush()
s = y_train * (np.cumsum(y_train) < (1 - true_rh1) * sum(y_train))
# In the presence of mislabeled negative (negative incorrectly labeled positive):
# pi1 is the fraction of mislabeled negative in the labeled set:
num_mislabeled = int(sum(y_train) * (1 - true_rh1) * pi1 / (1 - pi1))
if num_mislabeled > 0:
negative_set = s[y_train == 0]
mislabeled = np.random.choice(len(negative_set),
num_mislabeled,
replace=False)
negative_set[mislabeled] = 1
s[y_train == 0] = negative_set
print("image = {0}".format(image))
print(
"Training set: total = {0}, positives = {1}, negatives = {2}, P_noisy = {3}, N_noisy = {4}"
.format(len(X_train), sum(y_train),
len(y_train) - sum(y_train), sum(s),
len(s) - sum(s)))
print("Testing set: total = {0}, positives = {1}, negatives = {2}".
format(len(X_test), sum(y_test),
len(y_test) - sum(y_test)))
# Fit different models for PU learning
for key in ordered_models_keys:
fit_start_time = dt.now()
print("\n\nFitting {0} classifier. Default classifier is {1}.".
format(key, clf_type))
if clf_type == "logreg":
clf = LogisticRegression()
elif clf_type == "cnn":
from classifier_cnn import CNN
from keras import backend as K
K.clear_session()
clf = CNN(
dataset_name=dataset,
num_category=2,
epochs=epochs,
early_stopping=early_stopping,
verbose=1,
)
else:
raise ValueError(
"clf_type must be either logreg or cnn for this testing file."
)
ps1 = sum(s) / float(len(s))
py1 = sum(y_train) / float(len(y_train))
true_rh0 = pi1 * ps1 / float(1 - py1)
model = get_model(
key=key,
rh1=true_rh1,
rh0=true_rh0,
clf=clf,
)
try:
if key == "True Classifier":
model.fit(X_train, y_train)
elif key in [
"Rank Pruning", "Rank Pruning (noise rates given)",
"Liu16 (noise rates given)"
]:
model.fit(X_train,
s,
pulearning=pulearning,
cv_n_folds=cv_n_folds)
elif key in ["Nat13 (noise rates given)"]:
model.fit(X_train, s, pulearning=pulearning)
else: # Elk08, Baseline
model.fit(X_train, s)
pred = model.predict(X_test)
# Produces only P(y=1|x) for pulearning models because they are binary
pred_prob = model.predict_proba(X_test)
pred_prob = pred_prob[:,
1] if key == "True Classifier" else pred_prob
# Compute metrics
metrics_dict = get_metrics(pred, pred_prob, y_test)
elapsed = (dt.now() - fit_start_time).total_seconds()
if verbose:
print(
"\n{0} Model Performance at image {1}:\n=================\n"
.format(key, image))
print("Time Required", elapsed)
print("AUC:", metrics_dict["AUC"])
print("Error:", metrics_dict["Error"])
print("Precision:", metrics_dict["Precision"])
print("Recall:", metrics_dict["Recall"])
print("F1 score:", metrics_dict["F1 score"])
print("rh1:", model.rh1 if hasattr(model, 'rh1') else None)
print("rh0:", model.rh0 if hasattr(model, 'rh0') else None)
print()
metrics_dict["image"] = image
metrics_dict["time_seconds"] = elapsed
metrics_dict["rh1"] = model.rh1 if hasattr(model,
'rh1') else None
metrics_dict["rh0"] = model.rh0 if hasattr(model,
'rh0') else None
# Append dictionary of error and loss metrics
if key not in data_all["metrics"]:
data_all["metrics"][key] = [metrics_dict]
else:
data_all["metrics"][key].append(metrics_dict)
data_all["calculated"][(key, image)] = True
except Exception as e:
msg = "Error in {0}, image {1}, rh1 {2}, m {3}: {4}\n".format(
key, image, true_rh1, pi1, e)
print(msg)
make_sure_path_exists("failed_models/")
with open("failed_models/" + key + ".txt", "ab") as f:
f.write(msg)
if suppress_error:
continue
else:
raise
return data_all
def train(config):
with open(config.word_emb_file, "r") as fh:
word_mat = np.array(json.load(fh), dtype=np.float32)
with open(config.train_eval_file, "r") as fh:
train_eval_file = json.load(fh)
with open(config.dev_eval_file, "r") as fh:
dev_eval_file = json.load(fh)
with open(config.dev_meta, "r") as fh:
meta = json.load(fh)
dev_total = meta["total"]
print("Building model...")
parser = get_record_parser(config)
graph = tf.Graph()
with graph.as_default() as g:
train_dataset = get_batch_dataset(config.train_record_file, parser,
config)
dev_dataset = get_dataset(config.dev_record_file, parser, config)
handle = tf.placeholder(tf.string, shape=[])
iterator = tf.data.Iterator.from_string_handle(
handle, train_dataset.output_types, train_dataset.output_shapes)
train_iterator = train_dataset.make_one_shot_iterator()
dev_iterator = dev_dataset.make_one_shot_iterator()
model = Model(config, iterator, word_mat, graph=g)
sess_config = tf.ConfigProto(allow_soft_placement=True)
sess_config.gpu_options.allow_growth = True
loss_save = 100.0
patience = 0
best_f1 = 0.
best_em = 0.
with tf.Session(config=sess_config) as sess:
writer = tf.summary.FileWriter(config.log_dir)
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver()
train_handle = sess.run(train_iterator.string_handle())
dev_handle = sess.run(dev_iterator.string_handle())
if os.path.exists(os.path.join(config.save_dir, "checkpoint")):
saver.restore(sess,
tf.train.latest_checkpoint(config.save_dir))
global_step = max(sess.run(model.global_step), 1)
for _ in (range(global_step, config.num_steps + 1)):
global_step = sess.run(model.global_step) + 1
loss, train_op = sess.run([model.loss, model.train_op],
feed_dict={
handle: train_handle,
model.dropout: config.dropout
})
if global_step % config.period == 0:
loss_sum = tf.Summary(value=[
tf.Summary.Value(tag="model/loss", simple_value=loss),
])
print('Step\'{}s loss: {}'.format(global_step, loss))
writer.add_summary(loss_sum, global_step)
if global_step % config.checkpoint == 0:
metrics, summ = evaluate_batch(model,
config.val_num_batches,
train_eval_file, sess,
"train", handle,
train_handle)
for s in summ:
writer.add_summary(s, global_step)
print('train em: ', metrics["exact_match"])
print('train f1: ', metrics["f1"])
metrics, summ = evaluate_batch(
model, dev_total // config.batch_size + 1,
dev_eval_file, sess, "dev", handle, dev_handle)
dev_f1 = metrics["f1"]
dev_em = metrics["exact_match"]
print('dev em: ', metrics["exact_match"])
print('dev f1: ', metrics["f1"])
if dev_f1 < best_f1 and dev_em < best_em:
patience += 1
if patience > config.early_stop:
break
else:
patience = 0
best_em = max(best_em, dev_em)
best_f1 = max(best_f1, dev_f1)
for s in summ:
writer.add_summary(s, global_step)
writer.flush()
filename = os.path.join(
config.save_dir, "model_{}.ckpt".format(global_step))
saver.save(sess, filename)
def train(config):
with open(config.word_emb_file, 'r') as fh:
word_mat = np.array(json.load(fh),
dtype=np.float32) # word embedding matrix
with open(config.char_emb_file, 'r') as fh:
char_mat = np.array(json.load(fh),
dtype=np.float32) # char embedding matrix
# total examples number in valid file
with open(config.dev_meta, 'r') as fh:
dev_meta = json.load(fh)
dev_total = dev_meta['total']
print('Building model...')
parser = get_record_parser(config)
graph = tf.Graph()
with graph.as_default() as g:
train_dataset = get_batch_dataset(config.train_record_file, parser,
config)
dev_dataset = get_dataset(config.dev_record_file, parser, config)
handle = tf.placeholder(tf.string, shape=[])
iterator = tf.data.Iterator.from_string_handle(
handle, train_dataset.output_types, train_dataset.output_shapes)
train_iterator = train_dataset.make_one_shot_iterator()
dev_iterator = dev_dataset.make_one_shot_iterator()
model = Model(config, iterator, word_mat, char_mat, graph=g)
# model = QANet4CBT(config, iterator, word_mat, char_mat, graph=g)
sess_config = tf.ConfigProto(allow_soft_placement=True)
sess_config.gpu_options.allow_growth = True
loss_save = 10.0
patience = 0
best_acc = 0.
with tf.Session(config=sess_config) as sess:
writer = tf.summary.FileWriter(config.log_dir)
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver()
train_handle = sess.run(train_iterator.string_handle())
dev_handle = sess.run(dev_iterator.string_handle())
if os.path.exists(os.path.join(config.save_dir, 'checkpoint')):
saver.restore(sess,
tf.train.latest_checkpoint(config.save_dir))
global_step = max(sess.run(model.global_step), 1)
total_corrects = 0
total_loss = 0.0
# Training
for _ in tqdm(range(global_step, config.num_steps + 1)):
global_step = sess.run(model.global_step) + 1
loss_in_batch, corrects_in_batch, train_op = sess.run(
[model.loss, model.correct_prediction, model.train_op],
feed_dict={
handle: train_handle,
model.dropout: config.dropout
})
total_corrects += corrects_in_batch
total_loss += loss_in_batch * config.batch_size
if global_step % config.period == 0:
acc = total_corrects / (global_step * config.batch_size)
loss = total_loss / (global_step * config.batch_size)
loss_sum = tf.Summary(value=[
tf.Summary.Value(tag='model/loss', simple_value=loss),
])
writer.add_summary(loss_sum, global_step)
acc_sum = tf.Summary(value=[
tf.Summary.Value(tag='model/acc', simple_value=acc),
])
writer.add_summary(acc_sum, global_step)
# Validation and save model
if global_step % config.checkpoint == 0:
val_acc, val_loss, v_acc_sum, v_loss_sum = validate(
config, model, sess, dev_total, 'dev', handle,
dev_handle)
writer.add_summary(v_acc_sum, global_step)
writer.add_summary(v_loss_sum, global_step)
# Early Stopping
if val_acc < best_acc:
patience += 1
if patience > config.early_stop:
break
else:
patience = 0
best_acc = max(best_acc, val_acc)
# Save Model, keep top 5 best models.
filename = os.path.join(
config.save_dir, 'model_{}_val-acc_{}.ckpt'.format(
global_step, best_acc))
saver.save(sess, filename)
writer.flush()
def train(config):
with open(config.word_emb_file, "r") as fh:
word_mat = np.array(json.load(fh), dtype=np.float32)
with open(config.char_emb_file, "r") as fh:
char_mat = np.array(json.load(fh), dtype=np.float32)
with open(config.train_eval_file, "r") as fh:
train_eval_file = json.load(fh)
with open(config.dev_eval_file, "r") as fh:
dev_eval_file = json.load(fh)
with open(config.dev_meta, "r") as fh:
meta = json.load(fh)
dev_total = meta["total"]
print("Building model...")
parser = get_record_parser(config)
train_dataset = get_batch_dataset(config.train_record_file, parser, config)
dev_dataset = get_dataset(config.dev_record_file, parser, config)
handle = tf.placeholder(tf.string, shape=[])
iterator = tf.data.Iterator.from_string_handle(
handle, train_dataset.output_types, train_dataset.output_shapes)
train_iterator = train_dataset.make_one_shot_iterator()
dev_iterator = dev_dataset.make_one_shot_iterator()
model = Model(config, iterator, word_mat, char_mat)
sess_config = tf.ConfigProto(allow_soft_placement=True)
sess_config.gpu_options.allow_growth = True
loss_save = 100.0
patience = 0
lr = config.init_lr
with tf.Session(config=sess_config) as sess:
writer = tf.summary.FileWriter(config.log_dir)
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver()
train_handle = sess.run(train_iterator.string_handle())
dev_handle = sess.run(dev_iterator.string_handle())
sess.run(tf.assign(model.is_train, tf.constant(True, dtype=tf.bool)))
sess.run(tf.assign(model.lr, tf.constant(lr, dtype=tf.float32)))
for _ in tqdm(range(1, config.num_steps + 1)):
global_step = sess.run(model.global_step) + 1
loss, train_op = sess.run([model.loss, model.train_op], feed_dict={
handle: train_handle})
if global_step % config.period == 0:
loss_sum = tf.Summary(value=[tf.Summary.Value(
tag="model/loss", simple_value=loss), ])
writer.add_summary(loss_sum, global_step)
if global_step % config.checkpoint == 0:
sess.run(tf.assign(model.is_train,
tf.constant(False, dtype=tf.bool)))
_, summ = evaluate_batch(
model, config.val_num_batches, train_eval_file, sess, "train", handle, train_handle)
for s in summ:
writer.add_summary(s, global_step)
metrics, summ = evaluate_batch(
model, dev_total // config.batch_size + 1, dev_eval_file, sess, "dev", handle, dev_handle)
sess.run(tf.assign(model.is_train,
tf.constant(True, dtype=tf.bool)))
dev_loss = metrics["loss"]
if dev_loss < loss_save:
loss_save = dev_loss
patience = 0
else:
patience += 1
if patience >= config.patience:
lr /= 2.0
loss_save = dev_loss
patience = 0
sess.run(tf.assign(model.lr, tf.constant(lr, dtype=tf.float32)))
for s in summ:
writer.add_summary(s, global_step)
writer.flush()
filename = os.path.join(
config.save_dir, "model_{}.ckpt".format(global_step))
saver.save(sess, filename)
def train(config):
with open(config.word_emb_file, "r") as fh:
word_mat = np.array(json.load(fh), dtype=np.float32)
with open(config.char_emb_file, "r") as fh:
char_mat = np.array(json.load(fh), dtype=np.float32)
with open(config.train_eval_file, "r") as fh:
train_eval_file = json.load(fh)
with open(config.dev_eval_file, "r") as fh:
dev_eval_file = json.load(fh)
with open(config.dev_meta, "r") as fh:
meta = json.load(fh)
dev_total = meta["total"]
print("Building model...")
parser = get_record_parser(config)
graph = tf.Graph()
with graph.as_default() as g:
train_dataset = get_batch_dataset(config.train_record_file, parser, config)
dev_dataset = get_dataset(config.dev_record_file, parser, config)
handle = tf.placeholder(tf.string, shape=[])
iterator = tf.data.Iterator.from_string_handle(
handle, train_dataset.output_types, train_dataset.output_shapes)
train_iterator = train_dataset.make_one_shot_iterator()
dev_iterator = dev_dataset.make_one_shot_iterator()
model = Model(config, iterator, word_mat, char_mat, graph = g)
sess_config = tf.ConfigProto(allow_soft_placement=True)
sess_config.gpu_options.allow_growth = True
loss_save = 100.0
patience = 0
best_f1 = 0.
best_em = 0.
with tf.Session(config=sess_config) as sess:
writer = tf.summary.FileWriter(config.log_dir)
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver()
train_handle = sess.run(train_iterator.string_handle())
dev_handle = sess.run(dev_iterator.string_handle())
if os.path.exists(os.path.join(config.save_dir, "checkpoint")):
saver.restore(sess, tf.train.latest_checkpoint(config.save_dir))
global_step = max(sess.run(model.global_step), 1)
for _ in tqdm(range(global_step, config.num_steps + 1)):
global_step = sess.run(model.global_step) + 1
loss, train_op = sess.run([model.loss, model.train_op], feed_dict={
handle: train_handle, model.dropout: config.dropout})
if global_step % config.period == 0:
loss_sum = tf.Summary(value=[tf.Summary.Value(
tag="model/loss", simple_value=loss), ])
writer.add_summary(loss_sum, global_step)
if global_step % config.checkpoint == 0:
_, summ = evaluate_batch(
model, config.val_num_batches, train_eval_file, sess, "train", handle, train_handle)
for s in summ:
writer.add_summary(s, global_step)
metrics, summ = evaluate_batch(
model, dev_total // config.batch_size + 1, dev_eval_file, sess, "dev", handle, dev_handle)
dev_f1 = metrics["f1"]
dev_em = metrics["exact_match"]
if dev_f1 < best_f1 and dev_em < best_em:
patience += 1
if patience > config.early_stop:
break
else:
patience = 0
best_em = max(best_em, dev_em)
best_f1 = max(best_f1, dev_f1)
for s in summ:
writer.add_summary(s, global_step)
writer.flush()
filename = os.path.join(
config.save_dir, "model_{}.ckpt".format(global_step))
saver.save(sess, filename)
