def train_epoch(epoch, train_x, train_y, model, sess):
losses = []
accs = []
batches = data_generator(train_x, train_y, BATCH_SIZE)
for x_batch, y_batch in batches:
feed_dict = {
model.input_x: x_batch,
model.input_y: y_batch,
model.max_sentence_num: MAX_SENT_IN_DOC,
model.max_sentence_length: MAX_WORD_IN_SENT,
model.batch_size: BATCH_SIZE,
model.keep_prob: KEEP_PROB
}
_, cost, accuracy = sess.run([model.train_op, model.loss, model.acc],
feed_dict)
logging.info(
"++++++++++++++++++train++++++++++++++{}: step_loss {:g}, step_acc {:g}"
.format(epoch, cost, accuracy))
losses += [cost]
accs += [accuracy]
ave_loss = sum(losses) / len(losses)
ave_acc = sum(accs) / len(accs)
logging.info(
"ooooooooooooooooooooTRAINoooooooooooooooooo{}: ave_loss {:g}, ave_acc {:g}"
.format(epoch, ave_loss, ave_acc))
return ave_loss, ave_acc
def dev_epoch(dev_x, dev_y, model, sess):
losses = []
accs = []
predicts = []
batches = data_generator(dev_x, dev_y, BATCH_SIZE)
for x_batch, y_batch in batches:
feed_dict = {
model.input_x: x_batch,
model.input_y: y_batch,
model.max_sentence_num: MAX_SENT_IN_DOC,
model.max_sentence_length: MAX_WORD_IN_SENT,
model.batch_size: BATCH_SIZE,
model.keep_prob: 1.0
}
cost, accuracy, predict = sess.run(
[model.loss, model.acc, model.predict], feed_dict)
losses += [cost]
accs += [accuracy]
predicts.extend(predict)
ave_loss = sum(losses) / len(losses)
ave_acc = sum(accs) / len(accs)
time_str = str(int(time.time()))
logging.info(
"ooooooooooooooooooooDEVoooooooooooooooooo{}: ave_loss {:g}, ave_acc {:g}"
.format(time_str, ave_loss, ave_acc))
return ave_loss, ave_acc, predicts
def calculate_acc_on_testfile(file_path):
model = AnalysisScorerModel.create_from_existed_model(
model_name="lookup_disambiguator_wo_suffix")
test_data = data_generator(file_path,
add_gold_labels=True,
case_sensitive=True)
corrects = 0
total = 0
with open("data/incorrect_analyzes.csv", "w", encoding="UTF-8") as f:
f.write("Surface\tGold\tPredicted\n")
for sentence in test_data:
predicted_indexes = model.predict_indices(sentence)
for word, selected_index in zip(sentence, predicted_indexes):
gold_analysis = word.roots[0] + "+" + "+".join(word.tags[0])
gold_analysis = gold_analysis.replace("+DB", "^DB")
selected_analysis = word.roots[
selected_index] + "+" + "+".join(word.tags[selected_index])
selected_analysis = selected_analysis.replace("+DB", "^DB")
if to_lower(selected_analysis) == to_lower(gold_analysis):
corrects += 1
else:
f.write("{}\t{}\t{}\n".format(word.surface_word,
gold_analysis,
selected_analysis))
total += 1
print("Accuracy: {}".format(corrects * 1.0 / total))
def get_feature(files, batch_size=8):
preds = []
file_names = []
for fnames, imgs in tqdm(data_generator(files, batch=batch_size),
total=len(files) / batch_size):
predicts = inference_model.predict(imgs)
preds += predicts.tolist()
file_names += fnames
return np.array(preds), file_names
def gen_distance(base_model, file_id_mapping, multiple):
lable2file = defaultdict(list)
for file, label in file_id_mapping.items():
lable2file[label].append(file)
files = file_id_mapping.keys()
vectors = []
for fnames, imgs in tqdm(data_generator(files, batch=batch_size),
total=len(files) / batch_size,
desc='gen_vectors'):
predicts = base_model.predict(imgs)
vectors += predicts.tolist()
file_vertors = dict(zip(files, vectors))
file_distance = {}
for label, files in tqdm(lable2file.items(), desc='computer distances'):
for file in files:
# print file in file_vertors
file_distance[file] = {}
file_distance[file]['same'] = compute_distance(file_vertors,
file,
files,
multiple,
sample_num=100,
reverse=True)
temp_lf = copy.deepcopy(lable2file)
temp_lf.pop(label)
other_file = sum(temp_lf.values(), [])
file_distance[file]['different'] = compute_distance(file_vertors,
file,
other_file,
multiple,
sample_num=100,
reverse=False)
return file_distance
if __name__ == '__main__':
os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3"
os.environ["CUDA_VISIBLE_DEVICES"] = args.cuda_device
config = Config_Params_SASA(args)
logger = init_logger(config.log_save_file)
config_proto = tf.ConfigProto(log_device_placement=False,
allow_soft_placement=True)
config_proto.gpu_options.allow_growth = True
## data preparing..
src_train_generator = data_generator(
data_path=config.src_train_path,
segments_length=config.segments_length,
window_size=config.window_size,
batch_size=config.batch_size // 2,
dataset=config.dataset,
is_shuffle=True,
)
tgt_train_generator = data_generator(
data_path=config.tgt_train_path,
segments_length=config.segments_length,
window_size=config.window_size,
batch_size=config.batch_size // 2,
dataset=config.dataset,
is_shuffle=True,
)
tgt_test_generator = data_generator(
data_path=config.tgt_test_path,
segments_length=config.segments_length,
window_size=config.window_size,
def train(**kwargs):
"""
Train model
Load the whole train data in memory for faster operations
args: **kwargs (dict) keyword arguments that specify the model hyperparameters
"""
# Roll out the parameters
img_dim = kwargs["img_dim"]
patch_size = kwargs["patch_size"]
image_data_format = kwargs["image_data_format"]
generator_type = kwargs["generator_type"]
dset = kwargs["dset"]
use_identity_image = kwargs["use_identity_image"]
batch_size = kwargs["batch_size"]
n_batch_per_epoch = kwargs["n_batch_per_epoch"]
nb_epoch = kwargs["nb_epoch"]
augment_data = kwargs["augment_data"]
model_name = kwargs["model_name"]
save_weights_every_n_epochs = kwargs["save_weights_every_n_epochs"]
visualize_images_every_n_epochs = kwargs["visualize_images_every_n_epochs"]
save_only_last_n_weights = kwargs["save_only_last_n_weights"]
use_mbd = kwargs["use_mbd"]
label_smoothing = kwargs["use_label_smoothing"]
label_flipping_prob = kwargs["label_flipping_prob"]
use_l1_weighted_loss = kwargs["use_l1_weighted_loss"]
prev_model = kwargs["prev_model"]
change_model_name_to_prev_model = kwargs["change_model_name_to_prev_model"]
discriminator_optimizer = kwargs["discriminator_optimizer"]
n_run_of_gen_for_1_run_of_disc = kwargs["n_run_of_gen_for_1_run_of_disc"]
load_all_data_at_once = kwargs["load_all_data_at_once"]
MAX_FRAMES_PER_GIF = kwargs["MAX_FRAMES_PER_GIF"]
dont_train = kwargs["dont_train"]
# batch_size = args.batch_size
# n_batch_per_epoch = args.n_batch_per_epoch
# nb_epoch = args.nb_epoch
# save_weights_every_n_epochs = args.save_weights_every_n_epochs
# generator_type = args.generator_type
# patch_size = args.patch_size
# label_smoothing = False
# label_flipping_prob = False
# dset = args.dset
# use_mbd = False
if dont_train:
# Get the number of non overlapping patch and the size of input image to the discriminator
nb_patch, img_dim_disc = data_utils.get_nb_patch(img_dim, patch_size, image_data_format)
generator_model = models.load("generator_unet_%s" % generator_type,
img_dim,
nb_patch,
use_mbd,
batch_size,
model_name)
generator_model.compile(loss='mae', optimizer='adam')
return generator_model
# Check and make the dataset
# If .h5 file of dset is not present, try making it
if load_all_data_at_once:
if not os.path.exists("../../data/processed/%s_data.h5" % dset):
print("dset %s_data.h5 not present in '../../data/processed'!" % dset)
if not os.path.exists("../../data/%s/" % dset):
print("dset folder %s not present in '../../data'!\n\nERROR: Dataset .h5 file not made, and dataset not available in '../../data/'.\n\nQuitting." % dset)
return
else:
if not os.path.exists("../../data/%s/train" % dset) or not os.path.exists("../../data/%s/val" % dset) or not os.path.exists("../../data/%s/test" % dset):
print("'train', 'val' or 'test' folders not present in dset folder '../../data/%s'!\n\nERROR: Dataset must contain 'train', 'val' and 'test' folders.\n\nQuitting." % dset)
return
else:
print("Making %s dataset" % dset)
subprocess.call(['python3', '../data/make_dataset.py', '../../data/%s' % dset, '3'])
print("Done!")
else:
if not os.path.exists(dset):
print("dset does not exist! Given:", dset)
return
if not os.path.exists(os.path.join(dset, 'train')):
print("dset does not contain a 'train' dir! Given dset:", dset)
return
if not os.path.exists(os.path.join(dset, 'val')):
print("dset does not contain a 'val' dir! Given dset:", dset)
return
epoch_size = n_batch_per_epoch * batch_size
init_epoch = 0
if prev_model:
print('\n\nLoading prev_model from', prev_model, '...\n\n')
prev_model_latest_gen = sorted(glob.glob(os.path.join('../../models/', prev_model, '*gen*epoch*.h5')))[-1]
prev_model_latest_disc = sorted(glob.glob(os.path.join('../../models/', prev_model, '*disc*epoch*.h5')))[-1]
prev_model_latest_DCGAN = sorted(glob.glob(os.path.join('../../models/', prev_model, '*DCGAN*epoch*.h5')))[-1]
print(prev_model_latest_gen, prev_model_latest_disc, prev_model_latest_DCGAN)
if change_model_name_to_prev_model:
# Find prev model name, epoch
model_name = prev_model_latest_DCGAN.split('models')[-1].split('/')[1]
init_epoch = int(prev_model_latest_DCGAN.split('epoch')[1][:5]) + 1
# img_dim = X_target_train.shape[-3:]
# img_dim = (256, 256, 3)
# Get the number of non overlapping patch and the size of input image to the discriminator
nb_patch, img_dim_disc = data_utils.get_nb_patch(img_dim, patch_size, image_data_format)
try:
# Create optimizers
opt_dcgan = Adam(lr=1E-3, beta_1=0.9, beta_2=0.999, epsilon=1e-08)
if discriminator_optimizer == 'sgd':
opt_discriminator = SGD(lr=1E-3, momentum=0.9, nesterov=True)
elif discriminator_optimizer == 'adam':
opt_discriminator = Adam(lr=1E-3, beta_1=0.9, beta_2=0.999, epsilon=1e-08)
# Load generator model
generator_model = models.load("generator_unet_%s" % generator_type,
img_dim,
nb_patch,
use_mbd,
batch_size,
model_name)
generator_model.compile(loss='mae', optimizer=opt_dcgan)
# Load discriminator model
discriminator_model = models.load("DCGAN_discriminator",
img_dim_disc,
nb_patch,
use_mbd,
batch_size,
model_name)
discriminator_model.trainable = False
DCGAN_model = models.DCGAN(generator_model,
discriminator_model,
img_dim,
patch_size,
image_data_format)
if use_l1_weighted_loss:
loss = [l1_weighted_loss, 'binary_crossentropy']
else:
loss = [l1_loss, 'binary_crossentropy']
loss_weights = [1E1, 1]
DCGAN_model.compile(loss=loss, loss_weights=loss_weights, optimizer=opt_dcgan)
discriminator_model.trainable = True
discriminator_model.compile(loss='binary_crossentropy', optimizer=opt_discriminator)
# Load prev_model
if prev_model:
generator_model.load_weights(prev_model_latest_gen)
discriminator_model.load_weights(prev_model_latest_disc)
DCGAN_model.load_weights(prev_model_latest_DCGAN)
# Load .h5 data all at once
print('\n\nLoading data...\n\n')
check_this_process_memory()
if load_all_data_at_once:
X_target_train, X_sketch_train, X_target_val, X_sketch_val = data_utils.load_data(dset, image_data_format)
check_this_process_memory()
print('X_target_train: %.4f' % (X_target_train.nbytes/2**30), "GB")
print('X_sketch_train: %.4f' % (X_sketch_train.nbytes/2**30), "GB")
print('X_target_val: %.4f' % (X_target_val.nbytes/2**30), "GB")
print('X_sketch_val: %.4f' % (X_sketch_val.nbytes/2**30), "GB")
# To generate training data
X_target_batch_gen_train, X_sketch_batch_gen_train = data_utils.data_generator(X_target_train, X_sketch_train, batch_size, augment_data=augment_data)
X_target_batch_gen_val, X_sketch_batch_gen_val = data_utils.data_generator(X_target_val, X_sketch_val, batch_size, augment_data=False)
# Load data from images through an ImageDataGenerator
else:
X_batch_gen_train = data_utils.data_generator_from_dir(os.path.join(dset, 'train'), target_size=(img_dim[0], 2*img_dim[1]), batch_size=batch_size)
X_batch_gen_val = data_utils.data_generator_from_dir(os.path.join(dset, 'val'), target_size=(img_dim[0], 2*img_dim[1]), batch_size=batch_size)
check_this_process_memory()
# Setup environment (logging directory etc)
general_utils.setup_logging(**kwargs)
# Losses
disc_losses = []
gen_total_losses = []
gen_L1_losses = []
gen_log_losses = []
# Start training
print("\n\nStarting training...\n\n")
# For each epoch
for e in range(nb_epoch):
# Initialize progbar and batch counter
# progbar = generic_utils.Progbar(epoch_size)
batch_counter = 0
gen_total_loss_epoch = 0
gen_L1_loss_epoch = 0
gen_log_loss_epoch = 0
start = time.time()
# For each batch
# for X_target_batch, X_sketch_batch in data_utils.gen_batch(X_target_train, X_sketch_train, batch_size):
for batch in range(n_batch_per_epoch):
# Create a batch to feed the discriminator model
if load_all_data_at_once:
X_target_batch_train, X_sketch_batch_train = next(X_target_batch_gen_train), next(X_sketch_batch_gen_train)
else:
X_target_batch_train, X_sketch_batch_train = data_utils.load_data_from_data_generator_from_dir(X_batch_gen_train, img_dim=img_dim,
augment_data=augment_data,
use_identity_image=use_identity_image)
X_disc, y_disc = data_utils.get_disc_batch(X_target_batch_train,
X_sketch_batch_train,
generator_model,
batch_counter,
patch_size,
image_data_format,
label_smoothing=label_smoothing,
label_flipping_prob=label_flipping_prob)
# Update the discriminator
disc_loss = discriminator_model.train_on_batch(X_disc, y_disc)
# Create a batch to feed the generator model
if load_all_data_at_once:
X_gen_target, X_gen_sketch = next(X_target_batch_gen_train), next(X_sketch_batch_gen_train)
else:
X_gen_target, X_gen_sketch = data_utils.load_data_from_data_generator_from_dir(X_batch_gen_train, img_dim=img_dim,
augment_data=augment_data,
use_identity_image=use_identity_image)
y_gen_target = np.zeros((X_gen_target.shape[0], 2), dtype=np.uint8)
y_gen_target[:, 1] = 1
# Freeze the discriminator
discriminator_model.trainable = False
# Train generator
for _ in range(n_run_of_gen_for_1_run_of_disc-1):
gen_loss = DCGAN_model.train_on_batch(X_gen_sketch, [X_gen_target, y_gen_target])
gen_total_loss_epoch += gen_loss[0]/n_run_of_gen_for_1_run_of_disc
gen_L1_loss_epoch += gen_loss[1]/n_run_of_gen_for_1_run_of_disc
gen_log_loss_epoch += gen_loss[2]/n_run_of_gen_for_1_run_of_disc
if load_all_data_at_once:
X_gen_target, X_gen_sketch = next(X_target_batch_gen_train), next(X_sketch_batch_gen_train)
else:
X_gen_target, X_gen_sketch = data_utils.load_data_from_data_generator_from_dir(X_batch_gen_train, img_dim=img_dim,
augment_data=augment_data,
use_identity_image=use_identity_image)
gen_loss = DCGAN_model.train_on_batch(X_gen_sketch, [X_gen_target, y_gen_target])
# Add losses
gen_total_loss_epoch += gen_loss[0]/n_run_of_gen_for_1_run_of_disc
gen_L1_loss_epoch += gen_loss[1]/n_run_of_gen_for_1_run_of_disc
gen_log_loss_epoch += gen_loss[2]/n_run_of_gen_for_1_run_of_disc
# Unfreeze the discriminator
discriminator_model.trainable = True
# Progress
# progbar.add(batch_size, values=[("D logloss", disc_loss),
# ("G tot", gen_loss[0]),
# ("G L1", gen_loss[1]),
# ("G logloss", gen_loss[2])])
print("Epoch", str(init_epoch+e+1), "batch", str(batch+1), "D_logloss", disc_loss, "G_tot", gen_loss[0], "G_L1", gen_loss[1], "G_log", gen_loss[2])
gen_total_loss = gen_total_loss_epoch/n_batch_per_epoch
gen_L1_loss = gen_L1_loss_epoch/n_batch_per_epoch
gen_log_loss = gen_log_loss_epoch/n_batch_per_epoch
disc_losses.append(disc_loss)
gen_total_losses.append(gen_total_loss)
gen_L1_losses.append(gen_L1_loss)
gen_log_losses.append(gen_log_loss)
# Save images for visualization
if (e + 1) % visualize_images_every_n_epochs == 0:
data_utils.plot_generated_batch(X_target_batch_train, X_sketch_batch_train, generator_model, batch_size, image_data_format,
model_name, "training", init_epoch + e + 1, MAX_FRAMES_PER_GIF)
# Get new images for validation
if load_all_data_at_once:
X_target_batch_val, X_sketch_batch_val = next(X_target_batch_gen_val), next(X_sketch_batch_gen_val)
else:
X_target_batch_val, X_sketch_batch_val = data_utils.load_data_from_data_generator_from_dir(X_batch_gen_val, img_dim=img_dim, augment_data=False, use_identity_image=use_identity_image)
# Predict and validate
data_utils.plot_generated_batch(X_target_batch_val, X_sketch_batch_val, generator_model, batch_size, image_data_format,
model_name, "validation", init_epoch + e + 1, MAX_FRAMES_PER_GIF)
# Plot losses
data_utils.plot_losses(disc_losses, gen_total_losses, gen_L1_losses, gen_log_losses, model_name, init_epoch)
# Save weights
if (e + 1) % save_weights_every_n_epochs == 0:
# Delete all but the last n weights
purge_weights(save_only_last_n_weights, model_name)
# Save gen weights
gen_weights_path = os.path.join('../../models/%s/gen_weights_epoch%05d_discLoss%.04f_genTotL%.04f_genL1L%.04f_genLogL%.04f.h5' % (model_name, init_epoch + e, disc_losses[-1], gen_total_losses[-1], gen_L1_losses[-1], gen_log_losses[-1]))
print("Saving", gen_weights_path)
generator_model.save_weights(gen_weights_path, overwrite=True)
# Save disc weights
disc_weights_path = os.path.join('../../models/%s/disc_weights_epoch%05d_discLoss%.04f_genTotL%.04f_genL1L%.04f_genLogL%.04f.h5' % (model_name, init_epoch + e, disc_losses[-1], gen_total_losses[-1], gen_L1_losses[-1], gen_log_losses[-1]))
print("Saving", disc_weights_path)
discriminator_model.save_weights(disc_weights_path, overwrite=True)
# Save DCGAN weights
DCGAN_weights_path = os.path.join('../../models/%s/DCGAN_weights_epoch%05d_discLoss%.04f_genTotL%.04f_genL1L%.04f_genLogL%.04f.h5' % (model_name, init_epoch + e, disc_losses[-1], gen_total_losses[-1], gen_L1_losses[-1], gen_log_losses[-1]))
print("Saving", DCGAN_weights_path)
DCGAN_model.save_weights(DCGAN_weights_path, overwrite=True)
check_this_process_memory()
print('[{0:%Y/%m/%d %H:%M:%S}] Epoch {1:d}/{2:d} END, Time taken: {3:.4f} seconds'.format(datetime.datetime.now(), init_epoch + e + 1, init_epoch + nb_epoch, time.time() - start))
print('------------------------------------------------------------------------------------')
except KeyboardInterrupt:
pass
# SAVE THE MODEL
try:
# Save the model as it is, so that it can be loaded using -
# ```from keras.models import load_model; gen = load_model('generator_latest.h5')```
gen_weights_path = '../../models/%s/generator_latest.h5' % (model_name)
print("Saving", gen_weights_path)
generator_model.save(gen_weights_path, overwrite=True)
# Save model as json string
generator_model_json_string = generator_model.to_json()
print("Saving", '../../models/%s/generator_latest.txt' % model_name)
with open('../../models/%s/generator_latest.txt' % model_name, 'w') as outfile:
a = outfile.write(generator_model_json_string)
# Save model as json
generator_model_json_data = json.loads(generator_model_json_string)
print("Saving", '../../models/%s/generator_latest.json' % model_name)
with open('../../models/%s/generator_latest.json' % model_name, 'w') as outfile:
json.dump(generator_model_json_data, outfile)
except:
print(sys.exc_info()[0])
print("Done.")
return generator_model
def main(_):
hparam = Config()
dataloader = IMDB(hparam.batch_size, hparam.seq_len, hparam.max_vocab_size, hparam.index_from)
hparam.vocab_size = dataloader.vocab_size
model = RNNTextClassification(hparam)
with tf.Session() as sess:
sess.run(model.init)
train_writer = tf.summary.FileWriter(hparam.save_path+'/train', sess.graph)
test_writer = tf.summary.FileWriter(hparam.save_path+'/test', sess.graph)
train_generator = data_generator(dataloader.trainX, dataloader.trainY, hparam.batch_size)
test_generator = data_generator(dataloader.testX, dataloader.testY, hparam.batch_size)
print("Start Learning.")
best_test_results = 10000
for epoch in range(hparam.num_epoch):
print(" Starting Epoch [%d/%d]:" % (epoch, hparam.num_epoch))
train_loss = 0.0
train_accuracy = 0.0
train_num_batches = dataloader.train_num_batches
for b in tqdm(range(train_num_batches)):
x_batch, y_batch = next(train_generator)
seq_len = np.array([list(x).index(0) + 1 for x in x_batch])
lo, acc, _, summ = sess.run([model.loss, model.accuracy, model.train_op, model.merged],
feed_dict={model.inputs:x_batch,
model.targets: y_batch,
model.lengths:seq_len,
model.keep_prob:hparam.keep_prob})
train_loss += lo
train_accuracy += acc
train_writer.add_summary(summ, b + train_num_batches * epoch)
train_loss /= train_num_batches
train_accuracy /= train_num_batches
test_loss = 0.0
test_accuracy = 0.0
test_num_batches = dataloader.test_num_batches
for b in tqdm(range(test_num_batches)):
test_x_batch, test_y_batch = next(test_generator)
test_seq_len = np.array([list(x).index(0) + 1 for x in test_x_batch])
lo_test, acc_test, summ_test = sess.run([model.loss, model.accuracy, model.merged],
feed_dict={model.inputs:test_x_batch,
model.targets:test_y_batch,
model.lengths:test_seq_len,
model.keep_prob:1.0})
test_loss += lo_test
test_accuracy += acc_test
test_writer.add_summary(summ_test, b + test_num_batches * epoch)
test_loss /= test_num_batches
test_accuracy /= test_num_batches
print(" loss: {:.6f} | test_loss: {:.6f} | acc: {:.6f} | test_acc: {:.6f}".format(
train_loss, test_loss, train_accuracy, test_accuracy))
train_writer.close()
test_writer.close()
if best_test_results > test_loss:
best_test_results = test_loss
model.saver.save(sess, hparam.save_path + "/model.ckpt")
print("Run 'tensorboard --logdir=save' to checkout tensorboard logs.")
def test():
test_X, test_y = prepare_test_data(TEST_FILE)
# Evaluation
# ==================================================
checkpoint_file = tf.train.latest_checkpoint('../model_and_log/model7/')
graph = tf.Graph()
with graph.as_default():
sess = tf.Session()
with sess.as_default():
saver = tf.train.import_meta_graph(
"{}.meta".format(checkpoint_file))
saver.restore(sess, checkpoint_file)
# Get the placeholders from the graph by name
max_sentence_num = graph.get_operation_by_name(
'placeholder/max_sentence_num').outputs[0]
max_sentence_length = graph.get_operation_by_name(
'placeholder/max_sentence_length').outputs[0]
input_x = graph.get_operation_by_name(
"placeholder/input_x").outputs[0]
input_y = graph.get_operation_by_name(
"placeholder/input_y").outputs[0]
keep_prob = graph.get_operation_by_name(
'placeholder/keep_prob').outputs[0]
batch_size = graph.get_operation_by_name(
'placeholder/batch_size').outputs[0]
# Tensors we want to evaluate
predictions = graph.get_operation_by_name("predict").outputs[0]
acc = graph.get_operation_by_name('accuracy/acc').outputs[0]
# Generate batches for one epoch
batches = data_generator(test_X, test_y, BATCH_SIZE)
# Collect the predictions here
all_predictions = []
all_y_test = []
for x_test_batch, y_test_batch in batches:
batch_predictions = sess.run(
predictions, {
max_sentence_num: MAX_SENT_IN_DOC,
max_sentence_length: MAX_WORD_IN_SENT,
input_x: x_test_batch,
batch_size: BATCH_SIZE,
keep_prob: 1.0
})
all_predictions = np.concatenate(
[all_predictions, batch_predictions])
all_y_test.extend(y_test_batch)
# calculate accuracy
correct_predictions = float(sum(all_predictions == all_y_test))
logging.info("Total number of test examples: {}".format(len(all_y_test)))
logging.info('+++++++++++++{}+++++++++++++'.format(correct_predictions))
logging.info("Accuracy: {:g}".format(correct_predictions /
float(len(all_y_test))))
# transfer the id to label
preds = cat_label_obj.id_to_label(all_predictions)
targets = cat_label_obj.id_to_label(all_y_test)
logging.info("++++++++++++++++++++++HAN MODEL+++++++++++++++++++")
logging.info(classification_report(y_true=targets, y_pred=preds, digits=4))
rouge = Rouge()
if os.path.exists(save_path) is not True:
os.mkdir(save_path)
dev_data = pd.read_csv(dev_path, encoding='utf-8')
dev_texts = list(dev_data['text'].values)
dev_summaries = list(dev_data['summarization'].values)
summary_all = []
for summary in dev_summaries:
summary = ''.join([word + ' ' for words in summary for word in words])
summary_all.append(summary)
for i in range(args.epochs):
for txt, summary in data_generator(train_path, tokenizer, args.batch_size,
args.max_input_len, args.max_output_len):
loss, acc, iter = train(iter, txt, summary)
count_loss += loss
count_acc += acc
n += 1
if iter % 500 == 0:
count_loss = count_loss / n
count_acc = count_acc / n
pre_all = []
for test in dev_texts:
text_id = bert_token_and_to_id(tokenizer, test)
pre = eval(text_id[:args.max_input_len], args.topk)
# 防止生成[]导致rouge报错
if pre == []:
def __init__(self,
train_from_scratch=True,
char_representation_len=128,
word_lstm_rep_len=256,
train_data_path="data/data.train.txt",
dev_data_path="data/data.dev.txt",
test_data_paths=["data/data.test.txt"],
model_file_name=None,
case_sensitive=True):
assert word_lstm_rep_len % 2 == 0
self.case_sensitive = case_sensitive
self.candidate_generator = TurkishStemSuffixCandidateGenerator(
case_sensitive=case_sensitive)
if train_from_scratch:
assert train_data_path
logger.info("Loading data...")
self.train_data_path = train_data_path
self.dev_data_path = dev_data_path
self.test_data_paths = test_data_paths
self.train = data_generator(train_data_path, add_gold_labels=True)
logger.info("Creating or Loading Vocabulary...")
self.char2id = self._create_vocab_chars(self.train)
self.train = data_generator(train_data_path, add_gold_labels=True)
self.tag2id = self._create_vocab_tags(self.train)
self.dev = None
self.tests = []
for test_path in self.test_data_paths:
self.tests.append(
data_generator(test_path, add_gold_labels=True))
logger.info("Building model...")
self.model = dy.Model()
self.trainer = dy.AdamTrainer(self.model)
self.CHARS_LOOKUP = self.model.add_lookup_parameters(
(len(self.char2id) + 2, char_representation_len))
self.TAGS_LOOKUP = self.model.add_lookup_parameters(
(len(self.tag2id) + 2, char_representation_len))
self.fwdRNN_surface = dy.LSTMBuilder(1, char_representation_len,
word_lstm_rep_len / 2,
self.model)
self.bwdRNN_surface = dy.LSTMBuilder(1, char_representation_len,
word_lstm_rep_len / 2,
self.model)
self.fwdRNN_surface.set_dropout(0.2)
self.bwdRNN_surface.set_dropout(0.2)
self.fwdRNN_root = dy.LSTMBuilder(1, char_representation_len,
word_lstm_rep_len / 2,
self.model)
self.bwdRNN_root = dy.LSTMBuilder(1, char_representation_len,
word_lstm_rep_len / 2,
self.model)
self.fwdRNN_root.set_dropout(0.2)
self.bwdRNN_root.set_dropout(0.2)
self.fwdRNN_tag = dy.LSTMBuilder(1, char_representation_len,
word_lstm_rep_len / 2, self.model)
self.bwdRNN_tag = dy.LSTMBuilder(1, char_representation_len,
word_lstm_rep_len / 2, self.model)
self.fwdRNN_tag.set_dropout(0.2)
self.bwdRNN_tag.set_dropout(0.2)
self.fwdRNN_context = dy.LSTMBuilder(2, word_lstm_rep_len,
word_lstm_rep_len, self.model)
self.bwdRNN_context = dy.LSTMBuilder(2, word_lstm_rep_len,
word_lstm_rep_len, self.model)
self.fwdRNN_context.set_dropout(0.2)
self.bwdRNN_context.set_dropout(0.2)
self.train_model(model_name=model_file_name)
else:
logger.info("Loading Pre-Trained Model")
assert model_file_name
self.load_model(model_file_name, char_representation_len,
word_lstm_rep_len)
logger.info("Ready")
def error_analysis(file_path, output_path, add_gold_labels=True):
test_data = data_generator(file_path, add_gold_labels=add_gold_labels)
stemmer = AnalysisScorerModel.create_from_existed_model(
"lookup_disambiguator_wo_suffix")
corrects = 0
total = 0
ambiguous_count = 0
ambiguous_corrects = 0
with open("error_analysis/" + output_path, "w", encoding="UTF-8") as f:
for sentence_index, sentence in enumerate(test_data):
if sentence_index % 100 == 0:
print("{} sentences processed so far.".format(sentence_index))
scores = stemmer.propogate(sentence)
for word_index, (score, word) in enumerate(zip(scores, sentence)):
analysis_scores = {}
probs = dy.softmax(score)
analyzes_probs = probs.npvalue()
max_analysis = ""
max_prob = 0.0
for i, (root, analysis, analysis_prob) in enumerate(
zip(word.roots, word.tags, analyzes_probs)):
analysis_str = "+".join(analysis).replace("+DB", "^DB")
if "Prop" in analysis_str:
root = capitalize(root)
analysis_str = root + "+" + analysis_str
if i == 0:
correct_analysis = analysis_str
if analysis_prob > max_prob:
max_prob = analysis_prob
max_analysis = analysis_str
analysis_scores[analysis_str] = analysis_prob
if word.ambiguity_level > 0:
ambiguous_count += 1
if max_analysis == correct_analysis:
corrects += 1
if word.ambiguity_level > 0:
ambiguous_corrects += 1
else:
f.write("Surface: {}\n".format(word.surface_word))
f.write(
"Correct analysis: {}\tSelected analysis: {}\n".format(
correct_analysis, max_analysis))
if word_index < 2:
start = 0
else:
start = word_index - 2
if word_index > len(sentence) - 3:
end = len(sentence)
else:
end = word_index + 3
f.write("Context: {}\n".format(" ".join(
[w.surface_word for w in sentence[start:end]])))
for analysis_str, prob in analysis_scores.items():
f.write("{}:\t{}\n".format(analysis_str, prob))
f.write("\n\n")
total += 1
print("Corrects: {}\tTotal: {}\t Accuracy: {}".format(
corrects, total, corrects * 1.0 / total))
print(
"Ambiguous Corrects: {}\tTotal Ambiguous: {}\t Ambiguous Accuracy: {}".
format(corrects, total, corrects * 1.0 / total))
def train_model(self, model_name="model", early_stop=False, num_epoch=200):
logger.info("Starting training...")
max_acc = 0.0
epoch_loss = 0
for epoch in range(num_epoch):
self.train = load_data(self.train_data_path, add_gold_labels=True)
random.shuffle(self.train)
self.dev = data_generator(self.dev_data_path, add_gold_labels=True)
self.tests = []
for test_path in self.test_data_paths:
self.tests.append(
data_generator(test_path, add_gold_labels=True))
t1 = datetime.now()
count = 0
corrects = 0
total = 0
word_counter = 0
for i, sentence in enumerate(self.train, 1):
try:
word_counter += len(sentence)
scores = self.propogate(sentence)
errs = []
for score in scores:
err = dy.pickneglogsoftmax(score, 0)
errs.append(err)
probs = dy.softmax(score)
predicted_label_index = np.argmax(probs.npvalue())
if predicted_label_index == 0:
corrects += 1
total += 1
loss_exp = dy.esum(errs)
cur_loss = loss_exp.scalar_value()
epoch_loss += cur_loss
loss_exp.backward()
self.trainer.update()
if i % 100 == 0: # logger.info status
t2 = datetime.now()
delta = t2 - t1
logger.info(
"{} instances finished in {} seconds, loss={}, acc={}"
.format(i, delta.seconds,
epoch_loss / (word_counter * 1.0),
corrects * 1.0 / total))
count = i
except Exception as e:
logger.error(e)
t2 = datetime.now()
delta = t2 - t1
logger.info(
"Epoch {} finished in {} minutes. loss = {}, train accuracy: {}"
.format(epoch, delta.seconds / 60.0, epoch_loss / count * 1.0,
corrects * 1.0 / total))
# logger.info("Calculating Accuracy on dev set")
epoch_loss = 0
acc, _ = self.calculate_acc(self.dev)
logger.info("Accuracy on dev set:{}".format(acc))
if acc > max_acc:
max_acc = acc
logger.info("Max accuracy increased, saving model...")
self.save_model(model_name)
elif early_stop and max_acc > acc:
logger.info("Max accuracy did not increase, early stopping!")
break
# logger.info("Calculating Accuracy on test sets")
for q in range(len(self.test_data_paths)):
# logger.info("Calculating Accuracy on test set: {}".format(self.test_data_paths[q]))
acc, amb_acc = self.calculate_acc(self.tests[q])
logger.info("Test set: {}, accuracy: {}".format(
self.test_data_paths[q], acc))
model.compile(optimizer=Adam(learning_rate), loss="mse")
# create two generators for training and validation
train_transformations = Compose([
Random_HFlip(p=0.5),
Random_Brightness(p=0.5),
# Crop(top=50, bottom=-20, p=1.),
# Resize(new_size=(200, 66), p=1.),
# Convert_RGB2YUV(p=1.),
], p=1.0)
samples = get_data_infor()
train_samples, validation_samples = train_test_split(samples, test_size=0.2, shuffle=True, random_state=2020)
train_gen = data_generator(train_samples, transformations=train_transformations, batch_size=batch_size)
validation_gen = data_generator(validation_samples, transformations=None, batch_size=batch_size)
checkpoint = ModelCheckpoint(filepath='model.{epoch:02d}-{val_loss:.2f}.h5')
history = model.fit_generator(train_gen,
steps_per_epoch=len(train_samples) // batch_size,
epochs=number_of_epochs,
validation_data=validation_gen,
validation_steps=len(validation_samples) // batch_size,
callbacks=[checkpoint],
verbose=1)
model.save('model.h5')
output_images_dir = './images'
if not os.path.isdir(output_images_dir):