def train(
dim_word=100, # word vector dimensionality
dim=50, #100, # the number of GRU units
encoder='lstm', # encoder model
decoder='lstm', # decoder model
patience=10, # early stopping patience
max_epochs=5000,
finish_after=10000000, # finish after this many updates
decay_c=0., # L2 regularization penalty
clip_c=-1., # gradient clipping threshold
lrate=0.0004, # learning rate
alpha_balance=0.04,
n_words=100000, # vocabulary size
n_words_lemma=100000,
maxlen=100, # maximum length of the description
optimizer='adam',
batch_size=32,
valid_batch_size=32,
save_model='results/MIL/',
saveto='MIL.npz',
dispFreq=100,
validFreq=1000,
saveFreq=1000, # save the parameters after every saveFreq updates
use_dropout=False,
reload_=False,
verbose=False, # print verbose information for debug but slow speed
types='title',
cut_word=False,
cut_news=False,
keep_prob=0.8,
datasets=[],
valid_datasets=[],
test_datasets=[],
tech_data=[],
dictionary=[],
kb_dicts=[],
embedding='', # pretrain embedding file, such as word2vec, GLOVE
dim_kb=5,
RUN_NAME="histogram_visualization",
wait_N=10):
logging.basicConfig(
level=logging.DEBUG,
format="%(asctime)s: %(name)s: %(levelname)s: %(message)s",
filename='results/MIL/log_result.txt')
# Model options
model_options = locals().copy()
with open(dictionary, 'rb') as f:
worddicts = pkl.load(f)
logger.info("Loading knowledge base ...")
# reload options
if reload_ and os.path.exists(saveto):
logger.info("Reload options")
with open('%s.pkl' % saveto, 'rb') as f:
model_options = pkl.load(f)
logger.debug(pprint.pformat(model_options))
logger.info("Loading data")
train = TextIterator(datasets[0],
datasets[1],
dict=dictionary,
types=types,
n_words=n_words,
batch_size=batch_size,
cut_word=cut_word,
cut_news=cut_news,
shuffle=True,
shuffle_sentence=False,
quiet=False)
train_valid = TextIterator(datasets[0],
datasets[1],
dict=dictionary,
types=types,
n_words=n_words,
batch_size=valid_batch_size,
cut_word=cut_word,
cut_news=cut_news,
shuffle=False,
shuffle_sentence=False,
quiet=False)
valid = TextIterator(valid_datasets[0],
valid_datasets[1],
dict=dictionary,
types=types,
n_words=n_words,
batch_size=valid_batch_size,
cut_word=cut_word,
cut_news=cut_news,
shuffle=False,
shuffle_sentence=False,
quiet=False)
test = TextIterator(test_datasets[0],
test_datasets[1],
dict=dictionary,
types=types,
n_words=n_words,
batch_size=valid_batch_size,
cut_word=cut_word,
cut_news=cut_news,
shuffle=False,
shuffle_sentence=False,
quiet=False)
# Initialize (or reload) the parameters using 'model_options'
# then build the tensorflow graph
logger.info("init_word_embedding")
params = init_params(model_options, worddicts)
embedding = word_embedding(model_options, params)
is_training, cost, x, x_mask, y, n_timesteps, group_pred, summary = build_model(
embedding, model_options)
#is_training, cost, x, x_mask, y, n_timesteps, pred, summary = build_model(embedding, model_options)
with tf.variable_scope('train'):
lr = tf.Variable(0.0, trainable=False)
def assign_lr(session, lr_value):
session.run(tf.assign(lr, lr_value))
logger.info('Building optimizers...')
optimizer = tf.train.AdadeltaOptimizer(learning_rate=lr, rho=0.95)
logger.info('Done')
# print all variables
tvars = tf.trainable_variables()
for var in tvars:
print(var.name, var.shape)
lossL = tf.add_n([
tf.nn.l2_loss(v) for v in tvars
if ('embedding' not in v.name and 'bias' not in v.name)
]) #
lossL2 = lossL * 0.0005
print("don't do L2 variables:")
print([
v.name for v in tvars
if ('embedding' in v.name or 'bias' in v.name)
])
print("\n do L2 variables:")
print([
v.name for v in tvars
if ('embedding' not in v.name and 'bias' not in v.name)
])
cost = cost + lossL2
grads, _ = tf.clip_by_global_norm(tf.gradients(cost, tvars),
model_options['clip_c'])
extra_update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(extra_update_ops):
train_op = optimizer.apply_gradients(zip(grads, tvars))
# train_op = optimizer.minimize(cost)
op_loss = tf.reduce_mean(cost)
op_L2 = tf.reduce_mean(lossL)
logger.info("correct_pred")
#@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@2
correct_pred = tf.equal(group_pred, y) # make prediction
#correct_pred = tf.equal(tf.argmax(input=pred, axis=1), y) # make prediction
logger.info("Done")
temp_accuracy = tf.cast(correct_pred, tf.float32) # change to float32
#@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@temp_auc =
logger.info("init variables")
init = tf.global_variables_initializer()
logger.info("Done")
# saver
saver = tf.train.Saver(max_to_keep=15)
config = tf.ConfigProto()
# config.gpu_options.per_process_gpu_memory_fraction = 0.4
config.gpu_options.allow_growth = True
#gpu_options = tf.GPUOptions(allow_growth=True)
#sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options))
with tf.Session(config=tf.ConfigProto(allow_soft_placement=True,
log_device_placement=False)) as sess:
#sess = tf_debug.LocalCLIDebugWrapperSession(sess)
training_writer = tf.summary.FileWriter(
"results/MIL/logs/{}/training".format(RUN_NAME), sess.graph)
validate_writer = tf.summary.FileWriter(
"results/MIL/logs/{}/validate".format(RUN_NAME), sess.graph)
testing_writer = tf.summary.FileWriter(
"results/MIL/logs/{}/testing".format(RUN_NAME), sess.graph)
sess.run(init)
history_errs = []
history_valid_result = []
history_test_result = []
# reload history
if reload_ and os.path.exists(saveto):
logger.info("Reload history error")
history_errs = list(numpy.load(saveto)['history_errs'])
bad_counter = 0
if validFreq == -1:
validFreq = len(train[0]) / batch_size
if saveFreq == -1:
saveFreq = len(train[0]) / batch_size
loss_plot = defaultdict(list)
uidx = 0
estop = False
accs = []
#@@@@@@@@@@@@theta = np.random.random(self.embeddings_dimension)
#@@@@@@@@@@best_theta = theta
valid_acc_record = []
test_acc_record = []
best_num = -1
best_epoch_num = 0
#@@@@@@@@@best_acc = 0
lr_change_list = []
fine_tune_flag = 1
wait_counter = 0
wait_N = model_options['wait_N']
learning_rate = model_options['lrate']
alpha_balance = model_options['alpha_balance']
assign_lr(sess, learning_rate)
for eidx in range(max_epochs):
n_samples = 0
training_cost = 0
training_acc = 0
for x, y in train:
#@@@@@@@@@@@@@theta = model_options['momentum_value'] * theta - (1 - model_options['momentum_value']) * learning_rate / (uidx + 1) * theta_der
n_samples += len(x)
uidx += 1
keep_prob = model_options['keep_prob']
is_training = True
data_x, data_x_mask, data_y = prepare_data(x,
y,
model_options,
maxlen=maxlen)
print(data_x.shape, data_x_mask.shape, data_y.shape)
assert data_y.shape[0] == data_x.shape[
0], 'Size does not match'
if x is None:
logger.debug(
'Minibatch with zero sample under length {0}'.format(
maxlen))
uidx -= 1
continue
ud_start = time.time()
_, loss, loss_no_mean, temp_acc, l2_check = sess.run(
[train_op, op_loss, cost, temp_accuracy, op_L2],
feed_dict={
'input/x:0': data_x,
'input/x_mask:0': data_x_mask,
'input/y:0': data_y,
'input/keep_prob:0': keep_prob,
'input/is_training:0': is_training,
'input/alpha_balance:0': alpha_balance
})
ud = time.time() - ud_start
training_cost += loss_no_mean.sum()
training_acc += temp_acc.sum()
#@@@@@@@@@@@@@@@@@@@@training_auc += temp_auc.sum()
loss_plot['training'].append(loss)
'''train_summary = sess.run(summary, feed_dict={'input/x:0': data_x, 'input/x_mask:0': data_x_mask,
'input/y:0': data_y,'input/keep_prob:0':keep_prob,'input/is_training:0':is_training})
training_writer.add_summary(train_summary, eidx)'''
if numpy.mod(uidx, dispFreq) == 0:
logger.debug(
'Epoch {0} Update {1} Cost {2} L2 {3} TIME {4}'.format(
eidx, uidx, loss, l2_check, ud))
# validate model on validation set and early stop if necessary
if numpy.mod(uidx, validFreq) == 0:
is_training = False
#pred-->group_pred
valid_acc, valid_loss, valid_final_result = predict_pro_acc(
sess,
cost,
prepare_data,
model_options,
valid,
maxlen,
correct_pred,
group_pred,
summary,
eidx,
is_training,
train_op,
loss_plot,
validate_writer,
validate=True)
test_acc, test_loss, test_final_result = predict_pro_acc(
sess, cost, prepare_data, model_options, test, maxlen,
correct_pred, group_pred, summary, eidx, is_training,
train_op, loss_plot, testing_writer)
# valid_err = 1.0 - valid_acc
valid_err = valid_loss
history_errs.append(valid_err)
history_valid_result.append(valid_final_result)
history_test_result.append(test_final_result)
loss_plot['validate_ep'].append(valid_loss)
loss_plot['val_ep'].append(valid_acc)
loss_plot['testing_ep'].append(test_loss)
loss_plot['test_ep'].append(test_acc)
logger.debug('Epoch {0}'.format(eidx))
logger.debug('Valid cost {0}'.format(valid_loss))
logger.debug('Valid accuracy {0}'.format(valid_acc))
logger.debug('Test cost {0}'.format(test_loss))
logger.debug('Test accuracy {0}'.format(test_acc))
logger.debug('learning_rate: {0}'.format(learning_rate))
valid_acc_record.append(valid_acc)
test_acc_record.append(test_acc)
if uidx == 0 or valid_err <= numpy.array(
history_errs).min():
best_num = best_num + 1
best_epoch_num = eidx
wait_counter = 0
logger.info("Saving...")
saver.save(
sess,
_s(_s(_s(save_model, "epoch"), str(best_num)),
"model.ckpt"))
logger.info(
_s(_s(_s(save_model, "epoch"), str(best_num)),
"model.ckpt"))
numpy.savez(saveto,
history_errs=history_errs,
**params)
pkl.dump(model_options,
open('{}.pkl'.format(saveto), 'wb'))
logger.info("Done")
if valid_err > numpy.array(history_errs).min():
wait_counter += 1
# wait_counter +=1 if valid_err>numpy.array(history_errs).min() else 0
if wait_counter >= wait_N:
logger.info("wait_counter max, need to half the lr")
# print 'wait_counter max, need to half the lr'
bad_counter += 1
wait_counter = 0
logger.debug('bad_counter: {0}'.format(bad_counter))
# TODO change the learining rate
###################################################################
learning_rate = learning_rate * 0.9
learning_rate = learning_rate
assign_lr(sess, learning_rate)
lr_change_list.append(eidx)
logger.debug(
'lrate change to: {0}'.format(learning_rate))
print('lrate change to: ' + str(lrate))
if bad_counter > patience and fine_tune_flag == 0:
logger.debug(
'ATTENTION! INTO FINE TUNING STAGE !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!'
)
optimizer = tf.train.MomentumOptimizer(
learning_rate=0.000001, momentum=0.6)
fine_tune_flag = 1
bad_counter = 0
if bad_counter > patience and fine_tune_flag == 1:
logger.info("Early Stop!")
estop = True
break
if numpy.isnan(valid_err):
pdb.set_trace()
# finish after this many updates
if uidx >= finish_after:
logger.debug(
'Finishing after iterations! {0}'.format(uidx))
# print 'Finishing after %d iterations!' % uidx
estop = True
break
logger.debug('Seen samples: {0}'.format(n_samples))
logger.debug('Training accuracy: {0}'.format(1.0 * training_acc /
n_samples))
loss_plot['training_ep'].append(training_cost / n_samples)
loss_plot['train_ep'].append(training_acc / n_samples)
# print 'Seen %d samples' % n_samples
logger.debug('results/MIL/Saved loss_plot pickle')
with open("results/MIL/important_plot.pickle", 'wb') as handle:
pkl.dump(loss_plot, handle, protocol=pkl.HIGHEST_PROTOCOL)
if estop:
break
with tf.Session(config=tf.ConfigProto(allow_soft_placement=True,
log_device_placement=False)) as sess:
# Restore variables from disk.
saver.restore(
sess, _s(_s(_s(save_model, "epoch"), str(best_num)), "model.ckpt"))
keep_prob = 1
is_training = False
#alpha_balance = 1
logger.info('=' * 80)
logger.info('Final Result')
logger.info('=' * 80)
logger.debug('best epoch {0}'.format(best_epoch_num))
#pred-->group_pred
valid_acc, valid_cost, valid_final_result = predict_pro_acc(
sess, cost, prepare_data, model_options, valid, maxlen,
correct_pred, group_pred, summary, eidx, train_op, is_training,
None)
logger.debug('Valid cost {0}'.format(valid_cost))
logger.debug('Valid accuracy {0}'.format(valid_acc))
# print 'Valid cost', valid_cost
# print 'Valid accuracy', valid_acc
test_acc, test_cost, test_final_result = predict_pro_acc(
sess, cost, prepare_data, model_options, test, maxlen,
correct_pred, group_pred, summary, eidx, train_op, is_training,
None)
logger.debug('Test cost {0}'.format(test_cost))
logger.debug('Test accuracy {0}'.format(test_acc))
# print 'best epoch ', best_epoch_num
train_acc, train_cost, _ = predict_pro_acc(sess, cost, prepare_data,
model_options, train_valid,
maxlen, correct_pred,
group_pred, summary, eidx,
train_op, is_training, None)
logger.debug('Train cost {0}'.format(train_cost))
logger.debug('Train accuracy {0}'.format(train_acc))
valid_m = numpy.array(history_valid_result)
test_m = numpy.array(history_test_result)
valid_final_result = (numpy.array([valid_final_result]) == False)
test_final_result = (numpy.array([test_final_result]) == False)
#print(numpy.all(valid_m, axis = 0))
#print(numpy.all(test_m, axis=0))
print(
'validation: all prediction through every epoch that are the same:',
numpy.where(numpy.all(valid_m, axis=0)))
print('testing: all prediction through every epoch that are the same:',
numpy.where(numpy.all(test_m, axis=0)))
print('validation: final prediction that is False:',
numpy.where(valid_final_result))
print('testing: final prediction that is False:',
numpy.where(test_final_result))
if os.path.exists('results/MIL/history_predict.npz'):
logger.info("Load and save to history_predict.npz")
valid_history = numpy.load(
'results/MIL/history_predict.npz')['valid_final_result']
test_history = numpy.load(
'results/MIL/history_predict.npz')['test_final_result']
vv = numpy.concatenate((valid_history, valid_final_result), axis=0)
tt = numpy.concatenate((test_history, valid_final_result), axis=0)
print('Concate shape valid:', vv.shape)
print('Print all validate history outputs that return False',
numpy.where(numpy.all(vv, axis=0)))
print('Concate shape test:', tt.shape)
print('Print all test history outputs that return False',
numpy.where(numpy.all(tt, axis=0)))
numpy.savez('results/MIL/history_predict.npz',
valid_final_result=vv,
test_final_result=tt,
**params)
else:
numpy.savez('results/MIL/history_predict.npz',
valid_final_result=valid_final_result,
test_final_result=test_final_result,
**params)
# print 'Train cost', train_cost
# print 'Train accuracy', train_acc
# print 'Test cost ', test_cost
# print 'Test accuracy ', test_acc
return None
def train(
dim_word=100, # word vector dimensionality
encoder='lstm', # encoder model
decoder='lstm', # decoder model
patience=10, # early stopping patience
max_epochs=300,
finish_after=10000000, # finish after this many updates
decay_c=0., # L2 regularization penalty
clip_c=-1., # gradient clipping threshold
lrate=0.0004, # learning rate
n_words=100000, # vocabulary size
n_words_lemma=100000,
maxlen=100, # maximum length of the description
optimizer='adam',
batch_size=32,
valid_batch_size=32,
save_model='results/s_cnn/',
saveto='S_CNN.npz',
dispFreq=100,
validFreq=1000,
saveFreq=1000, # save the parameters after every saveFreq updates
use_dropout=False,
reload_=False,
verbose=False, # print verbose information for debug but slow speed
types='title',
cut_word=False,
cut_news=False,
CNN_filter=64,
keep_prob=0.8,
datasets=[],
valid_datasets=[],
test_datasets=[],
dictionary=[],
embedding='', # pretrain embedding file, such as word2vec, GLOVE
RUN_NAME="histogram_visualization",
wait_N=10):
logging.basicConfig(
level=logging.DEBUG,
format="%(asctime)s: %(name)s: %(levelname)s: %(message)s",
filename='results/s_cnn/log_result.txt')
# Model options
model_options = locals().copy()
with open(dictionary, 'rb') as f:
worddicts = pkl.load(f)
logger.info("Loading knowledge base ...")
# reload options
"""if reload_ and os.path.exists(saveto):
logger.info("Reload options")
with open('%s.pkl' % saveto, 'rb') as f:
model_options = pkl.load(f)"""
logger.debug(pprint.pformat(model_options))
logger.info("Loading data")
train = TextIterator(datasets[0],
datasets[1],
dict=dictionary,
types=types,
n_words=n_words,
batch_size=batch_size,
cut_word=cut_word,
cut_news=cut_news,
shuffle=True,
shuffle_sentence=False)
train_valid = TextIterator(datasets[0],
datasets[1],
dict=dictionary,
types=types,
n_words=n_words,
batch_size=valid_batch_size,
cut_word=cut_word,
cut_news=cut_news,
shuffle=False,
shuffle_sentence=False)
valid = TextIterator(valid_datasets[0],
valid_datasets[1],
dict=dictionary,
types=types,
n_words=n_words,
batch_size=valid_batch_size,
cut_word=cut_word,
cut_news=cut_news,
shuffle=True,
shuffle_sentence=False) #Shuffle = false
test = TextIterator(test_datasets[0],
test_datasets[1],
dict=dictionary,
types=types,
n_words=n_words,
batch_size=valid_batch_size,
cut_word=cut_word,
cut_news=cut_news,
shuffle=True,
shuffle_sentence=False)
# Initialize (or reload) the parameters using 'model_options'
# then build the tensorflow graph
logger.info("init_word_embedding")
params = init_params(model_options, worddicts)
embedding = word_embedding(model_options, params)
is_training, cost, x, y, n_timesteps, pred, summary = build_model(
embedding, model_options)
with tf.variable_scope('train'):
lr = tf.Variable(0.0, trainable=False)
def assign_lr(session, lr_value):
session.run(tf.assign(lr, lr_value))
logger.info('Building optimizers...')
#optimizer = tf.train.AdamOptimizer(learning_rate=lr)
optimizer = tf.train.AdadeltaOptimizer(learning_rate=lr, rho=0.95)
logger.info('Done')
# print all variables
tvars = tf.trainable_variables()
for var in tvars:
print(var.name, var.shape)
lossL = tf.add_n([
tf.nn.l2_loss(v) for v in tvars
if ('embeddings' not in v.name and 'bias' not in v.name)
]) #
lossL2 = lossL * 0.0005
print("don't do L2 variables:")
print([
v.name for v in tvars
if ('embeddings' in v.name or 'bias' in v.name)
])
print("\n do L2 variables:")
print([
v.name for v in tvars
if ('embeddings' not in v.name and 'bias' not in v.name)
])
cost = cost + lossL2
grads, _ = tf.clip_by_global_norm(tf.gradients(cost, tvars),
model_options['clip_c'])
extra_update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(extra_update_ops):
train_op = optimizer.apply_gradients(zip(grads, tvars))
# train_op = optimizer.minimize(cost)
op_loss = tf.reduce_mean(cost)
op_L2 = tf.reduce_mean(lossL)
logger.info("correct_pred")
correct_pred = tf.equal(tf.argmax(input=pred, axis=1),
y) # make prediction
logger.info("Done")
temp_accuracy = tf.cast(correct_pred, tf.float32) # change to float32
logger.info("init variables")
init = tf.global_variables_initializer()
logger.info("Done")
# saver
saver = tf.train.Saver(max_to_keep=15)
config = tf.ConfigProto()
# config.gpu_options.per_process_gpu_memory_fraction = 0.4
config.gpu_options.allow_growth = True
with tf.Session(config=tf.ConfigProto(allow_soft_placement=True,
log_device_placement=False)) as sess:
#sess = tf_debug.LocalCLIDebugWrapperSession(sess)
training_writer = tf.summary.FileWriter(
"results/s_cnn/logs/{}/training".format(RUN_NAME), sess.graph)
validate_writer = tf.summary.FileWriter(
"results/s_cnn/logs/{}/validate".format(RUN_NAME), sess.graph)
testing_writer = tf.summary.FileWriter(
"results/s_cnn/logs/{}/testing".format(RUN_NAME), sess.graph)
sess.run(init)
history_errs = []
history_valid_result = []
history_test_result = []
# reload history
"""if reload_ and os.path.exists(saveto):
logger.info("Reload history error")
history_errs = list(numpy.load(saveto)['history_errs'])"""
bad_counter = 0
if validFreq == -1:
validFreq = len(train[0]) / batch_size
if saveFreq == -1:
saveFreq = len(train[0]) / batch_size
loss_plot = defaultdict(list)
uidx = 0
estop = False
best_num = -1
best_epoch_num = 0
lr_change_list = []
wait_counter = 0
wait_N = model_options['wait_N']
learning_rate = model_options['lrate']
assign_lr(sess, learning_rate)
for eidx in range(max_epochs):
n_samples = 0
training_cost = 0
training_acc = 0
for x, y in train:
n_samples += len(x)
uidx += 1
keep_prob = model_options['keep_prob']
is_training = True
data_x, data_y = prepare_data(x,
y,
model_options,
maxlen=maxlen)
print(data_x.shape, data_y.shape)
assert data_y.shape[0] == data_x.shape[
0], 'Size does not match'
if x is None:
logger.debug(
'Minibatch with zero sample under length {0}'.format(
maxlen))
uidx -= 1
continue
ud_start = time.time()
_, loss, loss_no_mean, temp_acc, l2_check = sess.run(
[train_op, op_loss, cost, temp_accuracy, op_L2],
feed_dict={
'input/x:0': data_x,
'input/y:0': data_y,
'input/keep_prob:0': keep_prob,
'input/is_training:0': is_training
})
ud = time.time() - ud_start
training_cost += loss_no_mean.sum()
training_acc += temp_acc.sum()
loss_plot['training'].append(loss)
if numpy.mod(uidx, dispFreq) == 0:
logger.debug(
'Epoch {0} Update {1} Cost {2} L2 {3} TIME {4}'.format(
eidx, uidx, loss, l2_check, ud))
# validate model on validation set and early stop if necessary
if numpy.mod(uidx, validFreq) == 0:
is_training = False
valid_acc, valid_loss, valid_final_result = predict_pro_acc(
sess,
cost,
prepare_data,
model_options,
valid,
maxlen,
correct_pred,
pred,
summary,
eidx,
is_training,
train_op,
loss_plot,
validate_writer,
validate=True)
test_acc, test_loss, test_final_result = predict_pro_acc(
sess, cost, prepare_data, model_options, test, maxlen,
correct_pred, pred, summary, eidx, is_training,
train_op, loss_plot, testing_writer)
# valid_err = 1.0 - valid_acc
valid_err = valid_loss
history_errs.append(valid_err)
history_valid_result.append(valid_final_result)
history_test_result.append(test_final_result)
loss_plot['validate_ep'].append(valid_loss)
loss_plot['val_ac'].append(valid_acc) #加的
loss_plot['testing_ep'].append(test_loss)
loss_plot['test_ac'].append(test_acc) #加的
logger.debug('Epoch {0}'.format(eidx))
logger.debug('Valid cost {0}'.format(valid_loss))
logger.debug('Valid accuracy {0}'.format(valid_acc))
logger.debug('Test cost {0}'.format(test_loss))
logger.debug('Test accuracy {0}'.format(test_acc))
logger.debug('learning_rate: {0}'.format(learning_rate))
if uidx == 0 or valid_err <= numpy.array(
history_errs).min():
best_num = best_num + 1
best_epoch_num = eidx
wait_counter = 0
#logger.info("Saving...")
#saver.save(sess, _s(_s(_s(save_model, "epoch"), str(best_num)), "model.ckpt"))
#logger.info(_s(_s(_s(save_model, "epoch"), str(best_num)), "model.ckpt"))
#numpy.savez(saveto, history_errs=history_errs, **params)
#pkl.dump(model_options, open('{}.pkl'.format(saveto), 'wb'))
#logger.info("Done")
if valid_err > numpy.array(history_errs).min():
wait_counter += 1
# wait_counter +=1 if valid_err>numpy.array(history_errs).min() else 0
if wait_counter >= wait_N:
logger.info("wait_counter max, need to half the lr")
# print 'wait_counter max, need to half the lr'
bad_counter += 1
wait_counter = 0
logger.debug('bad_counter: {0}'.format(bad_counter))
# TODO change the learining rate
learning_rate = learning_rate * 0.9
# learning_rate = learning_rate
assign_lr(sess, learning_rate)
lr_change_list.append(eidx)
logger.debug(
'lrate change to: {0}'.format(learning_rate))
# print 'lrate change to: ' + str(lrate)
if bad_counter > patience:
logger.info("Early Stop!")
estop = True
break
if numpy.isnan(valid_err):
pdb.set_trace()
# finish after this many updates
if uidx >= finish_after:
logger.debug(
'Finishing after iterations! {0}'.format(uidx))
# print 'Finishing after %d iterations!' % uidx
estop = True
break
logger.debug('Seen samples: {0}'.format(n_samples))
logger.debug('Training accuracy: {0}'.format(1.0 * training_acc /
n_samples))
loss_plot['training_ep'].append(training_cost / n_samples)
loss_plot['train_ep'].append(training_acc / n_samples)
# print 'Seen %d samples' % n_samples
logger.debug('results/s_cnn/Saved loss_plot pickle')
with open("results/s_cnn/important_plot.pickle", 'wb') as handle:
pkl.dump(loss_plot, handle, protocol=pkl.HIGHEST_PROTOCOL)
if estop:
break
"""with tf.Session(config=tf.ConfigProto(allow_soft_placement=True, log_device_placement=False)) as sess: