def predict(self, X, batch_size=None):
"""
:param X: {array-like, sparse matrix}, shape (n_samples, n_features)
Training vector, where n_samples is the number of samples and
n_features is the number of features.
:param batch_size: int, batch size
:return: decoded predicted encodings
"""
X = np.array(X)
inds = np.arange(len(X))
predictions = []
batch_size = batch_size or len(X)
start = time.time()
while len(inds) > 0:
inds, batch_inds = next_batch(inds, batch_size)
batch_features = X[batch_inds]
batch_preds = self._sess.run(self._output, feed_dict={self._input_ph: batch_features})
predictions.extend(batch_preds)
if self._verbose:
print('The inference took {} seconds'.format(time.time() - start))
predictions = np.squeeze(predictions)
return predictions
def score(self, X, batch_size=None):
"""
:param X: {array-like, sparse matrix}, shape (n_samples, n_features)
Training vector, where n_samples is the number of samples and
n_features is the number of features.
:param y: array-like, shape (n_samples,)
Target vector relative to X.
:param batch_size: int, batch size
:return: mean l2 distance between input and output
"""
X = np.array(X)
inds = np.arange(len(X))
predictions = []
batch_size = batch_size or len(X)
while len(inds) > 0:
inds, batch_inds = next_batch(inds, batch_size)
batch_features = X[batch_inds]
batch_preds = self._sess.run(self._output, feed_dict={self._input_ph: batch_features})
predictions.extend(batch_preds)
predictions = np.array(predictions)
return np.linalg.norm(X - predictions, axis=-1).mean()
y = tf.nn.softmax(tf.matmul(x, W) + b)
y_ = tf.placeholder(tf.float32, [None, cuisine_count])
t = tf.placeholder(tf.float32, [None, ingredients_count])
p = tf.nn.softmax(tf.matmul(t, W) + b)
cross_entropy = -tf.reduce_sum(y_*tf.log(y))
train_step = tf.train.GradientDescentOptimizer(0.001).minimize(cross_entropy)
# train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy)
init = tf.initialize_all_variables()
sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True))
sess.run(init)
for d in ['/cpu:0', '/gpu:0']:
with tf.device(d):
for i in range(100):
batch_xs, batch_ys = utl.next_batch(xs, ys, 12000)
sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys})
correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
print(sess.run(accuracy, feed_dict={x: xs, y_: ys}))
cls = sess.run(p, feed_dict={t: ts})
utl.save_result('tf_gd', cuisine_list, np.argmax(cls, axis=1).tolist(), ids, 'number')
def fit(self, X,
seperate_validation=True, validation_ratio=0.2,
learning_rate=0.01, alpha=1., beta=0.0005,
n_epochs=10, batch_size=16,
use_batch_norm=True,
batch_norm_train=True,
dropout_keep_rate=1.,
early_stopping_epochs=None,
early_stopping_method='dlr',
early_stopping_iters=2,
save_best_model=False,
continue_fit=False):
"""
:param X: {array-like, sparse matrix}, shape (n_samples, n_features)
Training vector, where n_samples is the number of samples and
n_features is the number of features.
:param y: array-like, shape (n_samples,)
Target vector relative to X.
:param loss_type: string, 'rmse' or 'crossentropy'
:param seperate_validation: bool, seperate validation set from X
:param validation_ratio: float, between (0.0-1.0) the ratio of seperated validation (default=0.2)
:param learning_rate: float, the learning rate
:param beta: float, L2 regularization parameter for the weights
:param batch_norm_train: bool, train batch_normalization parameters
:param n_epochs: int, number of epochs to train the network
:param batch_size: int, batch size
:param dropout_keep_rate: float, 0.6 would drop 40% of weights
:param early_stopping_epochs: int, how many epochs to train without improvement (default=None)
:param early_stopping_method: string, 'dlr' for multiplying learning rate by 0.1 or 'stop'
:param early_stopping_iters: int, how many time to decrease learning rate (used if 'dlr' is True)
:param save_best_model: bool, whether or not to save the model with lowest loss
:param continue_fit, bool, do not reset the graph and continue with current weights
:return:
"""
X = np.array(X, np.float32)
if not continue_fit:
self._construct_nn(use_batch_norm, seperate_validation)
if seperate_validation:
X, val_X = train_test_split(X, test_size=validation_ratio, random_state=self._random_state)
assert len(X) > 0, "The training set is empty"
assert len(val_X) > 0, "The validation set is empty"
if batch_size is None:
batch_size = len(X)
train_inds = np.arange(len(X))
if not continue_fit:
self._num_batches_train = len(train_inds) // batch_size
b_w = 1. / self._num_batches_train
self._summary_op_step = int(pow(10, np.ceil(np.log10(self._num_batches_train))))
self._batch_step = int(np.floor(self._summary_op_step * b_w))
if seperate_validation:
val_inds = np.arange(len(val_X))
self._batch_size_val = len(val_inds) // self._num_batches_train
if not continue_fit:
train_vars = tf.trainable_variables()
l2_optimizable_vars = [i for i in train_vars if 'kernel' in i.name.split('/')[-1]]
self._train_vars = l2_optimizable_vars
with tf.name_scope('losses'):
self.define_loss(alpha=alpha, beta=beta)
self._summary_op = tf.summary.merge_all()
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
self._train_op = tf.train.AdamOptimizer(learning_rate).minimize(self._loss)
self._sess.run(tf.global_variables_initializer())
if not continue_fit:
self._start = time.time()
self.total_losses = []
self.best_loss = np.inf
sleep_time = 0.2
epochs_not_improved = 0
decreased_learning_rate = 0
for epoch in range(n_epochs):
if continue_fit:
self._epoch += 1
else:
self._epoch = epoch
# batch_accuracies = []
if self._verbose:
print('epoch %d started' % (self._epoch))
q = 0
cummulative_loss = 0
# for j in tqdm(range(self._num_batches_train)):
for j in range(self._num_batches_train):
train_inds, batch_inds = next_batch_shuffle(train_inds, batch_size)
batch_features = X[batch_inds]
_, train_summary, _loss = self._sess.run([self._train_op, self._summary_op, self._loss],
feed_dict={self._input_ph: batch_features,
self._labels: batch_features,
self._dropout_keep_rate: dropout_keep_rate,
self._train_mode: batch_norm_train})
if seperate_validation:
val_inds, batch_inds = next_batch(val_inds, self._batch_size_val)
batch_features = val_X[batch_inds]
assert len(batch_features) > 0, 'empty batch while validation'
val_summary, _loss = self._sess.run([self._summary_op, self._loss1],
feed_dict={self._input_ph: batch_features,
self._labels: batch_features})
self._val_writer.add_summary(val_summary,
self._epoch * self._summary_op_step + j * self._batch_step)
cummulative_loss += _loss
q += 1
self._train_writer.add_summary(train_summary,
self._epoch * self._summary_op_step + j * self._batch_step)
mean_loss = cummulative_loss / q
if mean_loss < self.best_loss:
self.best_loss = mean_loss
if save_best_model:
self.save_pb(self._ld + 'best.pb')
epochs_not_improved = 0
else:
epochs_not_improved += 1
if self._verbose:
print('%d epoch mean loss: %f' % (self._epoch, mean_loss))
self.total_losses.append(mean_loss)
if early_stopping_epochs is not None:
if epochs_not_improved > early_stopping_epochs:
if early_stopping_method == 'stop':
if self._verbose:
print('early stopping')
break
elif early_stopping_method == 'dlr':
if decreased_learning_rate > early_stopping_iters:
if self._verbose:
print('early stopping')
break
learning_rate /= 10
decreased_learning_rate += 1
epochs_not_improved = 0
if self._verbose:
print('new learning rate {}'.format(learning_rate))
else:
if self._verbose:
print('unknown method for early stopping. stopping the training.')
break
train_inds = np.arange(len(X))
if seperate_validation:
val_inds = np.arange(len(val_X))
time.sleep(sleep_time)
if self._verbose:
print('The training took {} seconds'.format(time.time() - self._start - self._epoch * sleep_time))
def train(self):
epochs = 120
# 准备运行训练步骤
section = '\n{0:=^40}\n'
print(section.format('开始训练'))
train_start = time.time()
for epoch in range(epochs): # 样本集迭代次数
epoch_start = time.time()
if epoch < self.startepo:
continue
print("第：", epoch, " 次迭代，一共要迭代 ", epochs, "次")
#######################run batch####
n_batches_epoch = int(np.ceil(len(self.text_labels) / batch_size))
print("在本次迭代中一共循环： ", n_batches_epoch, "每次取：", batch_size)
train_cost = 0
train_err = 0
next_idx = 0
for batch in range(n_batches_epoch): # 一次batch_size，取多少次
# 取数据
# temp_next_idx, temp_audio_features, temp_audio_features_len, temp_sparse_labels
next_idx, self.audio_features, self.audio_features_len, self.sparse_labels, wav_files = utils.next_batch(
next_idx,
batch_size,
n_input,
n_context,
self.text_labels,
self.wav_files,
self.word_num_map)
# 计算 avg_loss optimizer ;
batch_cost, _ = self.sess.run([self.avg_loss, self.optimizer], feed_dict=self.get_feed_dict())
train_cost += batch_cost
if (batch + 1) % 70 == 0:
rs = self.sess.run(self.merged, feed_dict=self.get_feed_dict())
self.writer.add_summary(rs, batch)
print('循环次数:', batch, '损失: ', train_cost / (batch + 1))
d, train_err = self.sess.run([self.decoded[0], self.label_err], feed_dict=self.get_feed_dict(dropout=1.0))
dense_decoded = tf.sparse_tensor_to_dense(d, default_value=-1).eval(session=self.sess)
dense_labels = utils.trans_tuple_to_texts_ch(self.sparse_labels, self.words)
print('错误率: ', train_err)
for orig, decoded_array in zip(dense_labels, dense_decoded):
# convert to strings
decoded_str = utils.trans_array_to_text_ch(decoded_array, self.words)
print('语音原始文本: {}'.format(orig))
print('识别出来的文本: {}'.format(decoded_str))
break
epoch_duration = time.time() - epoch_start
log = '迭代次数 {}/{}, 训练损失: {:.3f}, 错误率: {:.3f}, time: {:.2f} sec'
print(log.format(epoch, epochs, train_cost, train_err, epoch_duration))
self.saver.save(self.sess, self.savedir + self.conf.get("FILE_DATA").savefile, global_step=epoch)
train_duration = time.time() - train_start
print('Training complete, total duration: {:.2f} min'.format(train_duration / 60))
self.sess.close()
def test(self):
index = 0
next_idx = 20
for index in range(10):
next_idx, self.audio_features, self.audio_features_len, self.sparse_labels, wav_files = utils.next_batch(
next_idx,
1,
n_input,
n_context,
self.text_labels,
self.wav_files,
self.word_num_map)
print('读入语音文件: ', wav_files[0])
print('开始识别语音数据......')
d, train_ler = self.sess.run([self.decoded[0], self.label_err], feed_dict=self.get_feed_dict(dropout=1.0))
dense_decoded = tf.sparse_tensor_to_dense(d, default_value=-1).eval(session=self.sess)
dense_labels = utils.trans_tuple_to_texts_ch(self.sparse_labels, self.words)
for orig, decoded_array in zip(dense_labels, dense_decoded):
# 转成string
decoded_str = utils.trans_array_to_text_ch(decoded_array, self.words)
print('语音原始文本: {}'.format(orig))
print('识别出来的文本: {}'.format(decoded_str))
break
self.sess.close()
def train(args):
input_photo = tf.placeholder(
tf.float32, [args.batch_size, args.patch_size, args.patch_size, 3])
input_superpixel = tf.placeholder(
tf.float32, [args.batch_size, args.patch_size, args.patch_size, 3])
input_cartoon = tf.placeholder(
tf.float32, [args.batch_size, args.patch_size, args.patch_size, 3])
output = network.unet_generator(input_photo)
output = guided_filter(input_photo, output, r=1)
blur_fake = guided_filter(output, output, r=5, eps=2e-1)
blur_cartoon = guided_filter(input_cartoon, input_cartoon, r=5, eps=2e-1)
gray_fake, gray_cartoon = utils.color_shift(output, input_cartoon)
d_loss_gray, g_loss_gray = loss.lsgan_loss(network.disc_sn,
gray_cartoon,
gray_fake,
scale=1,
patch=True,
name='disc_gray')
d_loss_blur, g_loss_blur = loss.lsgan_loss(network.disc_sn,
blur_cartoon,
blur_fake,
scale=1,
patch=True,
name='disc_blur')
vgg_model = loss.Vgg19('weights/vgg19_no_fc.npy')
vgg_photo = vgg_model.build_conv4_4(input_photo)
vgg_output = vgg_model.build_conv4_4(output)
vgg_superpixel = vgg_model.build_conv4_4(input_superpixel)
h, w, c = vgg_photo.get_shape().as_list()[1:]
photo_loss = tf.reduce_mean(
tf.losses.absolute_difference(vgg_photo, vgg_output)) / (h * w * c)
superpixel_loss = tf.reduce_mean(tf.losses.absolute_difference\
(vgg_superpixel, vgg_output))/(h*w*c)
recon_loss = photo_loss + superpixel_loss
tv_loss = loss.total_variation_loss(output)
g_loss_total = 1e4 * tv_loss + 1e-1 * g_loss_blur + g_loss_gray + 2e2 * recon_loss
d_loss_total = d_loss_blur + d_loss_gray
all_vars = tf.trainable_variables()
gene_vars = [var for var in all_vars if 'gene' in var.name]
disc_vars = [var for var in all_vars if 'disc' in var.name]
tf.summary.scalar('tv_loss', tv_loss)
tf.summary.scalar('photo_loss', photo_loss)
tf.summary.scalar('superpixel_loss', superpixel_loss)
tf.summary.scalar('recon_loss', recon_loss)
tf.summary.scalar('d_loss_gray', d_loss_gray)
tf.summary.scalar('g_loss_gray', g_loss_gray)
tf.summary.scalar('d_loss_blur', d_loss_blur)
tf.summary.scalar('g_loss_blur', g_loss_blur)
tf.summary.scalar('d_loss_total', d_loss_total)
tf.summary.scalar('g_loss_total', g_loss_total)
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
g_optim = tf.train.AdamOptimizer(args.adv_train_lr, beta1=0.5, beta2=0.99)\
.minimize(g_loss_total, var_list=gene_vars)
d_optim = tf.train.AdamOptimizer(args.adv_train_lr, beta1=0.5, beta2=0.99)\
.minimize(d_loss_total, var_list=disc_vars)
'''
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
'''
gpu_options = tf.GPUOptions(
per_process_gpu_memory_fraction=args.gpu_fraction)
sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options))
train_writer = tf.summary.FileWriter(args.save_dir + '/train_log')
summary_op = tf.summary.merge_all()
saver = tf.train.Saver(var_list=gene_vars, max_to_keep=20)
with tf.device('/device:GPU:0'):
sess.run(tf.global_variables_initializer())
saver.restore(sess, tf.train.latest_checkpoint('pretrain/save_models'))
face_photo_dir = 'dataset/photo_face'
face_photo_list = utils.load_image_list(face_photo_dir)
scenery_photo_dir = 'dataset/photo_scenery'
scenery_photo_list = utils.load_image_list(scenery_photo_dir)
face_cartoon_dir = 'dataset/cartoon_face'
face_cartoon_list = utils.load_image_list(face_cartoon_dir)
scenery_cartoon_dir = 'dataset/cartoon_scenery'
scenery_cartoon_list = utils.load_image_list(scenery_cartoon_dir)
for total_iter in tqdm(range(args.total_iter)):
if np.mod(total_iter, 5) == 0:
photo_batch = utils.next_batch(face_photo_list,
args.batch_size)
cartoon_batch = utils.next_batch(face_cartoon_list,
args.batch_size)
else:
photo_batch = utils.next_batch(scenery_photo_list,
args.batch_size)
cartoon_batch = utils.next_batch(scenery_cartoon_list,
args.batch_size)
inter_out = sess.run(output,
feed_dict={
input_photo: photo_batch,
input_superpixel: photo_batch,
input_cartoon: cartoon_batch
})
'''
adaptive coloring has to be applied with the clip_by_value
in the last layer of generator network, which is not very stable.
to stabiliy reproduce our results, please use power=1.0
and comment the clip_by_value function in the network.py first
If this works, then try to use adaptive color with clip_by_value.
'''
if args.use_enhance:
superpixel_batch = utils.selective_adacolor(inter_out,
power=1.2)
else:
superpixel_batch = utils.simple_superpixel(inter_out,
seg_num=200)
_, g_loss, r_loss = sess.run(
[g_optim, g_loss_total, recon_loss],
feed_dict={
input_photo: photo_batch,
input_superpixel: superpixel_batch,
input_cartoon: cartoon_batch
})
_, d_loss, train_info = sess.run(
[d_optim, d_loss_total, summary_op],
feed_dict={
input_photo: photo_batch,
input_superpixel: superpixel_batch,
input_cartoon: cartoon_batch
})
train_writer.add_summary(train_info, total_iter)
if np.mod(total_iter + 1, 50) == 0:
print('Iter: {}, d_loss: {}, g_loss: {}, recon_loss: {}'.\
format(total_iter, d_loss, g_loss, r_loss))
wandb.log({
'd_loss': d_loss,
'g_loss': g_loss,
'r_loss': r_loss
})
if np.mod(total_iter + 1, 500) == 0:
saver.save(sess,
args.save_dir + '/saved_models/model',
write_meta_graph=False,
global_step=total_iter)
photo_face = utils.next_batch(face_photo_list,
args.batch_size)
cartoon_face = utils.next_batch(face_cartoon_list,
args.batch_size)
photo_scenery = utils.next_batch(scenery_photo_list,
args.batch_size)
cartoon_scenery = utils.next_batch(scenery_cartoon_list,
args.batch_size)
result_face = sess.run(output,
feed_dict={
input_photo: photo_face,
input_superpixel: photo_face,
input_cartoon: cartoon_face
})
result_scenery = sess.run(output,
feed_dict={
input_photo: photo_scenery,
input_superpixel:
photo_scenery,
input_cartoon:
cartoon_scenery
})
utils.write_batch_image(
result_face, args.save_dir + '/images',
str(total_iter) + '_face_result.jpg', 4)
utils.write_batch_image(
photo_face, args.save_dir + '/images',
str(total_iter) + '_face_photo.jpg', 4)
utils.write_batch_image(
result_scenery, args.save_dir + '/images',
str(total_iter) + '_scenery_result.jpg', 4)
utils.write_batch_image(
photo_scenery, args.save_dir + '/images',
str(total_iter) + '_scenery_photo.jpg', 4)
wandb.finish()
def train_next_loc(pre_model, dataset, batch_size, num_epoch, lr,
test_set_choice, early_stopping_round, device, **kwargs):
assert test_set_choice in [1, 2]
num_loc = len(dataset.poi_index_map)
pre_model = pre_model.to(device)
optimizer = torch.optim.Adam(pre_model.parameters(), lr=lr)
loss_func = nn.CrossEntropyLoss()
train_set = dataset.gen_split_session(set_choice=0, **kwargs)
eval_set = dataset.gen_split_session(set_choice=1, **kwargs)
test_set = eval_set if test_set_choice == 1 else dataset.gen_split_session(
set_choice=2, **kwargs)
def _pre_batch(input_model, batch):
""" Give prediction of one batch. """
user_seq, poi_sessions, time_sessions, label_seq, valid_num_session = zip(
*batch)
user_seq, label_seq = (torch.tensor(item).long().to(device)
for item in [user_seq, label_seq]
) # (batch_size)
bs = user_seq.size(0)
max_num_sessions = np.max(valid_num_session)
def _pad_session(s, fill_value):
# Each batch in poi_sessions contains a sequence of sessions, and one session contains a sub-sequence of location trajectory.
# Noted that the number of history sessions and the length of each session are variable.
flatten_session_batch = []
for session_i, session_batch in enumerate(s):
flatten_session_batch += (
[[]] * (max_num_sessions - valid_num_session[session_i]) +
session_batch
) # (batch_size * max_num_sessions, max_session_len)
filled_session_batches = np.array(
list(zip_longest(*flatten_session_batch,
fillvalue=fill_value))).transpose()
filled_session_batches = filled_session_batches.reshape(
(bs, max_num_sessions,
-1)) # (batch_size, max_num_sessions, max_seq_len)
return filled_session_batches
poi_sessions, time_sessions = _pad_session(poi_sessions, fill_value=num_loc), \
_pad_session(time_sessions, fill_value=24)
current_poi_s, history_poi_s = (
torch.tensor(item).long().to(device)
for item in [poi_sessions[:, -1, :], poi_sessions[:, :-1, :]])
current_time_s, history_time_s = (
torch.floor(torch.tensor(item)).long().to(device)
for item in [time_sessions[:, -1, :], time_sessions[:, :-1, :]])
pre_distribution = input_model(current_poi_s, current_time_s,
history_poi_s, history_time_s, user_seq)
return pre_distribution, label_seq
def _test_epoch(input_model, input_set):
""" Test model on the whole input set. """
input_model.eval()
pre_distributions, labels = [], []
for batch in next_batch(input_set, batch_size=256):
pre_distribution, label = _pre_batch(input_model, batch)
pre_distributions.append(pre_distribution.detach().cpu().numpy())
labels.append(label.detach().cpu().numpy())
pre_distributions, labels = (np.concatenate(item)
for item in (pre_distributions, labels))
pres = pre_distributions.argmax(-1)
return pre_distributions, pres, labels
max_metric = 0.0
worse_round = 0
for epoch in range(num_epoch):
for batch in next_batch(shuffle(train_set), batch_size):
pre_model.train()
pre_distribution, label = _pre_batch(pre_model, batch)
optimizer.zero_grad()
loss = loss_func(pre_distribution, label)
loss.backward()
optimizer.step()
pre_distributions, pres, labels = _test_epoch(pre_model, eval_set)
metric = accuracy_score(labels, pres)
nni.report_intermediate_result(metric)
if early_stopping_round > 0 and max_metric < metric:
max_metric = metric
worse_round = 0
torch.save(pre_model.state_dict(), NextLocParam.local_model_path)
else:
worse_round += 1
if 0 < early_stopping_round <= worse_round:
print('Early Stopping, best Epoch %d.' % (epoch - worse_round))
break
# Load the model with the best test metric value.
if early_stopping_round > 0:
pre_model.load_state_dict(torch.load(NextLocParam.local_model_path))
pre_distributions, pres, labels = _test_epoch(pre_model, test_set)
score_series = cal_classify_metric(pre_distributions, pres, labels,
NextLocParam.top_n_list)
print(score_series)
nni.report_final_result(score_series.loc['acc@1'])
os.remove(NextLocParam.local_model_path)
return score_series
def fit(self,
X,
seperate_validation=True,
validation_ratio=0.2,
learning_rate=0.01,
beta=0.0005,
n_epochs=10,
batch_size=16,
use_batch_norm=True,
batch_norm_train=True,
dropout_keep_rate=1.,
early_stopping_epochs=None,
early_stopping_method='dlr',
early_stopping_iters=2,
save_best_model=False,
continue_fit=False):
"""
:param X: {array-like, sparse matrix}, shape (n_samples, n_features)
Training vector, where n_samples is the number of samples and
n_features is the number of features.
Target vector relative to X.
:param seperate_validation: bool, seperate validation set from X
:param validation_ratio: float, between (0.0-1.0) the ratio of seperated validation (default=0.2)
:param learning_rate: float, the learning rate
:param beta: float, L2 regularization parameter for the weights
:param n_epochs: int, number of epochs to train the network
:param batch_size: int, batch size
:param use_batch_norm: bool, whether or not to use batch normalization
:param batch_norm_train: bool, train batch_normalization parameters
:param dropout_keep_rate: float, 0.6 would drop 40% of weights
:param early_stopping_epochs: int, how many epochs to train without improvement (default=None)
:param early_stopping_method: string, 'dlr' for multiplying learning rate by 0.1 or 'stop'
:param early_stopping_iters: int, how many time to decrease learning rate (used if 'dlr' is True)
:param save_best_model: bool, whether or not to save the model with lowest loss
:param continue_fit, bool, do not reset the graph and continue with current weights
:return:
"""
X = np.array(X, np.float32)
if not continue_fit:
self._construct_nn(use_batch_norm, seperate_validation)
if seperate_validation:
X, val_X = train_test_split(X,
test_size=validation_ratio,
random_state=self._random_state)
assert len(X) > 0, "The training set is empty"
assert len(val_X) > 0, "The validation set is empty"
if batch_size is None:
batch_size = len(X)
train_inds = np.arange(len(X))
if not continue_fit:
self._num_batches_train = len(train_inds) // batch_size
b_w = 1. / self._num_batches_train
self._summary_op_step = int(
pow(10, np.ceil(np.log10(self._num_batches_train))))
self._batch_step = int(np.floor(self._summary_op_step * b_w))
if seperate_validation:
val_inds = np.arange(len(val_X))
self._batch_size_val = len(val_inds) // self._num_batches_train
if not continue_fit:
train_vars = tf.trainable_variables()
l2_optimizable_vars = [
i for i in train_vars if 'kernel' in i.name.split('/')[-1]
]
self._train_vars = l2_optimizable_vars
with tf.name_scope('losses'):
self.define_loss(beta=beta)
self._summary_op = tf.compat.v1.summary.merge_all()
update_ops = tf.compat.v1.get_collection(
tf.compat.v1.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
self._train_op = tf.compat.v1.train.AdamOptimizer(
learning_rate).minimize(self._loss)
self._sess.run(tf.compat.v1.global_variables_initializer())
if not continue_fit:
self._start = time.time()
self.total_losses = []
self.best_loss = np.inf
sleep_time = 0.2
epochs_not_improved = 0
decreased_learning_rate = 0
for epoch in range(n_epochs):
if continue_fit:
self._epoch += 1
else:
self._epoch = epoch
if self._verbose:
print('epoch {} started'.format(self._epoch))
q = 0
cummulative_loss = 0
for j in range(self._num_batches_train):
train_inds, batch_inds = next_batch(train_inds,
batch_size,
shuffle=True)
batch_features = X[batch_inds]
_, train_summary, _loss = self._sess.run(
[self._train_op, self._summary_op, self._loss],
feed_dict={
self._input_ph: batch_features,
self._labels: batch_features,
self._dropout_keep_rate: dropout_keep_rate,
self._train_mode: batch_norm_train
})
if seperate_validation:
val_inds, batch_inds = next_batch(val_inds,
self._batch_size_val)
batch_features = val_X[batch_inds]
if len(batch_features) == 0:
raise AssertionError('empty batch while validation')
val_summary, _loss = self._sess.run(
[self._summary_op, self._loss1],
feed_dict={
self._input_ph: batch_features,
self._labels: batch_features
})
self._val_writer.add_summary(
val_summary, self._epoch * self._summary_op_step +
j * self._batch_step)
cummulative_loss += _loss
q += 1
self._train_writer.add_summary(
train_summary,
self._epoch * self._summary_op_step + j * self._batch_step)
mean_loss = cummulative_loss / q
if mean_loss < self.best_loss:
self.best_loss = mean_loss
if save_best_model:
self.save_model(path.join(self._ld, 'best.pb'))
epochs_not_improved = 0
else:
epochs_not_improved += 1
if self._verbose:
print('{} epoch mean loss: {}'.format(self._epoch, mean_loss))
self.total_losses.append(mean_loss)
if early_stopping_epochs is not None:
if epochs_not_improved > early_stopping_epochs:
if early_stopping_method == 'stop':
if self._verbose:
print('early stopping')
break
elif early_stopping_method == 'dlr':
if decreased_learning_rate > early_stopping_iters:
if self._verbose:
print('early stopping')
break
learning_rate /= 10
decreased_learning_rate += 1
epochs_not_improved = 0
if self._verbose:
print('new learning rate {}'.format(learning_rate))
else:
if self._verbose:
print(
'unknown method for early stopping. stopping the training.'
)
break
train_inds = np.arange(len(X))
if seperate_validation:
val_inds = np.arange(len(val_X))
time.sleep(sleep_time)
if self._verbose:
print('The training took {} seconds'.format(time.time() -
self._start -
self._epoch *
sleep_time))
opt = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(loss)
prediction = tf.argmax(logits, 1)
label = tf.argmax(tweet_label, 1)
#model evaluation
correct_prediction = tf.equal(tf.argmax(logits, 1), tf.argmax(tweet_label, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
#training loop
init = tf.global_variables_initializer()
with tf.Session() as sess:
sess.run(init)
batch_number = 0
iter = 1
while iter < 500:
x, y, seq, batch_number = next_batch(batch_number, batch_size, "train")
y = label_to_one_hot(y)
print(
sess.run(prediction,
feed_dict={
tweet_vec: x,
tweet_label: y,
seq_length: seq
}))
print(
sess.run(label,
feed_dict={
tweet_vec: x,
tweet_label: y,
seq_length: seq
}))
var_list=qgamma_variables,
optimizer=optimizer)
inference_beta0.initialize(scale={
x: scale_factor,
U: scale_factor,
gamma: scale_factor
},
var_list=qbeta0_variables,
optimizer=optimizer)
tf.global_variables_initializer().run()
loss = np.empty(inference_V.n_iter, dtype=np.float32)
for j in range(inference_V.n_iter):
x_batch, idx_batch = next_batch(train_data, M)
cau_batch = cau[idx_batch, :]
weights_batch = weights[idx_batch, :]
reconstr_cau_batch = reconstr_cau_train[idx_batch, :]
x_batch = x_batch.todense().astype('int')
cau_batch = cau_batch.todense()
weights_batch = weights_batch.todense()
sd_batch = 1. / np.sqrt(1 + weights_batch)
info_dict = inference_V.update(feed_dict={x_ph: x_batch, idx_ph: idx_batch, \
reconstr_cau_ph: reconstr_cau_batch, cau_ph: cau_batch, sd_ph: sd_batch})
inference_beta0.update(feed_dict={x_ph: x_batch, idx_ph: idx_batch, \
reconstr_cau_ph: reconstr_cau_batch, cau_ph: cau_batch, sd_ph: sd_batch})
inference_gamma.update(feed_dict={x_ph: x_batch, idx_ph: idx_batch, \
reconstr_cau_ph: reconstr_cau_batch, cau_ph: cau_batch, sd_ph: sd_batch})
def train_model(self):
if not os.path.exists(self.MODEL_NAME + '_result'):
os.mkdir(self.MODEL_NAME + '_result')
if not os.path.exists(self.LOGS_DIR): os.mkdir(self.LOGS_DIR)
if not os.path.exists(self.CKPT_DIR): os.mkdir(self.CKPT_DIR)
if not os.path.exists(self.OUTPUT_DIR): os.mkdir(self.OUTPUT_DIR)
train_set_path = read_data_path(self.TRAIN_IMAGE_PATH,
self.TRAIN_LABEL_PATH)
valid_set_path = read_data_path(self.VALID_IMAGE_PATH,
self.VALID_LABEL_PATH)
ckpt_save_path = os.path.join(
self.CKPT_DIR, self.MODEL_NAME + '_' + str(self.N_BATCH) + '_' +
str(self.LEARNING_RATE))
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
total_batch = int(len(train_set_path) / self.N_BATCH)
counter = 0
self.saver = tf.train.Saver()
self.writer = tf.summary.FileWriter(self.LOGS_DIR, sess.graph)
for epoch in tqdm(range(self.N_EPOCH)):
total_loss = 0
random.shuffle(train_set_path) # 매 epoch마다 데이터셋 shuffling
random.shuffle(valid_set_path)
for i in range(int(len(train_set_path) / self.N_BATCH)):
# print(i)
batch_xs_path, batch_ys_path = next_batch(
train_set_path, self.N_BATCH, i)
batch_xs = read_image(batch_xs_path,
[self.RESIZE, self.RESIZE])
batch_ys = read_annotation(batch_ys_path,
[self.RESIZE, self.RESIZE])
feed_dict = {
self.input_x: batch_xs,
self.label_y: batch_ys,
self.is_train: True
}
_, summary_str, loss = sess.run(
[self.optimizer, self.loss_summary, self.loss],
feed_dict=feed_dict)
self.writer.add_summary(summary_str, counter)
counter += 1
total_loss += loss
## validation 과정
valid_xs_path, valid_ys_path = next_batch(valid_set_path, 4, 0)
valid_xs = read_image(valid_xs_path,
[self.RESIZE, self.RESIZE])
valid_ys = read_annotation(valid_ys_path,
[self.RESIZE, self.RESIZE])
valid_pred = sess.run(self.pred,
feed_dict={
self.input_x: valid_xs,
self.label_y: valid_ys,
self.is_train: False
})
valid_pred = np.squeeze(valid_pred, axis=3)
valid_ys = np.squeeze(valid_ys, axis=3)
## plotting and save figure
img_save_path = self.OUTPUT_DIR + '/' + str(epoch).zfill(
3) + '.png'
draw_plot_segmentation(img_save_path, valid_xs, valid_pred,
valid_ys)
print('\nEpoch:', '%03d' % (epoch + 1),
'Avg Loss: {:.6}\t'.format(total_loss / total_batch))
self.saver.save(sess,
ckpt_save_path + '_' + str(epoch) + '.model',
global_step=counter)
self.saver.save(sess,
ckpt_save_path + '_' + str(epoch) + '.model',
global_step=counter)
print('Finish save model')
var_list=qgamma_variables,
optimizer=optimizer)
inference_beta0.initialize(scale={
x: scale_factor,
U: scale_factor,
gamma: scale_factor
},
var_list=qbeta0_variables,
optimizer=optimizer)
tf.global_variables_initializer().run()
loss = np.empty(inference_V.n_iter, dtype=np.float32)
for j in range(inference_V.n_iter):
x_batch, idx_batch = next_batch(train_data, M)
cau_batch = cau[idx_batch, :]
weights_batch = weights[idx_batch, :]
reconstr_cau_batch = reconstr_cau[idx_batch, :]
x_batch = x_batch.todense().astype('int')
cau_batch = cau_batch.todense()
weights_batch = weights_batch.todense()
sd_batch = 1. / np.sqrt(1 + weights_batch)
info_dict = inference_V.update(feed_dict={x_ph: x_batch, idx_ph: idx_batch, \
reconstr_cau_ph: reconstr_cau_batch, cau_ph: cau_batch, sd_ph: sd_batch})
inference_beta0.update(feed_dict={x_ph: x_batch, idx_ph: idx_batch, \
reconstr_cau_ph: reconstr_cau_batch, cau_ph: cau_batch, sd_ph: sd_batch})
inference_gamma.update(feed_dict={x_ph: x_batch, idx_ph: idx_batch, \
reconstr_cau_ph: reconstr_cau_batch, cau_ph: cau_batch, sd_ph: sd_batch})
train_iters = int(len(train_labels) / batch_size)
else:
train_iters = int(len(train_labels) / batch_size) + 1
print("Optimization starting..")
with tf.Session() as sess:
sess.run(init)
epoch_loss_list = []
epoch_index_list = []
for epoch in range(epochs):
epoch_loss = 0
train_inputs, train_labels = utils.suffle_data(train_inputs,
train_labels)
for i in range(train_iters):
batch_x = utils.next_batch(train_inputs, i, batch_size)
batch_y = utils.next_batch(train_labels, i, batch_size)
batch_x = batch_x.reshape((-1, TIME_SERIES_LENGTH, inputs_num))
batch_y = batch_y.reshape((-1, output_neurons))
_, c = sess.run([optimizer, loss],
feed_dict={
xplaceholder: batch_x,
yplaceholder: batch_y
})
epoch_loss += c / train_iters
#if(i==0 or i==train_iters-1):
#print(loaded_minmax_scaler.inverse_transform(sess.run(logits, feed_dict={xplaceholder: batch_x, yplaceholder: batch_y}).reshape(-1,1)).reshape(1,-1)[0])
epoch_loss_list.append(epoch_loss)
epoch_index_list.append(epoch + 1)
print('Epoch', epoch + 1, 'completed out of', epochs, 'loss:',
epoch_loss)
if pretrain:
print("Starting autoencoder pretraining...")
# Variables to save pretraining tensor content
embeddings = np.zeros((specs.n_samples, specs.embedding_size),
dtype=float)
# First, pretrain the autoencoder
## Loop over epochs
for epoch in range(n_pretrain_epochs):
print("Pretraining step: epoch {}".format(epoch))
# Loop over the samples
for _ in range(n_batches):
# Fetch a random data batch of the specified size
indices, batch_data, sec_data, masked_data = next_batch(
batch_size, data, data_sec, data_masked)
_, embedding_, ae_loss_, keywords_embedding_ = sess.run(
(cg.pretrain_op, cg.embedding, cg.ae_loss,
cg.keywords_embedding),
feed_dict={
cg.input: batch_data,
cg.data_masked: masked_data,
cg.data_second: sec_data
})
# Save the embeddings for batch samples
for j in range(len(indices)):
embeddings[indices[j], :] = embedding_[j, :]
#print("ae_loss_:", float(ae_loss_))
# Second, run k-means++ on the pretrained embeddings