def predict(self, X, batch_size=None): """ :param X: {array-like, sparse matrix}, shape (n_samples, n_features) Training vector, where n_samples is the number of samples and n_features is the number of features. :param batch_size: int, batch size :return: decoded predicted encodings """ X = np.array(X) inds = np.arange(len(X)) predictions = [] batch_size = batch_size or len(X) start = time.time() while len(inds) > 0: inds, batch_inds = next_batch(inds, batch_size) batch_features = X[batch_inds] batch_preds = self._sess.run(self._output, feed_dict={self._input_ph: batch_features}) predictions.extend(batch_preds) if self._verbose: print('The inference took {} seconds'.format(time.time() - start)) predictions = np.squeeze(predictions) return predictions
def score(self, X, batch_size=None): """ :param X: {array-like, sparse matrix}, shape (n_samples, n_features) Training vector, where n_samples is the number of samples and n_features is the number of features. :param y: array-like, shape (n_samples,) Target vector relative to X. :param batch_size: int, batch size :return: mean l2 distance between input and output """ X = np.array(X) inds = np.arange(len(X)) predictions = [] batch_size = batch_size or len(X) while len(inds) > 0: inds, batch_inds = next_batch(inds, batch_size) batch_features = X[batch_inds] batch_preds = self._sess.run(self._output, feed_dict={self._input_ph: batch_features}) predictions.extend(batch_preds) predictions = np.array(predictions) return np.linalg.norm(X - predictions, axis=-1).mean()
y = tf.nn.softmax(tf.matmul(x, W) + b) y_ = tf.placeholder(tf.float32, [None, cuisine_count]) t = tf.placeholder(tf.float32, [None, ingredients_count]) p = tf.nn.softmax(tf.matmul(t, W) + b) cross_entropy = -tf.reduce_sum(y_*tf.log(y)) train_step = tf.train.GradientDescentOptimizer(0.001).minimize(cross_entropy) # train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy) init = tf.initialize_all_variables() sess = tf.Session(config=tf.ConfigProto(allow_soft_placement=True)) sess.run(init) for d in ['/cpu:0', '/gpu:0']: with tf.device(d): for i in range(100): batch_xs, batch_ys = utl.next_batch(xs, ys, 12000) sess.run(train_step, feed_dict={x: batch_xs, y_: batch_ys}) correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1)) accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) print(sess.run(accuracy, feed_dict={x: xs, y_: ys})) cls = sess.run(p, feed_dict={t: ts}) utl.save_result('tf_gd', cuisine_list, np.argmax(cls, axis=1).tolist(), ids, 'number')
def fit(self, X, seperate_validation=True, validation_ratio=0.2, learning_rate=0.01, alpha=1., beta=0.0005, n_epochs=10, batch_size=16, use_batch_norm=True, batch_norm_train=True, dropout_keep_rate=1., early_stopping_epochs=None, early_stopping_method='dlr', early_stopping_iters=2, save_best_model=False, continue_fit=False): """ :param X: {array-like, sparse matrix}, shape (n_samples, n_features) Training vector, where n_samples is the number of samples and n_features is the number of features. :param y: array-like, shape (n_samples,) Target vector relative to X. :param loss_type: string, 'rmse' or 'crossentropy' :param seperate_validation: bool, seperate validation set from X :param validation_ratio: float, between (0.0-1.0) the ratio of seperated validation (default=0.2) :param learning_rate: float, the learning rate :param beta: float, L2 regularization parameter for the weights :param batch_norm_train: bool, train batch_normalization parameters :param n_epochs: int, number of epochs to train the network :param batch_size: int, batch size :param dropout_keep_rate: float, 0.6 would drop 40% of weights :param early_stopping_epochs: int, how many epochs to train without improvement (default=None) :param early_stopping_method: string, 'dlr' for multiplying learning rate by 0.1 or 'stop' :param early_stopping_iters: int, how many time to decrease learning rate (used if 'dlr' is True) :param save_best_model: bool, whether or not to save the model with lowest loss :param continue_fit, bool, do not reset the graph and continue with current weights :return: """ X = np.array(X, np.float32) if not continue_fit: self._construct_nn(use_batch_norm, seperate_validation) if seperate_validation: X, val_X = train_test_split(X, test_size=validation_ratio, random_state=self._random_state) assert len(X) > 0, "The training set is empty" assert len(val_X) > 0, "The validation set is empty" if batch_size is None: batch_size = len(X) train_inds = np.arange(len(X)) if not continue_fit: self._num_batches_train = len(train_inds) // batch_size b_w = 1. / self._num_batches_train self._summary_op_step = int(pow(10, np.ceil(np.log10(self._num_batches_train)))) self._batch_step = int(np.floor(self._summary_op_step * b_w)) if seperate_validation: val_inds = np.arange(len(val_X)) self._batch_size_val = len(val_inds) // self._num_batches_train if not continue_fit: train_vars = tf.trainable_variables() l2_optimizable_vars = [i for i in train_vars if 'kernel' in i.name.split('/')[-1]] self._train_vars = l2_optimizable_vars with tf.name_scope('losses'): self.define_loss(alpha=alpha, beta=beta) self._summary_op = tf.summary.merge_all() update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) with tf.control_dependencies(update_ops): self._train_op = tf.train.AdamOptimizer(learning_rate).minimize(self._loss) self._sess.run(tf.global_variables_initializer()) if not continue_fit: self._start = time.time() self.total_losses = [] self.best_loss = np.inf sleep_time = 0.2 epochs_not_improved = 0 decreased_learning_rate = 0 for epoch in range(n_epochs): if continue_fit: self._epoch += 1 else: self._epoch = epoch # batch_accuracies = [] if self._verbose: print('epoch %d started' % (self._epoch)) q = 0 cummulative_loss = 0 # for j in tqdm(range(self._num_batches_train)): for j in range(self._num_batches_train): train_inds, batch_inds = next_batch_shuffle(train_inds, batch_size) batch_features = X[batch_inds] _, train_summary, _loss = self._sess.run([self._train_op, self._summary_op, self._loss], feed_dict={self._input_ph: batch_features, self._labels: batch_features, self._dropout_keep_rate: dropout_keep_rate, self._train_mode: batch_norm_train}) if seperate_validation: val_inds, batch_inds = next_batch(val_inds, self._batch_size_val) batch_features = val_X[batch_inds] assert len(batch_features) > 0, 'empty batch while validation' val_summary, _loss = self._sess.run([self._summary_op, self._loss1], feed_dict={self._input_ph: batch_features, self._labels: batch_features}) self._val_writer.add_summary(val_summary, self._epoch * self._summary_op_step + j * self._batch_step) cummulative_loss += _loss q += 1 self._train_writer.add_summary(train_summary, self._epoch * self._summary_op_step + j * self._batch_step) mean_loss = cummulative_loss / q if mean_loss < self.best_loss: self.best_loss = mean_loss if save_best_model: self.save_pb(self._ld + 'best.pb') epochs_not_improved = 0 else: epochs_not_improved += 1 if self._verbose: print('%d epoch mean loss: %f' % (self._epoch, mean_loss)) self.total_losses.append(mean_loss) if early_stopping_epochs is not None: if epochs_not_improved > early_stopping_epochs: if early_stopping_method == 'stop': if self._verbose: print('early stopping') break elif early_stopping_method == 'dlr': if decreased_learning_rate > early_stopping_iters: if self._verbose: print('early stopping') break learning_rate /= 10 decreased_learning_rate += 1 epochs_not_improved = 0 if self._verbose: print('new learning rate {}'.format(learning_rate)) else: if self._verbose: print('unknown method for early stopping. stopping the training.') break train_inds = np.arange(len(X)) if seperate_validation: val_inds = np.arange(len(val_X)) time.sleep(sleep_time) if self._verbose: print('The training took {} seconds'.format(time.time() - self._start - self._epoch * sleep_time))
def train(self): epochs = 120 # 准备运行训练步骤 section = '\n{0:=^40}\n' print(section.format('开始训练')) train_start = time.time() for epoch in range(epochs): # 样本集迭代次数 epoch_start = time.time() if epoch < self.startepo: continue print("第:", epoch, " 次迭代,一共要迭代 ", epochs, "次") #######################run batch#### n_batches_epoch = int(np.ceil(len(self.text_labels) / batch_size)) print("在本次迭代中一共循环: ", n_batches_epoch, "每次取:", batch_size) train_cost = 0 train_err = 0 next_idx = 0 for batch in range(n_batches_epoch): # 一次batch_size,取多少次 # 取数据 # temp_next_idx, temp_audio_features, temp_audio_features_len, temp_sparse_labels next_idx, self.audio_features, self.audio_features_len, self.sparse_labels, wav_files = utils.next_batch( next_idx, batch_size, n_input, n_context, self.text_labels, self.wav_files, self.word_num_map) # 计算 avg_loss optimizer ; batch_cost, _ = self.sess.run([self.avg_loss, self.optimizer], feed_dict=self.get_feed_dict()) train_cost += batch_cost if (batch + 1) % 70 == 0: rs = self.sess.run(self.merged, feed_dict=self.get_feed_dict()) self.writer.add_summary(rs, batch) print('循环次数:', batch, '损失: ', train_cost / (batch + 1)) d, train_err = self.sess.run([self.decoded[0], self.label_err], feed_dict=self.get_feed_dict(dropout=1.0)) dense_decoded = tf.sparse_tensor_to_dense(d, default_value=-1).eval(session=self.sess) dense_labels = utils.trans_tuple_to_texts_ch(self.sparse_labels, self.words) print('错误率: ', train_err) for orig, decoded_array in zip(dense_labels, dense_decoded): # convert to strings decoded_str = utils.trans_array_to_text_ch(decoded_array, self.words) print('语音原始文本: {}'.format(orig)) print('识别出来的文本: {}'.format(decoded_str)) break epoch_duration = time.time() - epoch_start log = '迭代次数 {}/{}, 训练损失: {:.3f}, 错误率: {:.3f}, time: {:.2f} sec' print(log.format(epoch, epochs, train_cost, train_err, epoch_duration)) self.saver.save(self.sess, self.savedir + self.conf.get("FILE_DATA").savefile, global_step=epoch) train_duration = time.time() - train_start print('Training complete, total duration: {:.2f} min'.format(train_duration / 60)) self.sess.close()
def test(self): index = 0 next_idx = 20 for index in range(10): next_idx, self.audio_features, self.audio_features_len, self.sparse_labels, wav_files = utils.next_batch( next_idx, 1, n_input, n_context, self.text_labels, self.wav_files, self.word_num_map) print('读入语音文件: ', wav_files[0]) print('开始识别语音数据......') d, train_ler = self.sess.run([self.decoded[0], self.label_err], feed_dict=self.get_feed_dict(dropout=1.0)) dense_decoded = tf.sparse_tensor_to_dense(d, default_value=-1).eval(session=self.sess) dense_labels = utils.trans_tuple_to_texts_ch(self.sparse_labels, self.words) for orig, decoded_array in zip(dense_labels, dense_decoded): # 转成string decoded_str = utils.trans_array_to_text_ch(decoded_array, self.words) print('语音原始文本: {}'.format(orig)) print('识别出来的文本: {}'.format(decoded_str)) break self.sess.close()
def train(args): input_photo = tf.placeholder( tf.float32, [args.batch_size, args.patch_size, args.patch_size, 3]) input_superpixel = tf.placeholder( tf.float32, [args.batch_size, args.patch_size, args.patch_size, 3]) input_cartoon = tf.placeholder( tf.float32, [args.batch_size, args.patch_size, args.patch_size, 3]) output = network.unet_generator(input_photo) output = guided_filter(input_photo, output, r=1) blur_fake = guided_filter(output, output, r=5, eps=2e-1) blur_cartoon = guided_filter(input_cartoon, input_cartoon, r=5, eps=2e-1) gray_fake, gray_cartoon = utils.color_shift(output, input_cartoon) d_loss_gray, g_loss_gray = loss.lsgan_loss(network.disc_sn, gray_cartoon, gray_fake, scale=1, patch=True, name='disc_gray') d_loss_blur, g_loss_blur = loss.lsgan_loss(network.disc_sn, blur_cartoon, blur_fake, scale=1, patch=True, name='disc_blur') vgg_model = loss.Vgg19('weights/vgg19_no_fc.npy') vgg_photo = vgg_model.build_conv4_4(input_photo) vgg_output = vgg_model.build_conv4_4(output) vgg_superpixel = vgg_model.build_conv4_4(input_superpixel) h, w, c = vgg_photo.get_shape().as_list()[1:] photo_loss = tf.reduce_mean( tf.losses.absolute_difference(vgg_photo, vgg_output)) / (h * w * c) superpixel_loss = tf.reduce_mean(tf.losses.absolute_difference\ (vgg_superpixel, vgg_output))/(h*w*c) recon_loss = photo_loss + superpixel_loss tv_loss = loss.total_variation_loss(output) g_loss_total = 1e4 * tv_loss + 1e-1 * g_loss_blur + g_loss_gray + 2e2 * recon_loss d_loss_total = d_loss_blur + d_loss_gray all_vars = tf.trainable_variables() gene_vars = [var for var in all_vars if 'gene' in var.name] disc_vars = [var for var in all_vars if 'disc' in var.name] tf.summary.scalar('tv_loss', tv_loss) tf.summary.scalar('photo_loss', photo_loss) tf.summary.scalar('superpixel_loss', superpixel_loss) tf.summary.scalar('recon_loss', recon_loss) tf.summary.scalar('d_loss_gray', d_loss_gray) tf.summary.scalar('g_loss_gray', g_loss_gray) tf.summary.scalar('d_loss_blur', d_loss_blur) tf.summary.scalar('g_loss_blur', g_loss_blur) tf.summary.scalar('d_loss_total', d_loss_total) tf.summary.scalar('g_loss_total', g_loss_total) update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) with tf.control_dependencies(update_ops): g_optim = tf.train.AdamOptimizer(args.adv_train_lr, beta1=0.5, beta2=0.99)\ .minimize(g_loss_total, var_list=gene_vars) d_optim = tf.train.AdamOptimizer(args.adv_train_lr, beta1=0.5, beta2=0.99)\ .minimize(d_loss_total, var_list=disc_vars) ''' config = tf.ConfigProto() config.gpu_options.allow_growth = True sess = tf.Session(config=config) ''' gpu_options = tf.GPUOptions( per_process_gpu_memory_fraction=args.gpu_fraction) sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) train_writer = tf.summary.FileWriter(args.save_dir + '/train_log') summary_op = tf.summary.merge_all() saver = tf.train.Saver(var_list=gene_vars, max_to_keep=20) with tf.device('/device:GPU:0'): sess.run(tf.global_variables_initializer()) saver.restore(sess, tf.train.latest_checkpoint('pretrain/save_models')) face_photo_dir = 'dataset/photo_face' face_photo_list = utils.load_image_list(face_photo_dir) scenery_photo_dir = 'dataset/photo_scenery' scenery_photo_list = utils.load_image_list(scenery_photo_dir) face_cartoon_dir = 'dataset/cartoon_face' face_cartoon_list = utils.load_image_list(face_cartoon_dir) scenery_cartoon_dir = 'dataset/cartoon_scenery' scenery_cartoon_list = utils.load_image_list(scenery_cartoon_dir) for total_iter in tqdm(range(args.total_iter)): if np.mod(total_iter, 5) == 0: photo_batch = utils.next_batch(face_photo_list, args.batch_size) cartoon_batch = utils.next_batch(face_cartoon_list, args.batch_size) else: photo_batch = utils.next_batch(scenery_photo_list, args.batch_size) cartoon_batch = utils.next_batch(scenery_cartoon_list, args.batch_size) inter_out = sess.run(output, feed_dict={ input_photo: photo_batch, input_superpixel: photo_batch, input_cartoon: cartoon_batch }) ''' adaptive coloring has to be applied with the clip_by_value in the last layer of generator network, which is not very stable. to stabiliy reproduce our results, please use power=1.0 and comment the clip_by_value function in the network.py first If this works, then try to use adaptive color with clip_by_value. ''' if args.use_enhance: superpixel_batch = utils.selective_adacolor(inter_out, power=1.2) else: superpixel_batch = utils.simple_superpixel(inter_out, seg_num=200) _, g_loss, r_loss = sess.run( [g_optim, g_loss_total, recon_loss], feed_dict={ input_photo: photo_batch, input_superpixel: superpixel_batch, input_cartoon: cartoon_batch }) _, d_loss, train_info = sess.run( [d_optim, d_loss_total, summary_op], feed_dict={ input_photo: photo_batch, input_superpixel: superpixel_batch, input_cartoon: cartoon_batch }) train_writer.add_summary(train_info, total_iter) if np.mod(total_iter + 1, 50) == 0: print('Iter: {}, d_loss: {}, g_loss: {}, recon_loss: {}'.\ format(total_iter, d_loss, g_loss, r_loss)) wandb.log({ 'd_loss': d_loss, 'g_loss': g_loss, 'r_loss': r_loss }) if np.mod(total_iter + 1, 500) == 0: saver.save(sess, args.save_dir + '/saved_models/model', write_meta_graph=False, global_step=total_iter) photo_face = utils.next_batch(face_photo_list, args.batch_size) cartoon_face = utils.next_batch(face_cartoon_list, args.batch_size) photo_scenery = utils.next_batch(scenery_photo_list, args.batch_size) cartoon_scenery = utils.next_batch(scenery_cartoon_list, args.batch_size) result_face = sess.run(output, feed_dict={ input_photo: photo_face, input_superpixel: photo_face, input_cartoon: cartoon_face }) result_scenery = sess.run(output, feed_dict={ input_photo: photo_scenery, input_superpixel: photo_scenery, input_cartoon: cartoon_scenery }) utils.write_batch_image( result_face, args.save_dir + '/images', str(total_iter) + '_face_result.jpg', 4) utils.write_batch_image( photo_face, args.save_dir + '/images', str(total_iter) + '_face_photo.jpg', 4) utils.write_batch_image( result_scenery, args.save_dir + '/images', str(total_iter) + '_scenery_result.jpg', 4) utils.write_batch_image( photo_scenery, args.save_dir + '/images', str(total_iter) + '_scenery_photo.jpg', 4) wandb.finish()
def train_next_loc(pre_model, dataset, batch_size, num_epoch, lr, test_set_choice, early_stopping_round, device, **kwargs): assert test_set_choice in [1, 2] num_loc = len(dataset.poi_index_map) pre_model = pre_model.to(device) optimizer = torch.optim.Adam(pre_model.parameters(), lr=lr) loss_func = nn.CrossEntropyLoss() train_set = dataset.gen_split_session(set_choice=0, **kwargs) eval_set = dataset.gen_split_session(set_choice=1, **kwargs) test_set = eval_set if test_set_choice == 1 else dataset.gen_split_session( set_choice=2, **kwargs) def _pre_batch(input_model, batch): """ Give prediction of one batch. """ user_seq, poi_sessions, time_sessions, label_seq, valid_num_session = zip( *batch) user_seq, label_seq = (torch.tensor(item).long().to(device) for item in [user_seq, label_seq] ) # (batch_size) bs = user_seq.size(0) max_num_sessions = np.max(valid_num_session) def _pad_session(s, fill_value): # Each batch in poi_sessions contains a sequence of sessions, and one session contains a sub-sequence of location trajectory. # Noted that the number of history sessions and the length of each session are variable. flatten_session_batch = [] for session_i, session_batch in enumerate(s): flatten_session_batch += ( [[]] * (max_num_sessions - valid_num_session[session_i]) + session_batch ) # (batch_size * max_num_sessions, max_session_len) filled_session_batches = np.array( list(zip_longest(*flatten_session_batch, fillvalue=fill_value))).transpose() filled_session_batches = filled_session_batches.reshape( (bs, max_num_sessions, -1)) # (batch_size, max_num_sessions, max_seq_len) return filled_session_batches poi_sessions, time_sessions = _pad_session(poi_sessions, fill_value=num_loc), \ _pad_session(time_sessions, fill_value=24) current_poi_s, history_poi_s = ( torch.tensor(item).long().to(device) for item in [poi_sessions[:, -1, :], poi_sessions[:, :-1, :]]) current_time_s, history_time_s = ( torch.floor(torch.tensor(item)).long().to(device) for item in [time_sessions[:, -1, :], time_sessions[:, :-1, :]]) pre_distribution = input_model(current_poi_s, current_time_s, history_poi_s, history_time_s, user_seq) return pre_distribution, label_seq def _test_epoch(input_model, input_set): """ Test model on the whole input set. """ input_model.eval() pre_distributions, labels = [], [] for batch in next_batch(input_set, batch_size=256): pre_distribution, label = _pre_batch(input_model, batch) pre_distributions.append(pre_distribution.detach().cpu().numpy()) labels.append(label.detach().cpu().numpy()) pre_distributions, labels = (np.concatenate(item) for item in (pre_distributions, labels)) pres = pre_distributions.argmax(-1) return pre_distributions, pres, labels max_metric = 0.0 worse_round = 0 for epoch in range(num_epoch): for batch in next_batch(shuffle(train_set), batch_size): pre_model.train() pre_distribution, label = _pre_batch(pre_model, batch) optimizer.zero_grad() loss = loss_func(pre_distribution, label) loss.backward() optimizer.step() pre_distributions, pres, labels = _test_epoch(pre_model, eval_set) metric = accuracy_score(labels, pres) nni.report_intermediate_result(metric) if early_stopping_round > 0 and max_metric < metric: max_metric = metric worse_round = 0 torch.save(pre_model.state_dict(), NextLocParam.local_model_path) else: worse_round += 1 if 0 < early_stopping_round <= worse_round: print('Early Stopping, best Epoch %d.' % (epoch - worse_round)) break # Load the model with the best test metric value. if early_stopping_round > 0: pre_model.load_state_dict(torch.load(NextLocParam.local_model_path)) pre_distributions, pres, labels = _test_epoch(pre_model, test_set) score_series = cal_classify_metric(pre_distributions, pres, labels, NextLocParam.top_n_list) print(score_series) nni.report_final_result(score_series.loc['acc@1']) os.remove(NextLocParam.local_model_path) return score_series
def fit(self, X, seperate_validation=True, validation_ratio=0.2, learning_rate=0.01, beta=0.0005, n_epochs=10, batch_size=16, use_batch_norm=True, batch_norm_train=True, dropout_keep_rate=1., early_stopping_epochs=None, early_stopping_method='dlr', early_stopping_iters=2, save_best_model=False, continue_fit=False): """ :param X: {array-like, sparse matrix}, shape (n_samples, n_features) Training vector, where n_samples is the number of samples and n_features is the number of features. Target vector relative to X. :param seperate_validation: bool, seperate validation set from X :param validation_ratio: float, between (0.0-1.0) the ratio of seperated validation (default=0.2) :param learning_rate: float, the learning rate :param beta: float, L2 regularization parameter for the weights :param n_epochs: int, number of epochs to train the network :param batch_size: int, batch size :param use_batch_norm: bool, whether or not to use batch normalization :param batch_norm_train: bool, train batch_normalization parameters :param dropout_keep_rate: float, 0.6 would drop 40% of weights :param early_stopping_epochs: int, how many epochs to train without improvement (default=None) :param early_stopping_method: string, 'dlr' for multiplying learning rate by 0.1 or 'stop' :param early_stopping_iters: int, how many time to decrease learning rate (used if 'dlr' is True) :param save_best_model: bool, whether or not to save the model with lowest loss :param continue_fit, bool, do not reset the graph and continue with current weights :return: """ X = np.array(X, np.float32) if not continue_fit: self._construct_nn(use_batch_norm, seperate_validation) if seperate_validation: X, val_X = train_test_split(X, test_size=validation_ratio, random_state=self._random_state) assert len(X) > 0, "The training set is empty" assert len(val_X) > 0, "The validation set is empty" if batch_size is None: batch_size = len(X) train_inds = np.arange(len(X)) if not continue_fit: self._num_batches_train = len(train_inds) // batch_size b_w = 1. / self._num_batches_train self._summary_op_step = int( pow(10, np.ceil(np.log10(self._num_batches_train)))) self._batch_step = int(np.floor(self._summary_op_step * b_w)) if seperate_validation: val_inds = np.arange(len(val_X)) self._batch_size_val = len(val_inds) // self._num_batches_train if not continue_fit: train_vars = tf.trainable_variables() l2_optimizable_vars = [ i for i in train_vars if 'kernel' in i.name.split('/')[-1] ] self._train_vars = l2_optimizable_vars with tf.name_scope('losses'): self.define_loss(beta=beta) self._summary_op = tf.compat.v1.summary.merge_all() update_ops = tf.compat.v1.get_collection( tf.compat.v1.GraphKeys.UPDATE_OPS) with tf.control_dependencies(update_ops): self._train_op = tf.compat.v1.train.AdamOptimizer( learning_rate).minimize(self._loss) self._sess.run(tf.compat.v1.global_variables_initializer()) if not continue_fit: self._start = time.time() self.total_losses = [] self.best_loss = np.inf sleep_time = 0.2 epochs_not_improved = 0 decreased_learning_rate = 0 for epoch in range(n_epochs): if continue_fit: self._epoch += 1 else: self._epoch = epoch if self._verbose: print('epoch {} started'.format(self._epoch)) q = 0 cummulative_loss = 0 for j in range(self._num_batches_train): train_inds, batch_inds = next_batch(train_inds, batch_size, shuffle=True) batch_features = X[batch_inds] _, train_summary, _loss = self._sess.run( [self._train_op, self._summary_op, self._loss], feed_dict={ self._input_ph: batch_features, self._labels: batch_features, self._dropout_keep_rate: dropout_keep_rate, self._train_mode: batch_norm_train }) if seperate_validation: val_inds, batch_inds = next_batch(val_inds, self._batch_size_val) batch_features = val_X[batch_inds] if len(batch_features) == 0: raise AssertionError('empty batch while validation') val_summary, _loss = self._sess.run( [self._summary_op, self._loss1], feed_dict={ self._input_ph: batch_features, self._labels: batch_features }) self._val_writer.add_summary( val_summary, self._epoch * self._summary_op_step + j * self._batch_step) cummulative_loss += _loss q += 1 self._train_writer.add_summary( train_summary, self._epoch * self._summary_op_step + j * self._batch_step) mean_loss = cummulative_loss / q if mean_loss < self.best_loss: self.best_loss = mean_loss if save_best_model: self.save_model(path.join(self._ld, 'best.pb')) epochs_not_improved = 0 else: epochs_not_improved += 1 if self._verbose: print('{} epoch mean loss: {}'.format(self._epoch, mean_loss)) self.total_losses.append(mean_loss) if early_stopping_epochs is not None: if epochs_not_improved > early_stopping_epochs: if early_stopping_method == 'stop': if self._verbose: print('early stopping') break elif early_stopping_method == 'dlr': if decreased_learning_rate > early_stopping_iters: if self._verbose: print('early stopping') break learning_rate /= 10 decreased_learning_rate += 1 epochs_not_improved = 0 if self._verbose: print('new learning rate {}'.format(learning_rate)) else: if self._verbose: print( 'unknown method for early stopping. stopping the training.' ) break train_inds = np.arange(len(X)) if seperate_validation: val_inds = np.arange(len(val_X)) time.sleep(sleep_time) if self._verbose: print('The training took {} seconds'.format(time.time() - self._start - self._epoch * sleep_time))
opt = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(loss) prediction = tf.argmax(logits, 1) label = tf.argmax(tweet_label, 1) #model evaluation correct_prediction = tf.equal(tf.argmax(logits, 1), tf.argmax(tweet_label, 1)) accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) #training loop init = tf.global_variables_initializer() with tf.Session() as sess: sess.run(init) batch_number = 0 iter = 1 while iter < 500: x, y, seq, batch_number = next_batch(batch_number, batch_size, "train") y = label_to_one_hot(y) print( sess.run(prediction, feed_dict={ tweet_vec: x, tweet_label: y, seq_length: seq })) print( sess.run(label, feed_dict={ tweet_vec: x, tweet_label: y, seq_length: seq }))
var_list=qgamma_variables, optimizer=optimizer) inference_beta0.initialize(scale={ x: scale_factor, U: scale_factor, gamma: scale_factor }, var_list=qbeta0_variables, optimizer=optimizer) tf.global_variables_initializer().run() loss = np.empty(inference_V.n_iter, dtype=np.float32) for j in range(inference_V.n_iter): x_batch, idx_batch = next_batch(train_data, M) cau_batch = cau[idx_batch, :] weights_batch = weights[idx_batch, :] reconstr_cau_batch = reconstr_cau_train[idx_batch, :] x_batch = x_batch.todense().astype('int') cau_batch = cau_batch.todense() weights_batch = weights_batch.todense() sd_batch = 1. / np.sqrt(1 + weights_batch) info_dict = inference_V.update(feed_dict={x_ph: x_batch, idx_ph: idx_batch, \ reconstr_cau_ph: reconstr_cau_batch, cau_ph: cau_batch, sd_ph: sd_batch}) inference_beta0.update(feed_dict={x_ph: x_batch, idx_ph: idx_batch, \ reconstr_cau_ph: reconstr_cau_batch, cau_ph: cau_batch, sd_ph: sd_batch}) inference_gamma.update(feed_dict={x_ph: x_batch, idx_ph: idx_batch, \ reconstr_cau_ph: reconstr_cau_batch, cau_ph: cau_batch, sd_ph: sd_batch})
def train_model(self): if not os.path.exists(self.MODEL_NAME + '_result'): os.mkdir(self.MODEL_NAME + '_result') if not os.path.exists(self.LOGS_DIR): os.mkdir(self.LOGS_DIR) if not os.path.exists(self.CKPT_DIR): os.mkdir(self.CKPT_DIR) if not os.path.exists(self.OUTPUT_DIR): os.mkdir(self.OUTPUT_DIR) train_set_path = read_data_path(self.TRAIN_IMAGE_PATH, self.TRAIN_LABEL_PATH) valid_set_path = read_data_path(self.VALID_IMAGE_PATH, self.VALID_LABEL_PATH) ckpt_save_path = os.path.join( self.CKPT_DIR, self.MODEL_NAME + '_' + str(self.N_BATCH) + '_' + str(self.LEARNING_RATE)) with tf.Session() as sess: sess.run(tf.global_variables_initializer()) total_batch = int(len(train_set_path) / self.N_BATCH) counter = 0 self.saver = tf.train.Saver() self.writer = tf.summary.FileWriter(self.LOGS_DIR, sess.graph) for epoch in tqdm(range(self.N_EPOCH)): total_loss = 0 random.shuffle(train_set_path) # 매 epoch마다 데이터셋 shuffling random.shuffle(valid_set_path) for i in range(int(len(train_set_path) / self.N_BATCH)): # print(i) batch_xs_path, batch_ys_path = next_batch( train_set_path, self.N_BATCH, i) batch_xs = read_image(batch_xs_path, [self.RESIZE, self.RESIZE]) batch_ys = read_annotation(batch_ys_path, [self.RESIZE, self.RESIZE]) feed_dict = { self.input_x: batch_xs, self.label_y: batch_ys, self.is_train: True } _, summary_str, loss = sess.run( [self.optimizer, self.loss_summary, self.loss], feed_dict=feed_dict) self.writer.add_summary(summary_str, counter) counter += 1 total_loss += loss ## validation 과정 valid_xs_path, valid_ys_path = next_batch(valid_set_path, 4, 0) valid_xs = read_image(valid_xs_path, [self.RESIZE, self.RESIZE]) valid_ys = read_annotation(valid_ys_path, [self.RESIZE, self.RESIZE]) valid_pred = sess.run(self.pred, feed_dict={ self.input_x: valid_xs, self.label_y: valid_ys, self.is_train: False }) valid_pred = np.squeeze(valid_pred, axis=3) valid_ys = np.squeeze(valid_ys, axis=3) ## plotting and save figure img_save_path = self.OUTPUT_DIR + '/' + str(epoch).zfill( 3) + '.png' draw_plot_segmentation(img_save_path, valid_xs, valid_pred, valid_ys) print('\nEpoch:', '%03d' % (epoch + 1), 'Avg Loss: {:.6}\t'.format(total_loss / total_batch)) self.saver.save(sess, ckpt_save_path + '_' + str(epoch) + '.model', global_step=counter) self.saver.save(sess, ckpt_save_path + '_' + str(epoch) + '.model', global_step=counter) print('Finish save model')
var_list=qgamma_variables, optimizer=optimizer) inference_beta0.initialize(scale={ x: scale_factor, U: scale_factor, gamma: scale_factor }, var_list=qbeta0_variables, optimizer=optimizer) tf.global_variables_initializer().run() loss = np.empty(inference_V.n_iter, dtype=np.float32) for j in range(inference_V.n_iter): x_batch, idx_batch = next_batch(train_data, M) cau_batch = cau[idx_batch, :] weights_batch = weights[idx_batch, :] reconstr_cau_batch = reconstr_cau[idx_batch, :] x_batch = x_batch.todense().astype('int') cau_batch = cau_batch.todense() weights_batch = weights_batch.todense() sd_batch = 1. / np.sqrt(1 + weights_batch) info_dict = inference_V.update(feed_dict={x_ph: x_batch, idx_ph: idx_batch, \ reconstr_cau_ph: reconstr_cau_batch, cau_ph: cau_batch, sd_ph: sd_batch}) inference_beta0.update(feed_dict={x_ph: x_batch, idx_ph: idx_batch, \ reconstr_cau_ph: reconstr_cau_batch, cau_ph: cau_batch, sd_ph: sd_batch}) inference_gamma.update(feed_dict={x_ph: x_batch, idx_ph: idx_batch, \ reconstr_cau_ph: reconstr_cau_batch, cau_ph: cau_batch, sd_ph: sd_batch})
train_iters = int(len(train_labels) / batch_size) else: train_iters = int(len(train_labels) / batch_size) + 1 print("Optimization starting..") with tf.Session() as sess: sess.run(init) epoch_loss_list = [] epoch_index_list = [] for epoch in range(epochs): epoch_loss = 0 train_inputs, train_labels = utils.suffle_data(train_inputs, train_labels) for i in range(train_iters): batch_x = utils.next_batch(train_inputs, i, batch_size) batch_y = utils.next_batch(train_labels, i, batch_size) batch_x = batch_x.reshape((-1, TIME_SERIES_LENGTH, inputs_num)) batch_y = batch_y.reshape((-1, output_neurons)) _, c = sess.run([optimizer, loss], feed_dict={ xplaceholder: batch_x, yplaceholder: batch_y }) epoch_loss += c / train_iters #if(i==0 or i==train_iters-1): #print(loaded_minmax_scaler.inverse_transform(sess.run(logits, feed_dict={xplaceholder: batch_x, yplaceholder: batch_y}).reshape(-1,1)).reshape(1,-1)[0]) epoch_loss_list.append(epoch_loss) epoch_index_list.append(epoch + 1) print('Epoch', epoch + 1, 'completed out of', epochs, 'loss:', epoch_loss)
if pretrain: print("Starting autoencoder pretraining...") # Variables to save pretraining tensor content embeddings = np.zeros((specs.n_samples, specs.embedding_size), dtype=float) # First, pretrain the autoencoder ## Loop over epochs for epoch in range(n_pretrain_epochs): print("Pretraining step: epoch {}".format(epoch)) # Loop over the samples for _ in range(n_batches): # Fetch a random data batch of the specified size indices, batch_data, sec_data, masked_data = next_batch( batch_size, data, data_sec, data_masked) _, embedding_, ae_loss_, keywords_embedding_ = sess.run( (cg.pretrain_op, cg.embedding, cg.ae_loss, cg.keywords_embedding), feed_dict={ cg.input: batch_data, cg.data_masked: masked_data, cg.data_second: sec_data }) # Save the embeddings for batch samples for j in range(len(indices)): embeddings[indices[j], :] = embedding_[j, :] #print("ae_loss_:", float(ae_loss_)) # Second, run k-means++ on the pretrained embeddings