def preprocess(opt): print('Begin preprocessing') train_dataset = data.DataSet(opt.train_data, display_freq=opt.display_freq) train_dataset.max_dict = opt.dict_size train_dataset.build_dict() print('Save training data') torch.save(train_dataset, opt.train_data + '.prep.train.pt') val_dataset = data.DataSet(opt.val_data, display_freq=opt.display_freq) val_dataset.change_dict(train_dataset.dictionary) print('Save validation data') torch.save(val_dataset, opt.val_data + '.prep.val.pt') print('Preprocessing done')
def train_wrapper(model): '''Data loader''' train_set = data.DataSet(FLAGS.root_dir, FLAGS.dataset, 'train', FLAGS.batch_size, FLAGS.n_label, data_aug=False, shuffle=True) valid_set = data.DataSet(FLAGS.root_dir, FLAGS.dataset, 'val', FLAGS.batch_size, FLAGS.n_label, data_aug=False, shuffle=False) '''create a tf session for training and validation TODO: to run your model, you may call model.train(), model.save(), model.valid()''' best_accuracy = 0 acc_train = [] acc_valid = [] for step in range(1, FLAGS.max_iteration + 1): if not train_set.has_next_batch(): train_set.init_epoch() batch_x, batch_y = train_set.next_batch() if len(batch_x) == FLAGS.batch_size: loss, acc = model.train(batch_x, batch_y) if step == 1 or step % 10 == 0: print("Step " + str(step) + ", Minibatch Loss= " + \ "{:.4f}".format(loss) + ", Training Accuracy= " + "{:.3f}".format(acc)) if step % FLAGS.valid_iteration == 0: acc_train.append(acc) tot_acc = 0.0 tot_input = 0 while valid_set.has_next_batch(): valid_ims, valid_labels = valid_set.next_batch() loss, acc = model.valid(valid_ims, valid_labels) tot_acc += acc * len(valid_ims) tot_input += len(valid_ims) acc = tot_acc / tot_input valid_set.init_epoch() print("Current Accuracy= " + "{:.3f}".format(acc)) acc_valid.append(acc) if acc > best_accuracy: model.save(step) best_accuracy = acc print("Optimization Finished!")
def dump(): dataset = data.DataSet() dataset.classes experiment = '2' dataset.dumpNumpyFiles(seq_len_limit=20, experiment=experiment) return (experiment, dataset)
def test_save_load(dataset_type, repeat): """ Test initialising a dataset subclass with a random number of input and output dimensions and number of points in the training and test sets, and check that it can be saved and loaded, and that the saved and loaded dataset objects are equivalent """ set_random_seed_from_args("test_save_load", dataset_type, repeat) dataset = get_dataset_and_name_from_type(dataset_type) dataset_name = repr(dataset) dataset.save(dataset_name, output_dir) dataset_loaded = data.DataSet().load(dataset_name, output_dir) assert np.all(dataset.train.x == dataset_loaded.train.x) assert np.all(dataset.train.y == dataset_loaded.train.y) assert np.all(dataset.train.n == dataset_loaded.train.n) assert np.all(dataset.test.x == dataset_loaded.test.x) assert np.all(dataset.test.y == dataset_loaded.test.y) assert np.all(dataset.test.n == dataset_loaded.test.n)
def preprocess(): data_info = data.DataSet(FLAGS.positive_data_file, FLAGS.negative_data_file) x_text, y = data_info.x_text, data_info.y # Build vocabulary max_document_length = max([len(x.split(" ")) for x in x_text]) print('max_document_length', max_document_length) # 根据所有已分词好的文本建立好一个词典,然后找出每个词在词典中对应的索引,不足长度或者不存在的词补0 vocab_processor = learn.preprocessing.VocabularyProcessor( max_document_length) x = np.array(list(vocab_processor.fit_transform(x_text))) tf.logging.info("Shape of X :{}".format(str(x.shape))) with open(os.path.join(FLAGS.model_dir, 'vocab_processor.pkl'), 'wb') as f: pickle.dump(vocab_processor, f) # Random shuffle data np.random.seed(10) shuffle_indices = np.random.permutation(np.arange(len(y))) x_shuffled = x[shuffle_indices] y_shuffled = y[shuffle_indices] # Split train/test set dev_sample_index = -1 * int(0.1 * float(len(y))) x_train, x_dev = x_shuffled[:dev_sample_index], x_shuffled[ dev_sample_index:] y_train, y_dev = y_shuffled[:dev_sample_index], y_shuffled[ dev_sample_index:] # Init model config model_config = ModelConfig(embedding_dim=FLAGS.embedding_dim, filter_sizes=FLAGS.filter_sizes, num_filters=FLAGS.num_filters, dropout_rate=FLAGS.drop_rate, l2_reg_lambda=FLAGS.l2_reg_lambda, max_seq_length=max_document_length, vocab_size=len(vocab_processor.vocabulary_), label_size=2) tf.logging.info("Vocabulary size: {:d}".format( len(vocab_processor.vocabulary_))) tf.logging.info("Train/dev split: {:d}/{:d}".format( len(y_train), len(y_dev))) tf.logging.info("*******Init Model CONFIG*************") tf.logging.info(model_config.to_string()) return x_train, y_train, vocab_processor, x_dev, y_dev, model_config
def main(data_set_name, data_directory, log_directory, results_directory, splitting_method, splitting_fraction, latent_size, hidden_sizes, reconstruction_distribution, number_of_reconstruction_classes, number_of_warm_up_epochs, number_of_epochs, batch_size, learning_rate, reset_training): # Data data_set = data.DataSet(data_set_name, data_directory) training_set, validation_set, test_set = data_set.split( splitting_method, splitting_fraction) feature_size = data_set.number_of_features # Modeling model = modeling.VariationalAutoEncoder( feature_size, latent_size, hidden_sizes, reconstruction_distribution, number_of_reconstruction_classes, ) log_directory = os.path.join(log_directory, data_set_name, model.name) model.train(training_set, validation_set, number_of_epochs, batch_size, learning_rate, # number_of_warm_up_epochs, log_directory, reset_training) # Analysis results_directory = os.path.join(results_directory, data_set_name, model.name) analysis.analyseModel(log_directory, results_directory) reconstructed_test_set, latent_set, test_metrics = model.evaluate(test_set, batch_size, log_directory) analysis.analyseResults(test_set, reconstructed_test_set, latent_set, results_directory)
def test_wrapper(model): '''Test your code.''' test_set = data.DataSet(FLAGS.root_dir, FLAGS.dataset, 'test', FLAGS.batch_size, FLAGS.n_label, data_aug=False, shuffle=False) '''TODO: Your code here.''' model.load() tot_acc = 0.0 tot_input = 0 while test_set.has_next_batch(): test_ims, test_labels = test_set.next_batch() _, acc = model.valid(test_ims, test_labels) tot_acc += acc * len(test_ims) tot_input += len(test_ims) acc = tot_acc / tot_input print("Test Accuracy= " + "{:.3f}".format(acc)) print("Test Finished!")
def train(self): saver = tf.train.Saver() with tf.Session() as sess: BEST_LOSS = 999999.0 sess.run(tf.global_variables_initializer()) for e in range(self.EPOCH): if len(self.INPUT) < self.BATCH_SIZE: self.BATCH_SIZE = len(self.INPUT) TRAIN_DATA = dataset.DataSet(self.INPUT, self.LABEL) BATCH_TOTAL = math.ceil(len(self.INPUT) / self.BATCH_SIZE) print('Total Batch = {}, Batch Size = {}'.format( BATCH_TOTAL, self.BATCH_SIZE)) for b in range(BATCH_TOTAL): feed = { self.MODEL.inputs: self.INPUT, self.MODEL.labels: self.LABEL, self.MODEL.learning_rate: self.LEARNING_RATE, self.MODEL.is_training: True } TRAIN_LOSS, _, TRAIN_ACC = sess.run([ self.MODEL.cost, self.MODEL.optimizer, self.MODEL.accuracy ], feed_dict=feed) if BEST_LOSS > TRAIN_LOSS: BEST_LOSS = TRAIN_LOSS if b % (BATCH_TOTAL // 5) == 0: print( 'Epoch [{0:2d}/{1:2d}] ---------------------------------- \n' .format(e + 1, self.EPOCH) + 'Batch [{0:4d}/{1:4d}]: Loss = {2:4.2f}, best Loss = {3:4.2f}' .format((b + 1), BATCH_TOTAL, TRAIN_LOSS, BEST_LOSS)) # VALIDATION valid_acc_list = [] pred_class_all = [] pred_class_all = [] total_vali_loss = [] target_class_all = [] VALID_DATA = dataset.DataSet(self.VAL_INPUT, self.VAL_LABEL) BATCH_VALID_TOTAL = math.ceil( len(self.VAL_INPUT) / self.BATCH_SIZE) for v in range(BATCH_VALID_TOTAL): BATCH_VALID_IMAGES, BATCH_VALID_LABELS = VALID_DATA.next_batch( self.BATCH_SIZE) feed = { self.MODEL.inputs: self.VAL_INPUT, self.MODEL.labels: self.VAL_LABEL, self.MODEL.is_training: False } VAL_LOSS, VAL_ACC, VALID_CLASS = sess.run([ self.MODEL.cost, self.MODEL.accuracy, self.MODEL.predicted ], feed_dict=feed) valid_acc_list.append(VAL_ACC) pred_class_all.extend(VALID_CLASS) total_vali_loss.append(VAL_LOSS) target_class_all.extend(BATCH_VALID_LABELS) if v == BATCH_VALID_TOTAL - 1: valid_acc_avg = np.mean(valid_acc_list) print("Valid Accuracy Average: ", valid_acc_avg) print('[Weight] Update.') saver.save(sess, "checkpoint/porto_pilsa.ckpt")
help='是否一起训练embeddding') parser.add_argument('--mode', type=str, default='test', choices=['train', 'test', 'demo', 'export'], help='train:训练模型;test:测试模型;demo:使用模型') args = parser.parse_args() config = config.model_config() # for arg in vars(args): # setattr(config,arg,getattr(args,arg)) if args.mode == 'train': train_data = data.DataSet(config.word2id_path, config.embedding_path) val_data = data.DataSet(config.word2id_path, config.embedding_path) train_data.read_tag_file(config.train_data) val_data.read_tag_file(config.val_data) config.embedding_matrix = train_data.embeddding model = model.BiLSTM_CRF(config) model.build() model.train(train_data, val_data) if args.mode == 'test': test_data = data.DataSet(config.word2id_path, config.embedding_path) test_data.read_tag_file(config.test_data) config.embedding_matrix = test_data.embeddding model = model.BiLSTM_CRF(config) model.build() error_list = model.test(test_data)
def __init__(self): self.data_set = data.DataSet()
def load_data(self): t_data, te_data, t_label, te_label = dataset.read_CIFAR10_subset() train_set = dataset.DataSet(t_data, t_label) test_set = dataset.DataSet(te_data, te_label) return train_set, test_set
label = int(image_dir) for image_file in os.listdir(os.path.join(image_root_dir, image_dir)): image_raw_data = tf.gfile.FastGFile(os.path.join(image_root_dir, image_dir, image_file), 'rb').read() # 灰度图像可以收敛 奇怪 img_data = tf.image.rgb_to_grayscale(tf.image.decode_jpeg(image_raw_data)).eval() / 255 img_data = tf.image.decode_jpeg(image_raw_data).eval() / 255 image_array[i] = img_data label_array[i, label] = 1 i += 1 print("i=", i) image_array = tf.cast(image_array, tf.float32).eval() label_array = tf.cast(label_array, tf.float32).eval() return image_array, label_array # 读取图片训练集 images, labels = read_images_and_labels("E:/singleCaptcha") train = data.DataSet(images, labels) # 测试随机洗牌是否准确 # for i in range(100000): # train_image, train_label = train.next_batch(20) # weight initialization # tf.truncated_normal(shape, mean=0.0, stddev=1.0, dtype=tf.float32, seed=None, name=None) # 生产正态分布,均值为0 标准差为0.1 def weight_variable(shape): initial = tf.truncated_normal(shape, stddev=0.1) # 声明一个变量 return tf.Variable(initial)
def runGraph(gdef, batch_size, num_loops, input_name, outputs, dtype=np.float32, input_data=None): #set up graph gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.5) tf.reset_default_graph() g = tf.Graph() outlist = [] with g.as_default(): #input if not input_data: input_data = np.random.uniform(size=(batch_size * 100, 3, 224, 224)).astype(dtype) dataset = tf.data.Dataset.from_tensor_slices(input_data) dataset = dataset.batch(batch_size) iterator = dataset.make_initializable_iterator() next_image = iterator.get_next() elif isinstance(input_data, str): #scan input path filelist = sorted([ os.path.join(input_data, x) for x in os.listdir(input_data) if x.endswith('.h5') ]) #instantiate reader: h5ir = data.DataSet(filelist, dtype=dtype) if dtype == np.float32: tftype = tf.float32 elif dtype == np.float16: tftype = tf.float16 else: raise ValueError( "Error, type {dt} not supported.".format(dt=dtype)) #create dataset dataset = tf.data.Dataset.from_generator( h5ir.next, output_types=(tftype, tf.int32, tf.float32, tf.float32, tf.int32), output_shapes=((3, 224, 224), (1), (1), (1), (1))) dataset = dataset.prefetch(batch_size) dataset = dataset.batch(batch_size, drop_remainder=True) iterator = dataset.make_initializable_iterator() next_element = iterator.get_next() next_image = next_element[0] out = tf.import_graph_def(graph_def=gdef, input_map={input_name: next_image}, return_elements=outputs) out = out[0].outputs[0] outlist.append(out) outlist.append(next_element[1]) outlist.append(next_element[3]) outlist.append(next_element[4]) with tf.Session(graph=g, config=tf.ConfigProto(gpu_options=gpu_options)) as sess: predictions = [] labels = [] weights = [] psr = [] #loop over epochs for epoch in range(num_loops): sess.run(iterator.initializer) #loop over dataset while True: try: vals = sess.run(outlist) predictions.append(vals[0][:, 1]) labels.append(vals[1][:, 0]) weights.append(vals[2][:, 0]) psr.append(vals[3][:, 0]) except tf.errors.OutOfRangeError: print("Epoch {} done.".format(epoch)) break predictions = np.concatenate(predictions, axis=0) labels = np.concatenate(labels, axis=0) weights = np.concatenate(weights, axis=0) psr = np.concatenate(psr, axis=0) return predictions, labels, weights, psr
def train(rank, world_size, options, gpus): os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" os.environ["CUDA_VISIBLE_DEVICES"] = gpus[rank] if options.DDP: os.environ['MASTER_ADDR'] = 'localhost' os.environ['MASTER_PORT'] = '77777' torch.distributed.init_process_group(backend="nccl", rank=rank, world_size=world_size) is_first = rank == 0 else: is_first = True # initialize data train_labaled_dataset = data.DataSet(source_file_name='labeled_train.h5') train_labaled_sampler = torch.utils.data.RandomSampler( train_labaled_dataset, replacement=True, num_samples=options.epoch_samples) train_labaled_loader = data.FastDataLoader(dataset=train_labaled_dataset, batch_size=options.batch_size, sampler=train_labaled_sampler, num_workers=2, drop_last=True, prefetch_factor=4) train_dataset = train_labaled_dataset train_loader = train_labaled_loader test_dataset = data.DataSet(source_file_name='test_data.h5') # initialize model if options.fine_tune_last: #options.max_lr = 1e-4 if options.max_lr >= 1e-3 else options.max_lr datetime_regex = re.compile(r'[a-zA-Z_]_(\d{4}.*)$') logs = [(foldername, datetime_regex.search(foldername).groups()[0]) for foldername in os.listdir(options.log_path)] logs = [(foldername, datetime.strptime(timestamp, '%Y_%b_%d_%p_%I_%M_%S')) for foldername, timestamp in logs] logs.sort(key=lambda item: item[1], reverse=True) last_log = logs[0][0] # load the pickle object with open(f'{options.log_path}/{last_log}/model.pickle', 'rb') as filein: model = pickle.load(filein) print(f'loading check point {last_log}/check_point.pth') # load last saved state dicts and setup loss function epoch_start = model.load_check_point( f'{options.log_path}/{last_log}/check_point.pth') # change log_path log_path = f'{options.log_path}/{last_log}' elif options.model_pickle_path: with open(options.model_pickle_path, 'rb') as filein: model = pickle.load(filein) epoch_start = 0 else: model = candidate.single_task(options=options, input_shape=(patch_size, patch_size)) epoch_start = 0 model.update_rules(options) model.update_scheduler(options, epoch_start) if is_first: # initialize new folder for logs, only do this when folder not exist if not options.fine_tune_last: time = datetime.now().strftime('%Y_%b_%d_%p_%I_%M_%S') log_path = f'{options.log_path}/{model.name}_{time}' os.mkdir(log_path) [ os.mkdir(f'{log_path}/{foldername}') for foldername in ['models', 'test_images'] ] shutil.copytree(os.getcwd(), f'{log_path}/code_used') with open(f'{log_path}/train_epoch_log.csv', 'w') as fileout: fileout.write(','.join( ['epoch'] + [f'train_{name}' for name in model.loss_names] + [f'test_{name}' for name in model.loss_names]) + '\n') # dump model and options with open(f'{log_path}/model.pickle', 'wb') as fileout: pickle.dump(model, fileout) with open(f'{log_path}/options.pickle', 'wb') as fileout: pickle.dump(options, fileout) print(model) print('-' * line_width) print( f'{len(train_dataset):_} images, {options.epoch_samples:_} samples per epoch, ' f'{len(train_dataset) // options.epoch_samples} epochs for full coverage' ) print(f'{model.name} parameters: {model.compute_params()}') print(f'foldername: {log_path}') max_batch = len(train_loader) current_lr = options.max_lr epoch_start += 1 epoch_max = epoch_start + options.max_epoch # start training for epoch_current in range(epoch_start, epoch_max): start_time = datetime.now() model.set_to_train() print('-' * line_width) if is_first else None # do batches for batch_index, input_data in enumerate(train_loader, 1): # this should not be changed, modification of loss and input should be done in model methods model.set_input(input_data) current_lr = model.train(batch_index) # dynamic alignment by length line = f'[{epoch_current:>{len(str(epoch_max))}}/{epoch_max-1}]' + \ f'[{batch_index:>{len(str(max_batch))}}/{max_batch}] ' + \ f'current learning rate: {current_lr:5e}' print(f'\r{options.schedule}{line}', end=' ', flush=True) if is_first else None # exit training, do test if is_first: model.set_to_test() validation = [] for image_index, input_data in enumerate(test_dataset): input_data = torch.unsqueeze(input_data, 0) model.set_input(input_data) # prep the data prediction, target = model.test() # flush image to disk validation.append([test_dataset.name, *prediction, target]) validation = pd.DataFrame(columns=[ 'name', *[f'prediction_{number}' for number in range(1, 10)], 'target' ], data=validation) validation.to_csv( f'{log_path}/test_images/epoch_{epoch_current}.csv', index=False) # we're done with doing test, write to log file train_loss = [ f'{value:.5f}' for value in model.gather_loss('train') ] test_loss = [f'{value:.5f}' for value in model.gather_loss('test')] with open(f'{log_path}/train_epoch_log.csv', 'a') as fileout: fileout.write(','.join([str(epoch_current)] + train_loss + test_loss) + '\n') # save model state_dict and check_point model.save_inference( f'{log_path}/models/epoch_{epoch_current}.pth') model.save_check_point(f'{log_path}/check_point.pth', epoch=epoch_current) # use df.to_string() for convenience epoch_df = pd.DataFrame(columns=model.loss_names, data=[train_loss, test_loss]) epoch_df.index = ['train', 'test'] time_used = datetime.now() - start_time time_str = f'epoch time: {time_used.seconds // 60 :>02}:{time_used.seconds % 60 :>02}' print(time_str, epoch_df.to_string(), sep='\n') # exit validation phase # exit training if options.DDP: torch.distributed.destroy_process_group() return None
from datetime import datetime import data import model from benchmark import timeit # os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' # Load config file # ! Specify base_dir with open('/workspaces/ps_project/config.yaml') as file: config = yaml.safe_load(file) # Load train, test and validation dataset train_dataset = data.DataSet(batch_size=config['batch_size'], mode='train', path=config["base_dir"] + 'data/', patch_size=config["patch_size"], wavelengths=config["wavelengths"]) train_ds = train_dataset.load_data() print("Loading test dataset...") validation_dataset = data.DataSet(batch_size=config["batch_size"], mode='validation', path=config["base_dir"] + 'data/', patch_size=config["patch_size"], wavelengths=config["wavelengths"]) validation_ds = validation_dataset.load_data() print("Loading validation dataset...") test_dataset = data.DataSet(batch_size=config["batch_size"], mode='test',
# Loading config gile # ! Specify base_dir with open('/workspaces/ps_project/config.yaml') as file: config = yaml.safe_load(file) # Create dataset from images in sample/ prepare = sample_prepare.SamplePrepare(dest_path=config["base_dir"] + 'sample/', src_path=config["base_dir"] + 'sample/', size=config["patch_size"]) prepare.prepare_sample() # Create tf object for dataset sample_dataset = data.DataSet(batch_size=config["batch_size"], mode='sample', path=config["base_dir"] + 'sample/', patch_size=config["patch_size"], wavelengths=config["wavelengths"]) sample_ds = sample_dataset.load_data() print("\nDataset created and loaded") # Model setup checkpoint_dir_u = config["base_dir"] + 'models/unet/train/' checkpoint_path_u = config["base_dir"] + 'models/unet/train/cp-{epoch:04d}.ckpt' latest_checkpoint_u = tf.train.latest_checkpoint(checkpoint_dir_u) checkpoint_dir_r = config["base_dir"] + 'models/resnet/train/' checkpoint_path_r = config[ "base_dir"] + 'models/resnet/train/cp-{epoch:04d}.ckpt' latest_checkpoint_r = tf.train.latest_checkpoint(checkpoint_dir_r)
} # Build the model print "Compiling model" start = time.time() NFE = nfe.NeuralFoldingEngine(num_gpus=ARGS.num_gpus, use_profiles=ARGS.profiles) elapsed = time.time() - start print " compiles in " + str(elapsed) + " s\n" # Load data CATH_FILE = "data/cath_L200_class123.txt" CATH_SETS_FILE = "data/cath_L200_class123_sets.json" dataset = data.DataSet(cath_file=CATH_FILE, cath_sets_file=CATH_SETS_FILE, max_length=ARGS.length, num_gpus=ARGS.num_gpus, placeholder_sets=NFE.placeholder_sets) # Load auxilliary sequence data prefix = "data/" if ARGS.enrich: seq_file = prefix + "sequences_cath_L200.txt" weights_file = prefix + "sequence_weights_cath_L200.txt" dataset.load_alignments(seq_file, weights_file) if ARGS.profiles: profile_file = prefix + "profiles_cath_L200.json" profile_dict = dataset.load_profiles(profile_file) # Build experiment paths base_folder = time.strftime("log/%y%b%d_%I%M%p/", time.localtime())
def run_training(): print "==================\n================\nLooking for Directory: {}".format(FLAGS.train) data_set = data.DataSet(FLAGS.train) if data_set.length() < FLAGS.batch_size: FLAGS.batch_size = data_set.length() if not FLAGS.val: print "missing --val flag" exit(1) val_data_set = data.DataSet(FLAGS.val) rgb = tf.placeholder("float", [FLAGS.batch_size, 224, 224, 3], name="rgb") variables_to_restore = {} step_var = tf.Variable(0, trainable=False) variables_to_restore["step"] = step_var inc_step = step_var.assign_add(1) loss, rgb_dist, summary_image = training_model(rgb, variables_to_restore) tf.scalar_summary("loss", loss) tf.scalar_summary("learning_rate", FLAGS.learning_rate) #avg_loss = define_avg("avg_loss", step_var, loss) #avg_rgb_dist = define_avg("avg_rgb_dist", step_var, rgb_dist) # Create a saver for writing training checkpoints. print_variables_to_restore(variables_to_restore) saver = tf.train.Saver(var_list=variables_to_restore, max_to_keep=3) # Create a session for running Ops on the Graph. sess = tf.Session() # Instantiate a SummaryWriter to output summaries and the Graph. summary_writer = tf.train.SummaryWriter('log', flush_secs=5) opt = tf.train.MomentumOptimizer(FLAGS.learning_rate, 0.0) train_op = opt.minimize(loss) # Important step for batch norm see batch_norm.py with tf.control_dependencies([train_op]): train_op2 = tf.group(*tf.get_collection("update_assignments")) # Build the summary operation based on the TF collection of Summaries. summary_op = tf.merge_all_summaries() sess.run(tf.initialize_all_variables()) train_csv = TextData('train-loss.csv') val_csv = TextData('val-loss.csv') maybe_restore_checkpoint(sess, saver) print "begin training" print "learning rate", FLAGS.learning_rate print "batch size", FLAGS.batch_size if FLAGS.fixed: feed_dict = { rgb: data_set.next_batch(FLAGS.batch_size) } while True: step = sess.run(inc_step) start_time = time.time() if not FLAGS.fixed: feed_dict = { rgb: data_set.next_batch(FLAGS.batch_size) } _, loss_value, rgb_dist_val = sess.run([train_op2, loss, rgb_dist], feed_dict=feed_dict) duration = time.time() - start_time # Print status to stdout. print('Step %d: rgb_dist = %.4f / loss = %.4f (%.1f sec)' % (step, rgb_dist_val, loss_value, duration)) train_csv.write(step, loss_value, rgb_dist_val) if step % 50 == 0: # Update the events file. summary_str = sess.run(summary_op, feed_dict=feed_dict) summary_writer.add_summary(summary_str, step) # Save a checkpoint saver.save(sess, './checkpoint', global_step=step) # now test the validation if step % 100 == 0: print "running validation..." count = 0 summary_image_index = 0 val_loss = [] val_rgb_dist = [] while count < 100: feed_dict = { rgb: val_data_set.next_batch(FLAGS.batch_size) } loss_value, rgb_dist_value, summary_image_value = sess.run([loss, rgb_dist, summary_image], feed_dict=feed_dict) for i in range(0, FLAGS.batch_size): img = summary_image_value[i] summary_image_index+=1 if summary_image_index <= 4: fn = "val-imgs/val-%06d-%d.jpg" % (step, summary_image_index) imsave(fn, img) val_loss.append(loss_value) val_rgb_dist.append(rgb_dist_value) count += FLAGS.batch_size val_avg_loss = np.mean(val_loss) val_avg_rgb_dist = np.mean(val_rgb_dist) print('VALIDATION SET: count = %d / rgb_dist = %.4f / loss = %.4f' % (count, val_avg_rgb_dist, val_avg_loss)) val_csv.write(step, val_avg_loss, val_avg_rgb_dist)
parser.add_argument('--model', type=str, default='./model/penn-lm.best.pt', help='Path to trained model') parser.add_argument('--cuda', action='store_true', help='Use GPU') parser.add_argument('--batch_size', type=int, default=64, help='Evaluate batch size if use gpu') opt = parser.parse_args() # assert not os.path.exists(opt.model) or not os.path.exists(opt.eval_data) model = torch.load(opt.model) model.eval() eval_dataset = data.DataSet(opt.eval_data) eval_dataset.change_dict(model.dictionary) if opt.cuda: model.cuda() eval_dataset.set_batch_size(opt.batch_size) print('Start Evaluation ...') loss = evaluate(model, eval_dataset, opt.cuda) print('Evaluation Result') print('Loss : %f, Perplexity : %f' % (loss, math.exp(loss)))
def main(input_file_or_name, data_directory = "data", log_directory = "log", results_directory = "results", temporary_log_directory = None, map_features = False, feature_selection = [], example_filter = [], preprocessing_methods = [], noisy_preprocessing_methods = [], split_data_set = True, splitting_method = "default", splitting_fraction = 0.9, model_type = "VAE", latent_size = 50, hidden_sizes = [500], number_of_importance_samples = [5], number_of_monte_carlo_samples = [10], inference_architecture = "MLP", latent_distribution = "gaussian", number_of_classes = None, parameterise_latent_posterior = False, generative_architecture = "MLP", reconstruction_distribution = "poisson", number_of_reconstruction_classes = 0, prior_probabilities_method = "uniform", number_of_warm_up_epochs = 0, kl_weight = 1, proportion_of_free_KL_nats = 0.0, batch_normalisation = True, dropout_keep_probabilities = [], count_sum = True, number_of_epochs = 200, plotting_interval_during_training = None, batch_size = 100, learning_rate = 1e-4, run_id = None, new_run = False, prediction_method = None, prediction_training_set_name = "training", prediction_decomposition_method = None, prediction_decomposition_dimensionality = None, decomposition_methods = ["PCA"], highlight_feature_indices = [], reset_training = False, skip_modelling = False, model_versions = ["all"], analyse = True, evaluation_set_name = "test", analyse_data = False, analyses = ["default"], analysis_level = "normal", fast_analysis = False, export_options = []): # Setup model_versions = parseModelVersions(model_versions) ## Analyses if fast_analysis: analyse = True analyses = ["simple"] analysis_level = "limited" ## Distributions reconstruction_distribution = parseDistribution( reconstruction_distribution) latent_distribution = parseDistribution(latent_distribution) ## Model configuration validation if not skip_modelling: if run_id: run_id = checkRunID(run_id) model_valid, model_errors = validateModelParameters( model_type, latent_distribution, reconstruction_distribution, number_of_reconstruction_classes, parameterise_latent_posterior ) if not model_valid: print("Model configuration is invalid:") for model_error in model_errors: print(" ", model_error) print() if analyse_data: print("Skipping modelling.") print("") skip_modelling = True else: print("Modelling cancelled.") return ## Binarisation binarise_values = False if reconstruction_distribution == "bernoulli": if noisy_preprocessing_methods: if noisy_preprocessing_methods[-1] != "binarise": noisy_preprocessing_methods.append("binarise") print("Appended binarisation method to noisy preprocessing,", "because of the Bernoulli distribution.\n") else: binarise_values = True ## Data sets if not split_data_set or analyse_data or evaluation_set_name == "full" \ or prediction_training_set_name == "full": full_data_set_needed = True else: full_data_set_needed = False # Data print(title("Data")) data_set = data.DataSet( input_file_or_name, directory = data_directory, map_features = map_features, feature_selection = feature_selection, example_filter = example_filter, preprocessing_methods = preprocessing_methods, binarise_values = binarise_values, noisy_preprocessing_methods = noisy_preprocessing_methods ) if full_data_set_needed: data_set.load() if split_data_set: training_set, validation_set, test_set = data_set.split( splitting_method, splitting_fraction) all_data_sets = [data_set, training_set, validation_set, test_set] else: splitting_method = None training_set = data_set validation_set = None test_set = data_set all_data_sets = [data_set] evaluation_set_name = "full" prediction_training_set_name = "full" ## Setup of log and results directories log_directory = data.directory(log_directory, data_set, splitting_method, splitting_fraction) data_results_directory = data.directory(results_directory, data_set, splitting_method, splitting_fraction, preprocessing = False) results_directory = data.directory(results_directory, data_set, splitting_method, splitting_fraction) if temporary_log_directory: main_temporary_log_directory = temporary_log_directory temporary_log_directory = data.directory(temporary_log_directory, data_set, splitting_method, splitting_fraction) ## Data analysis if analyse and analyse_data: print(subtitle("Analysing data")) analysis.analyseData( data_sets = all_data_sets, decomposition_methods = decomposition_methods, highlight_feature_indices = highlight_feature_indices, analyses = analyses, analysis_level = analysis_level, export_options = export_options, results_directory = data_results_directory ) print() ## Full data set clean up if not full_data_set_needed: data_set.clear() # Modelling if skip_modelling: print("Modelling skipped.") return print(title("Modelling")) # Set the number of features for the model feature_size = training_set.number_of_features # Parse numbers of samples number_of_monte_carlo_samples = parseSampleLists( number_of_monte_carlo_samples) number_of_importance_samples = parseSampleLists( number_of_importance_samples) # Use analytical KL term for single-Gaussian-VAE if "VAE" in model_type: if latent_distribution == "gaussian": analytical_kl_term = True else: analytical_kl_term = False # Change latent distribution to Gaussian mixture if not already set if model_type == "GMVAE" and latent_distribution != "gaussian mixture": latent_distribution = "gaussian mixture" print("The latent distribution was changed to", "a Gaussian-mixture model, because of the model chosen.\n") # Set the number of classes if not already set if not number_of_classes: if training_set.has_labels: number_of_classes = training_set.number_of_classes \ - training_set.number_of_excluded_classes elif "mixture" in latent_distribution: raise ValueError( "For a mixture model and a data set without labels, " "the number of classes has to be set." ) else: number_of_classes = 1 print(subtitle("Model setup")) if model_type == "VAE": model = VariationalAutoencoder( feature_size = feature_size, latent_size = latent_size, hidden_sizes = hidden_sizes, number_of_monte_carlo_samples =number_of_monte_carlo_samples, number_of_importance_samples = number_of_importance_samples, analytical_kl_term = analytical_kl_term, inference_architecture = inference_architecture, latent_distribution = latent_distribution, number_of_latent_clusters = number_of_classes, parameterise_latent_posterior = parameterise_latent_posterior, generative_architecture = generative_architecture, reconstruction_distribution = reconstruction_distribution, number_of_reconstruction_classes = number_of_reconstruction_classes, batch_normalisation = batch_normalisation, dropout_keep_probabilities = dropout_keep_probabilities, count_sum = count_sum, number_of_warm_up_epochs = number_of_warm_up_epochs, kl_weight = kl_weight, log_directory = log_directory, results_directory = results_directory ) elif model_type == "GMVAE": if prior_probabilities_method == "uniform": prior_probabilities = None elif prior_probabilities_method == "infer": prior_probabilities = training_set.class_probabilities elif prior_probabilities_method == "literature": prior_probabilities = training_set.literature_probabilities else: prior_probabilities = None if not prior_probabilities: prior_probabilities_method = "uniform" prior_probabilities_values = None else: prior_probabilities_values = list(prior_probabilities.values()) prior_probabilities = { "method": prior_probabilities_method, "values": prior_probabilities_values } model = GaussianMixtureVariationalAutoencoder( feature_size = feature_size, latent_size = latent_size, hidden_sizes = hidden_sizes, number_of_monte_carlo_samples = number_of_monte_carlo_samples, number_of_importance_samples = number_of_importance_samples, analytical_kl_term = analytical_kl_term, prior_probabilities = prior_probabilities, number_of_latent_clusters = number_of_classes, proportion_of_free_KL_nats = proportion_of_free_KL_nats, reconstruction_distribution = reconstruction_distribution, number_of_reconstruction_classes = number_of_reconstruction_classes, batch_normalisation = batch_normalisation, dropout_keep_probabilities = dropout_keep_probabilities, count_sum = count_sum, number_of_warm_up_epochs = number_of_warm_up_epochs, kl_weight = kl_weight, log_directory = log_directory, results_directory = results_directory ) else: raise ValueError("Model type not found: `{}`.".format(model_type)) print(model.description) print() print(model.parameters) print() ## Training print(subtitle("Model training")) status, run_id = model.train( training_set, validation_set, number_of_epochs = number_of_epochs, batch_size = batch_size, learning_rate = learning_rate, plotting_interval = plotting_interval_during_training, run_id = run_id, new_run = new_run, reset_training = reset_training, temporary_log_directory = temporary_log_directory ) # Remove temporary directories created and emptied during training if temporary_log_directory and os.path.exists(main_temporary_log_directory): removeEmptyDirectories(main_temporary_log_directory) if not status["completed"]: print(status["message"]) return status_filename = "status" if "epochs trained" in status: status_filename += "-" + status["epochs trained"] status_path = os.path.join( model.logDirectory(run_id = run_id), status_filename + ".log" ) with open(status_path, "w") as status_file: for status_field, status_value in status.items(): if status_value: status_file.write( status_field + ": " + str(status_value) + "\n" ) print() # Evaluation, prediction, and analysis ## Setup if analyse: if prediction_method: predict_labels_using_model = False elif "GM" in model.type: predict_labels_using_model = True prediction_method = "model" else: predict_labels_using_model = False else: predict_labels_using_model = False evaluation_title_parts = ["evaluation"] if analyse: if prediction_method: evaluation_title_parts.append("prediction") evaluation_title_parts.append("analysis") evaluation_title = enumerateListOfStrings(evaluation_title_parts) print(title(evaluation_title.capitalize())) ### Set selection for data_subset in all_data_sets: clear_subset = True if data_subset.kind == evaluation_set_name: evaluation_set = data_subset clear_subset = False if prediction_method \ and data_subset.kind == prediction_training_set_name: prediction_training_set = data_subset clear_subset = False if clear_subset: data_subset.clear() ### Evaluation set evaluation_subset_indices = analysis.evaluationSubsetIndices( evaluation_set) print("Evaluation set: {} set.".format(evaluation_set.kind)) ### Prediction method if prediction_method: prediction_method = properString( prediction_method, PREDICTION_METHOD_NAMES ) prediction_method_specifications = PREDICTION_METHOD_SPECIFICATIONS\ .get(prediction_method, {}) prediction_method_inference = prediction_method_specifications.get( "inference", None) prediction_method_fixed_number_of_clusters \ = prediction_method_specifications.get( "fixed number of clusters", None) prediction_method_cluster_kind = prediction_method_specifications.get( "cluster kind", None) if prediction_method_fixed_number_of_clusters: number_of_clusters = number_of_classes else: number_of_clusters = None if prediction_method_inference \ and prediction_method_inference == "transductive": prediction_training_set = None prediction_training_set_name = None else: prediction_training_set_name = prediction_training_set.kind prediction_details = { "method": prediction_method, "number_of_classes": number_of_clusters, "training_set_name": prediction_training_set_name, "decomposition_method": prediction_decomposition_method, "decomposition_dimensionality": prediction_decomposition_dimensionality } print("Prediction method: {}.".format(prediction_method)) if number_of_clusters: print("Number of clusters: {}.".format(number_of_clusters)) if prediction_training_set: print("Prediction training set: {} set.".format( prediction_training_set.kind)) prediction_id_parts = [] if prediction_decomposition_method: prediction_decomposition_method = properString( prediction_decomposition_method, DECOMPOSITION_METHOD_NAMES ) if not prediction_decomposition_dimensionality: prediction_decomposition_dimensionality \ = DEFAULT_DECOMPOSITION_DIMENSIONALITY prediction_id_parts += [ prediction_decomposition_method, prediction_decomposition_dimensionality ] prediction_details.update({ "decomposition_method": prediction_decomposition_method, "decomposition_dimensionality": prediction_decomposition_dimensionality }) print("Decomposition method before prediction: {}-d {}.".format( prediction_decomposition_dimensionality, prediction_decomposition_method )) prediction_id_parts.append(prediction_method) if number_of_clusters: prediction_id_parts.append(number_of_clusters) if prediction_training_set \ and prediction_training_set.kind != "training": prediction_id_parts.append(prediction_training_set.kind) prediction_id = "_".join(map( lambda s: normaliseString(str(s)).replace("_", ""), prediction_id_parts )) prediction_details["id"] = prediction_id else: prediction_details = {} ### Model parameter sets model_parameter_set_names = [] if "end_of_training" in model_versions: model_parameter_set_names.append("end of training") if "best_model" in model_versions \ and betterModelExists(model, run_id = run_id): model_parameter_set_names.append("best model") if "early_stopping" in model_versions \ and modelStoppedEarly(model, run_id = run_id): model_parameter_set_names.append("early stopping") print("Model parameter sets: {}.".format(enumerateListOfStrings( model_parameter_set_names))) print() ## Model analysis if analyse: print(subtitle("Model analysis")) analysis.analyseModel( model = model, run_id = run_id, analyses = analyses, analysis_level = analysis_level, export_options = export_options, results_directory = results_directory ) ## Results evaluation, prediction, and analysis for model_parameter_set_name in model_parameter_set_names: if model_parameter_set_name == "best model": use_best_model = True else: use_best_model = False if model_parameter_set_name == "early stopping": use_early_stopping_model = True else: use_early_stopping_model = False model_parameter_set_name = model_parameter_set_name.capitalize() print(subtitle(model_parameter_set_name)) # Evaluation model_parameter_set_name = model_parameter_set_name.replace(" ", "-") print(heading("{} evaluation".format(model_parameter_set_name))) if "VAE" in model.type: transformed_evaluation_set, reconstructed_evaluation_set,\ latent_evaluation_sets = model.evaluate( evaluation_set = evaluation_set, evaluation_subset_indices = evaluation_subset_indices, batch_size = batch_size, predict_labels = predict_labels_using_model, run_id = run_id, use_best_model = use_best_model, use_early_stopping_model = use_early_stopping_model ) else: transformed_evaluation_set, reconstructed_evaluation_set = \ model.evaluate( evaluation_set = evaluation_set, evaluation_subset_indices = evaluation_subset_indices, batch_size = batch_size, run_id = run_id, use_best_model = use_best_model, use_early_stopping_model = use_early_stopping_model ) latent_evaluation_sets = None print() # Prediction if analyse and "VAE" in model.type and prediction_method \ and not transformed_evaluation_set.has_predictions: print(heading("{} prediction".format(model_parameter_set_name))) latent_prediction_evaluation_set = latent_evaluation_sets["z"] if prediction_method_inference \ and prediction_method_inference == "inductive": latent_prediction_training_sets = model.evaluate( evaluation_set = prediction_training_set, batch_size = batch_size, run_id = run_id, use_best_model = use_best_model, use_early_stopping_model = use_early_stopping_model, output_versions = "latent", log_results = False ) latent_prediction_training_set \ = latent_prediction_training_sets["z"] print() else: latent_prediction_training_set = None if prediction_decomposition_method: if latent_prediction_training_set: latent_prediction_training_set, \ latent_prediction_evaluation_set \ = data.decomposeDataSubsets( latent_prediction_training_set, latent_prediction_evaluation_set, method = prediction_decomposition_method, number_of_components = prediction_decomposition_dimensionality, random = True ) else: latent_prediction_evaluation_set \ = data.decomposeDataSubsets( latent_prediction_evaluation_set, method = prediction_decomposition_method, number_of_components = prediction_decomposition_dimensionality, random = True ) print() cluster_ids, predicted_labels, predicted_superset_labels \ = predict( latent_prediction_training_set, latent_prediction_evaluation_set, prediction_method, number_of_clusters ) transformed_evaluation_set.updatePredictions( predicted_cluster_ids = cluster_ids, predicted_labels = predicted_labels, predicted_superset_labels = predicted_superset_labels ) reconstructed_evaluation_set.updatePredictions( predicted_cluster_ids = cluster_ids, predicted_labels = predicted_labels, predicted_superset_labels = predicted_superset_labels ) for variable in latent_evaluation_sets: latent_evaluation_sets[variable].updatePredictions( predicted_cluster_ids = cluster_ids, predicted_labels = predicted_labels, predicted_superset_labels = predicted_superset_labels ) print() # Analysis if analyse: print(heading("{} results analysis".format(model_parameter_set_name))) analysis.analyseResults( evaluation_set = transformed_evaluation_set, reconstructed_evaluation_set = reconstructed_evaluation_set, latent_evaluation_sets = latent_evaluation_sets, model = model, run_id = run_id, decomposition_methods = decomposition_methods, evaluation_subset_indices = evaluation_subset_indices, highlight_feature_indices = highlight_feature_indices, prediction_details = prediction_details, best_model = use_best_model, early_stopping = use_early_stopping_model, analyses = analyses, analysis_level = analysis_level, export_options = export_options, results_directory = results_directory ) # Clean up if transformed_evaluation_set.version == "original": transformed_evaluation_set.resetPredictions()
def train(self): start_time = time.time() mode = [key for (key, value) in self.IS_MODEL.items() if value == True] if len(mode) > 1 or len(mode) == 0: print("You can't select more than one mode.") return x_train, y_train, x_valid, y_valid, input_size = 0, 0, 0, 0, (0, 0) channel = 3 class_names = [] mode = mode[0] network_model = self.NETWORK_MODEL[mode] #_____________________________________________________________load dataset(Train, Validation) __________________________________________________________________________# x_train, y_train, x_valid, y_valid, class_names = self._loadDataSetClassification( ) # x_train => type : np.array(uint8) list, ex : [array([[[68,68,68]...,array(...] # y_train => type : tuple, ex : ('4','2','0',...) self.DATASET_CROP_COUNT = 1 class_num = len(class_names) train_image_num = len(x_train) #__________________________________________________________________________ create model __________________________________________________________________________# model = build_model.build_model( mode, network_model, (self.DATASET_CROP_SIZE, self.DATASET_CROP_SIZE), self.BATCH_SIZE, train_image_num, class_num, self.PATH_PRETRAIN, self.HYPERP_LEARNING_RATE, self.HYPERP_DECAY_FACTOR, channel, channel) #__________________________________________________________________________ Training __________________________________________________________________________# print('Starting at Epoch = {}, learning rate = {}'.format( self.HYPERP_EPOCH, self.HYPERP_LEARNING_RATE)) config = tf.ConfigProto() config.gpu_options.allow_growth = True deviceList = ['/gpu:0', '/gpu:1'] GPU_index = 0 with tf.Session(config=config) as sess: with tf.device(deviceList[GPU_index]): # Initialize all variables #tf.global_variables_initializer().run() sess.run(tf.global_variables_initializer()) sess.run(tf.local_variables_initializer()) best_loss = 999999.0 totallCnt = 0 loss_list = [] # checkpoint & pretrain checkpoit load if not os.path.exists(self.PATH_CHECKPOINT): os.makedirs(self.PATH_CHECKPOINT) if not os.path.exists(self.PATH_CHECKPOINT_SAVE): os.makedirs(self.PATH_CHECKPOINT_SAVE) if mode == 'CLASSIFICATION': model.restore_fn(sess) elif self.CHECKPOINT_CONTINUE == True: model.restore_fn(sess, self.PATH_CHECKPOINT_SAVE) # Data generate - Validation input_valid_images = x_valid.copy() label_valid_images = y_valid input_valid_count = len(input_valid_images) for e in range(self.HYPERP_EPOCH_START, self.HYPERP_EPOCH): if self.TRAINING_STOP: break input_images = x_train.copy() label_images = y_train input_count = len(input_images) label_count = 0 start = time.time() # Data Augmentation dataset.dataAugmentation_run_thread( mode=mode, dict_rotation=self.AUG_ROTATION, dict_blur_gaussian=self.AUG_BLUR_GAUSSIAN, dict_noise_gaussian=self.AUG_NOISE_GAUSSIAN, input_images=input_images, label_images=label_images) print( 'Data generator : Input image count = {}, Label image count = {}' .format(input_count, label_count)) if input_count < self.BATCH_SIZE: self.BATCH_SIZE = input_count # Dataset object 생성 train_data = dataset.DataSet(input_images, label_images) # train_data => type : data.DataSet Object if input_valid_count > 0: valid_data = dataset.DataSet(input_valid_images, label_valid_images) batch_total = math.ceil(input_count / self.BATCH_SIZE) print('Total Batch = {}, Batch size = {}'.format( batch_total, self.BATCH_SIZE)) for b in range(batch_total): if self.TRAINING_STOP: print('Train Stop...!!!') break # Batch 추출 batch_images, batch_labels = train_data.next_batch( self.BATCH_SIZE) batch_labels_num = batch_labels.copy() ## labels to one hot encoding ## batch_one_hot_list = [] for i in range(len(batch_labels)): a = np.zeros(class_num) a[batch_labels[i]] = 1 batch_one_hot_list.append(list(a)) batch_labels = batch_one_hot_list ## apply cutmix ## if self.apply_cutmix == True: #c_size = np.ceil(96*(e+1)/self.HYPERP_EPOCH) increasing window c_size = 35 + e batch_images, batch_labels = self.cutmix( batch_images, batch_labels, class_num, c_size, rate=1) # Train Step loss, predicted_class, cur_learning_rate, acc = model.train_step( sess, batch_images, batch_labels) acc = acc[1] if best_loss > loss: best_loss = loss if b % (batch_total // 5) == 0: self._print_train_info( e, self.HYPERP_EPOCH, b, batch_total, Loss=loss, BestLoss=best_loss, CurrentLearningRate=cur_learning_rate) print('chart=0,{0:},{1:}'.format(totallCnt, loss)) loss_list.append(loss) totallCnt += 1 #acc = model.train_step_accuracy(sess, batch_images, batch_labels) #print('accuracy:{}'.format(acc)) if b == batch_total - 1: # save model checkpoint checkpoint_save_path = "%s/%04d-%04d" % ( self.PATH_CHECKPOINT_SAVE, e, b) if (e + 1) % 10 == 0: model.save_checkpoint(sess, checkpoint_save_path, mode) model.save_checkpoint( sess, self.PATH_CHECKPOINT_SAVE, mode) print('Saved the checkpoint.(path:{})'.format( checkpoint_save_path)) ## input image save #model.save_image(utils, checkpoint_save_path, batch_images, batch_labels_num, output_image, class_names, predicted_class, 'Train') #hcw, cutmix #print('[Weight] Update.') #loss_list.append(loss) #self._SaveTrainInfoTextFile(checkpoint_save_path, e, loss, best_loss) @ hcw, cutmix # validation # hcw, cutmix valid_acc_list = [] batch_valid_total = math.ceil(input_valid_count / self.BATCH_SIZE) for v in range(batch_valid_total): if self.TRAINING_STOP: break batch_valid_images, batch_valid_labels = valid_data.next_batch( self.BATCH_SIZE) ################################### batch_one_hot_list = [] for i in range(len(batch_valid_labels)): a = np.zeros(class_num) a[batch_valid_labels[i]] = 1 batch_one_hot_list.append(a) batch_valid_labels = batch_one_hot_list ###################################### loss, predicted_valid_class, acc = model.validation_step( sess, batch_valid_images, batch_valid_labels) #print(acc[1]) valid_acc_list.append(acc[1]) if v == batch_valid_total - 1: valid_acc_avg = np.mean(valid_acc_list) #model.save_image(utils, checkpoint_save_path, batch_valid_images, batch_valid_labels, output_valid_image, class_names, predicted_valid_class, 'Validation') #hcw, cutmix print('[Weight] Update.') print("Validation Accuracy : ") print(valid_acc_avg) # txt file save #self._SaveTrainInfoTextFile(self.PATH_CHECKPOINT_SAVE, e, 0.0, best_loss) # hcw, cutmix #self._SaveLossTextFile(self.PATH_CHECKPOINT_SAVE, loss_list) # hcw, cutmix end_time = time.time() print('Train Time :', end_time - start_time) time.sleep(1) print('\n')