def _load_data(self): self.config_data['batch_size'] = self.batch_size self.config_data['patch_type_x'] = self.config_net['patch_type_x'] self.config_data['patch_type_y'] = self.config_net['patch_type_y'] self.config_data['patch_shape_x'] = self.config_net['patch_shape_x'] self.config_data['patch_shape_y'] = self.config_net['patch_shape_y'] if(self.mode == 'train'): # Place data loading and preprocessing on the cpu with tf.device('/cpu:0'): self.data_agent = ImageLoader(self.config_data) # create an reinitializable iterator given the dataset structure train_dataset = self.data_agent.get_dataset('train') valid_dataset = self.data_agent.get_dataset('valid') train_iterator = Iterator.from_structure(train_dataset.output_types, train_dataset.output_shapes) valid_iterator = Iterator.from_structure(valid_dataset.output_types, valid_dataset.output_shapes) self.next_train_batch = train_iterator.get_next() self.next_valid_batch = valid_iterator.get_next() # Ops for initializing the two different iterators self.train_init_op = train_iterator.make_initializer(train_dataset) self.valid_init_op = valid_iterator.make_initializer(valid_dataset) else: self.data_agent = ImageLoader(self.config_data) self.test_dataset = self.data_agent.get_dataset('test')
def init_tfdata(self, batch_size, main_dir, resize_shape, mode='train'): self.data_session = tf.Session() print("Creating the iterator for training data") with tf.device('/cpu:0'): segdl = SegDataLoader(main_dir, batch_size, (resize_shape[0], resize_shape[1]), resize_shape, # * 2), resize_shape, 'data/cityscapes_tfdata/train.txt') iterator = Iterator.from_structure(segdl.data_tr.output_types, segdl.data_tr.output_shapes) next_batch = iterator.get_next() self.init_op = iterator.make_initializer(segdl.data_tr) self.data_session.run(self.init_op) print("Loading Validation data in memoryfor faster training..") self.val_data = {'X': np.load(self.args.data_dir + "X_val.npy"), 'Y': np.load(self.args.data_dir + "Y_val.npy")} # self.crop() # import cv2 # cv2.imshow('crop1', self.val_data['X'][0,:,:,:]) # cv2.imshow('crop2', self.val_data['X'][1,:,:,:]) # cv2.imshow('seg1', self.val_data['Y'][0,:,:]) # cv2.imshow('seg2', self.val_data['Y'][1,:,:]) # cv2.waitKey() self.val_data_len = self.val_data['X'].shape[0] - self.val_data['X'].shape[0] % self.args.batch_size # self.num_iterations_validation_per_epoch = ( # self.val_data_len + self.args.batch_size - 1) // self.args.batch_size self.num_iterations_validation_per_epoch = self.val_data_len // self.args.batch_size print("Val-shape-x -- " + str(self.val_data['X'].shape) + " " + str(self.val_data_len)) print("Val-shape-y -- " + str(self.val_data['Y'].shape)) print("Num of iterations on validation data in one epoch -- " + str(self.num_iterations_validation_per_epoch)) print("Validation data is loaded") return next_batch, segdl.data_len
def tf_create_iterator(dataset, batch_size): """ """ dataset_prefix = os.path.join(SNPX_DATASET_ROOT, dataset, dataset) train_rec_file = dataset_prefix + "_train.tfrecords" val_rec_file = dataset_prefix + "_val.tfrecords" # Create the training dataset object train_set = TFRecordDataset(train_rec_file) train_set = train_set.map(tf_parse_record, num_threads=4, output_buffer_size=1000) train_set = train_set.shuffle(buffer_size=50000) train_set = train_set.batch(batch_size) # Create the validation dataset object val_set = TFRecordDataset(val_rec_file) val_set = val_set.map(tf_parse_record) val_set = val_set.batch(batch_size) # Create a reinitializable iterator from both datasets iterator = Iterator.from_structure(train_set.output_types, train_set.output_shapes) train_init_op = iterator.make_initializer(train_set) val_init_op = iterator.make_initializer(val_set) iter_op = iterator.get_next() return train_init_op, val_init_op, iter_op
def generate_input_pipline(tfrecords_path, filenames, batch_size, test_batch_size, capacity, _parse_function_train, _parse_function_test): ''' :param sess: :param tfrecords_path: :param filenames: :param epochs: :param batch_size: :param capacity: :param _parse_function: tf dataset api parse function, you should provide this function by yourself :return: ''' file_paths = [os.path.join(tfrecords_path, name) for name in filenames] train_dataset = tf.contrib.data.TFRecordDataset(file_paths[0]) train_dataset = train_dataset.map(_parse_function_train) train_dataset = train_dataset.repeat(1) train_dataset = train_dataset.batch(batch_size) train_dataset = train_dataset.shuffle(buffer_size=capacity) test_dataset = tf.contrib.data.TFRecordDataset(file_paths[1]) test_dataset = test_dataset.map(_parse_function_test) test_dataset = test_dataset.repeat(1) test_dataset = test_dataset.batch(test_batch_size) iterator = Iterator.from_structure(train_dataset.output_types, train_dataset.output_shapes) next_element = iterator.get_next() training_init_op = iterator.make_initializer(train_dataset) test_init_op = iterator.make_initializer(test_dataset) return next_element, training_init_op, test_init_op
def validate(model_def): """ Validate my alexnet implementation Args: model_def: the model class/definition """ img_dir = os.path.join('.', 'images') images = [] print "loading images ..." files = fnmatch.filter(os.listdir(img_dir), '*.jpeg') for f in files: print "> " + f img_file = tf.read_file(os.path.join(img_dir, f)) img_decoded = tf.image.decode_jpeg(img_file, channels=3) img_processed = model_def.image_prep.preprocess_image( image=img_decoded, output_height=model_def.image_size, output_width=model_def.image_size, is_training=False) images.append(img_processed) # create TensorFlow Iterator object images = Dataset.from_tensors(images) iterator = Iterator.from_structure(images.output_types, images.output_shapes) next_element = iterator.get_next() iterator_init_op = iterator.make_initializer(images) # create the model and get scores (pipe to softmax) model = model_def(next_element) scores = model.get_final_op() softmax = tf.nn.softmax(scores) print 'start validation ...' with tf.Session() as sess: # Initialize all variables and the iterator sess.run(tf.global_variables_initializer()) sess.run(iterator_init_op) # Load the pretrained weights into the model model.load_initial_weights(sess) # run the graph probs = sess.run(softmax) if model_def is ResNetV2: probs = prep_resnet_results(probs) # sometime we have an offset offset = len(class_names) - len(probs[0]) # print the results for prob in probs: best_index = np.argmax(prob) print "> " + class_names[best_index + offset] + " -> %.4f" % prob[best_index]
def get_dataset_ops(data_train, data_val, batch_size, train_size, val_size, shuffle=True): """ """ # shuffle the dataset and create batches if shuffle: data_train = data_train.shuffle(train_size) data_val = data_val.shuffle(val_size) data_train = data_train.batch(batch_size) data_val = data_val.batch(batch_size) # create an reinitializable iterator given the dataset structure iterator = Iterator.from_structure(data_train.output_types, data_train.output_shapes) next_batch = iterator.get_next() # Ops for initializing the two different iterators init_op_train = iterator.make_initializer(data_train) init_op_val = iterator.make_initializer(data_val) return init_op_train, init_op_val, next_batch
def finalize(self, shapes, batchsize): assert not hasattr(self, 'output_shapes') self.output_shapes = shapes self.batchsize = batchsize ishapes = [[None] + list(s) for s in self.output_shapes] itypes = (tf.float32, tf.float32) handle = tf.placeholder(tf.string, shape=[]) iterator = Iterator.from_string_handle(handle, itypes, tuple(ishapes)) getter = iterator.get_next() return getter, handle
def train(self): tf.global_variables_initializer().run() could_load = 0 # load ckpt #could_load, checkpoint_counter = self.load(self.checkpoint_dir) if could_load: counter = checkpoint_counter print(" [*] Load SUCCESS") else: print(" [!] Load failed...") tr_data = ImageDataGenerator( FLAGS.train_file, img_size=[self.DATA_HEIGHT, self.DATA_WIDTH], label_size=[self.LABEL_HEIGHT, self.LABEL_WIDTH], batch_size=self.BATCH_SIZE, shuffle=True) print('[Model] Data done ...') # create an reinitializable iterator given the dataset structure iterator = Iterator.from_structure(tr_data.data.output_types, tr_data.data.output_shapes) next_batch = iterator.get_next() batch_idxs = int(np.floor(tr_data.data_size / self.BATCH_SIZE)) print(tr_data.data_size) for epoch in range(self.num_epochs): # print('[Ves9] Begin epoch ', epoch, '...') for iteration in range(batch_idxs): start_time = time.time() batch_images, batch_gts = self.sess.run(next_batch) print(batch_images[0].shape, batch_images[0]) print(batch_gts[0].shape, batch_gts[0]) ''' _cost, result, acc_, _ = self.sess.run( [self.cost, self.output, self.accuracy, self.train_op], feed_dict={self.images: batch_images, self.ground_truth: batch_gts}) ''' tm = time.time() - start_time ''' if np.mod(iteration, 5000) == 0 and iteration != 0: print('[INFO] Save checkpoint...') self.save(self.checkpoint_dir, iteration) if (iteration % 50 == 0): print(iteration, ': ', '%.2f' % (tm), _cost) ''' print('[INFO] Save the epoch checkpoint ... ')
def eval_data_generator(file_path, batch_size, sub_img_folder): temp = imageDataGenerator(file_path, batch_size, sub_img_folder) iterator = Iterator.from_structure(temp.data.output_types, temp.data.output_shapes) next_batch = iterator.get_next() dataset_init_op = iterator.make_initializer(temp.data) with tf.Session() as sess: sess.run(dataset_init_op) if sub_img_folder == '/test/': img, label = sess.run(next_batch) img, label = sess.run(next_batch) else: img, label = sess.run(next_batch) # print(img.shape, label) # imgplot = plt.imshow(img[0]) # plt.show() return img, label
def confusion_matrix(val_file, num_classes, checkpoint_path): with tf.device('/cpu:0'): val_data = ImageDataGenerator(val_file, mode='inference', shuffle=True) iterator = Iterator.from_structure(val_data.data.output_types, val_data.data.output_shapes) next_batch = iterator.get_next() val_init_op = iterator.make_initializer(val_data.data) x = tf.placeholder(tf.float32, shape=[1, None, None, 3], name="input_image") keep_prob = tf.placeholder(tf.float32) model = VGG16_FCN(x, num_classes, keep_prob) cm = np.zeros([num_classes, num_classes], dtype=np.uint32) with tf.Session() as sess: sess.run(tf.global_variables_initializer()) sess.run(val_init_op) saver = tf.train.Saver() ckpt = tf.train.get_checkpoint_state(checkpoint_path) saver.restore(sess, ckpt.model_checkpoint_path) for step in range(val_data.data_size): image, label = sess.run(next_batch) result_label = sess.run(model.pred_up, feed_dict={ x: image, keep_prob: 1.0 }) cml = confusion_matrix_label(result_label[0], label[0], num_classes) cm += cml return cm
def get_dataset_tensors(args): with tf.device('/cpu:0'), tf.variable_scope('input_pipeline'): # TODO move this to hem.init() # find all dataset plugins available p = get_dataset(args.dataset) # ensure that the dataset exists if not p.check_prepared_datasets(args.dataset_dir): if not p.check_raw_datasets(args.raw_dataset_dir): print('Downloading dataset...') # TODO: datasets should be able to be marked as non-downloadable p.download(args.raw_dataset_dir) print('Converting to tfrecord...') p.convert_to_tfrecord(args.raw_dataset_dir, args.dataset_dir) # load the dataset datasets = p.get_datasets(args) dataset_iterators = {} # tensor to hold which training/eval phase we are in handle = tf.placeholder(tf.string, shape=[]) # add a dataset for train, validation, and testing for k, v in datasets.items(): # skip test set if not needed if len(args.test_epochs) == 0 and k == 'test': continue d = v[0] n = sum([1 for r in tf.python_io.tf_record_iterator(v[1])]) cache_fn = '{}.cache.{}'.format(args.dataset, k) d = d.cache(os.path.join( args.cache_dir, cache_fn)) if args.cache_dir else d.cache() d = d.repeat() d = d.shuffle(buffer_size=args.buffer_size, seed=args.seed) d = d.batch(args.batch_size * args.n_gpus) x_iterator = d.make_initializable_iterator() dataset_iterators[k] = { 'x': x_iterator, 'n': n, 'batches': int(n / (args.batch_size * args.n_gpus)), 'handle': x_iterator.string_handle() } # feedable dataset that will swap between train/test/val iterator = Iterator.from_string_handle(handle, d.output_types, d.output_shapes) return iterator.get_next(), handle, dataset_iterators
def input_pipeline(data_dir): BATCH_SIZE = 16 filenames = [ join(data_dir, filename) for filename in os.listdir(data_dir) if isfile(join(data_dir, filename)) ] np.random.shuffle(filenames) train_images = tf.constant(filenames) tr_data = Dataset.from_tensor_slices(train_images) tr_data = tr_data.map(input_parser).map(distort_input) tr_data = tr_data.batch(16) # create TensorFlow Iterator object iterator = Iterator.from_structure(tr_data.output_types, tr_data.output_shapes) training_init_op = iterator.make_initializer(tr_data) next_element = iterator.get_next() with tf.Session() as sess: # initialize the iterator on the training data sess.run(training_init_op) # get each element of the training dataset until the end is reached while True: try: elem = sess.run(next_element) print(elem) except tf.errors.OutOfRangeError: print("End of training dataset.") break
# Place data loading and preprocessing on the cpu with tf.device('/cpu:0'): tr_data = ImageDataGenerator(train_file, batch_size=batch_size, num_classes=num_classes, shuffle=False) val_data = ImageDataGenerator(val_file, batch_size=batch_size, num_classes=num_classes, shuffle=False) # create an reinitializable iterator given the dataset structure iterator = Iterator.from_structure(tr_data.data.output_types, tr_data.data.output_shapes) next_batch = iterator.get_next() # Ops for initializing the two different iterators training_init_op = iterator.make_initializer(tr_data.data) validation_init_op = iterator.make_initializer(val_data.data) # TF placeholder for graph input and output x = tf.placeholder(tf.float32, [batch_size, 227, 227, 3]) y = tf.placeholder(tf.float32, [batch_size, num_classes]) keep_prob = tf.placeholder(tf.float32) # Initialize model model = AlexNet(x, keep_prob, num_classes, train_layers) # Link variable to model output
# Mapping train_dataset train_dataset = train_dataset.map(data_generator, num_threads=4, output_buffer_size=8 * batch_size) train_dataset = train_dataset.shuffle(8 * batch_size) train_dataset = train_dataset.batch(batch_size) # Mapping valid_dataset valid_dataset = valid_dataset.map(data_generator, num_threads=4, output_buffer_size=8 * batch_size) valid_dataset = valid_dataset.shuffle(8 * batch_size) valid_dataset = valid_dataset.batch(batch_size) # create TensorFlow Iterator object iterator = Iterator.from_structure(train_dataset.output_types, train_dataset.output_shapes) next_element = iterator.get_next() # create two initialization ops to switch between the datasets training_init_op = iterator.make_initializer(train_dataset) validation_init_op = iterator.make_initializer(valid_dataset) ###### Simplified Object detection model ##### # Define single layers # convolution def conv2d(name, input_layer, kernel_size, filters, padding='same', relu=True): if relu: output = tf.layers.conv2d(inputs=input_layer, filters=filters,
def train(model, saver, sess, exp_string, train_file_list, test_file, resume_itr=0): SUMMARY_INTERVAL = 100 SAVE_INTERVAL = 10000 PRINT_INTERVAL = 100 TEST_PRINT_INTERVAL = 100 dropout_rate = 0.5 if FLAGS.log: train_writer = tf.summary.FileWriter(FLAGS.logdir + '/' + exp_string, sess.graph) print('Done initializing, starting training.') prelosses, postlosses = [], [] num_classes = FLAGS.num_classes # for classification, 1 otherwise # Defining data loaders with tf.device('/cpu:0'): tr_data_list = [] train_iterator_list = [] train_next_list = [] for i in range(len(train_file_list)): tr_data = ImageDataGenerator(train_file_list[i], dataroot=FLAGS.dataroot, mode='training', batch_size=FLAGS.meta_batch_size, num_classes=num_classes, shuffle=True) tr_data_list.append(tr_data) train_iterator_list.append(Iterator.from_structure(tr_data_list[i].data.output_types, tr_data_list[i].data.output_shapes)) train_next_list.append(train_iterator_list[i].get_next()) test_data = ImageDataGenerator(test_file, dataroot=FLAGS.dataroot, mode='inference', batch_size=1, num_classes=num_classes, shuffle=False) test_iterator = Iterator.from_structure(test_data.data.output_types, test_data.data.output_shapes) test_next_batch = test_iterator.get_next() # create an reinitializable iterator given the dataset structure # Ops for initializing different iterators training_init_op = [] train_batches_per_epoch = [] for i in range(len(train_file_list)): training_init_op.append(train_iterator_list[i].make_initializer(tr_data_list[i].data)) train_batches_per_epoch.append(int(np.floor(tr_data_list[i].data_size/FLAGS.meta_batch_size))) test_init_op = test_iterator.make_initializer(test_data.data) test_batches_per_epoch = int(np.floor(test_data.data_size / 1)) # Training begins for itr in range(resume_itr, FLAGS.pretrain_iterations + FLAGS.metatrain_iterations): feed_dict = {} # Sampling training and test tasks num_training_tasks = len(train_file_list) num_meta_train = num_training_tasks-1 num_meta_test = num_training_tasks-num_meta_train # Randomly choosing meta train and meta test domains task_list = np.random.permutation(num_training_tasks) meta_train_index_list = task_list[:num_meta_train] meta_test_index_list = task_list[num_meta_train:] for i in range(len(train_file_list)): if itr%train_batches_per_epoch[i] == 0: sess.run(training_init_op[i]) # Populating input tensors # Sampling meta train data for i in range(num_meta_train): task_ind = meta_train_index_list[i] if i == 0: inputa, labela = sess.run(train_next_list[task_ind]) else: inp_tmp, lab_tmp = sess.run(train_next_list[task_ind]) inputa = np.concatenate((inputa, inp_tmp), axis=0) labela = np.concatenate((labela, lab_tmp), axis=0) inputs_all = list(zip(inputa, labela)) shuffle(inputs_all) inputa, labela = zip(*inputs_all) # Sampling meta test data for i in range(num_meta_test): task_ind = meta_test_index_list[i] if i == 0: inputb, labelb = sess.run(train_next_list[task_ind]) else: inp_tmp, lab_tmp = sess.run(train_next_list[task_ind]) inputb = np.concatenate((inputb, inp_tmp), axis=0) labelb = np.concatenate((labelb, lab_tmp), axis=0) feed_dict = {model.inputa: inputa, model.inputb: inputb, model.labela: labela, model.labelb: labelb, model.KEEP_PROB: dropout_rate} if itr<FLAGS.pretrain_iterations: input_tensors = [model.pretrain_op] else: input_tensors = [model.metatrain_op] if (itr % SUMMARY_INTERVAL == 0 or itr % PRINT_INTERVAL == 0): input_tensors.extend([model.summ_op, model.total_loss1, model.total_losses2[FLAGS.num_updates-1]]) input_tensors.extend([model.total_accuracy1, model.total_accuracies2[FLAGS.num_updates-1]]) result = sess.run(input_tensors, feed_dict) if itr % SUMMARY_INTERVAL == 0: prelosses.append(result[-2]) if FLAGS.log: train_writer.add_summary(result[1], itr) postlosses.append(result[-1]) if (itr!=0) and itr % PRINT_INTERVAL == 0: print_str = 'Iteration ' + str(itr - FLAGS.pretrain_iterations) print_str += ': ' + str(np.mean(prelosses)) + ', ' + str(np.mean(postlosses)) print(print_str) prelosses, postlosses = [], [] if (itr!=0) and itr % SAVE_INTERVAL == 0: saver.save(sess, FLAGS.logdir + '/' + exp_string + '/model' + str(itr)) # Testing periodically if itr % TEST_PRINT_INTERVAL == 0: test_acc = 0. test_loss = 0. test_count = 0 sess.run(test_init_op) for it in range(test_batches_per_epoch): test_input, test_label = sess.run(test_next_batch) feed_dict = {model.test_input: test_input, model.test_label: test_label, model.KEEP_PROB: 1.} input_tensors = [model.test_loss, model.test_acc] result = sess.run(input_tensors, feed_dict) test_loss += result[0] test_acc += result[1] test_count += 1 print('Validation results: Iteration %d, Loss: %f, Accuracy: %f' %(itr, test_loss/test_count, test_acc/test_count)) saver.save(sess, FLAGS.logdir + '/' + exp_string + '/model' + str(itr))
def main(): """Run the demo.""" data = fetch_gpml_sarcos_data() Xr = data.train.data.astype(np.float32) Yr = data.train.targets.astype(np.float32)[:, np.newaxis] Xs = data.test.data.astype(np.float32) Ys = data.test.targets.astype(np.float32)[:, np.newaxis] N, D = Xr.shape print("Iterations: {}".format(int(round(N * NEPOCHS / BATCH_SIZE)))) # Scale and centre the data, as per the original experiment ss = StandardScaler() Xr = ss.fit_transform(Xr) Xs = ss.transform(Xs) ym = Yr.mean() Yr -= ym Ys -= ym # Training batches data_tr = Dataset.from_tensor_slices({'X': Xr, 'Y': Yr}) \ .shuffle(buffer_size=1000) \ .batch(BATCH_SIZE) # Testing iterators data_ts = Dataset.from_tensors({'X': Xs, 'Y': Ys}).repeat() with tf.name_scope("DataIterators"): iterator = Iterator.from_structure(data_tr.output_types, data_tr.output_shapes) data = iterator.get_next() training_init = iterator.make_initializer(data_tr) testing_init = iterator.make_initializer(data_ts) with tf.name_scope("Deepnet"): phi, kl = net(X=data['X']) std = tf.Variable(NOISE, name="noise") lkhood = tf.distributions.Normal(phi, scale=ab.pos(std)) loss = ab.elbo(lkhood, data['Y'], N, kl) tf.summary.scalar('loss', loss) with tf.name_scope("Train"): optimizer = tf.train.AdamOptimizer() global_step = tf.train.create_global_step() train = optimizer.minimize(loss, global_step=global_step) with tf.name_scope("Test"): r2 = rsquare(data['Y'], phi) # Logging log = tf.train.LoggingTensorHook( {'step': global_step, 'loss': loss}, every_n_iter=1000 ) with tf.train.MonitoredTrainingSession( config=CONFIG, scaffold=tf.train.Scaffold(local_init_op=training_init), checkpoint_dir="./sarcos/", save_summaries_steps=None, save_checkpoint_secs=20, save_summaries_secs=20, hooks=[log] ) as sess: summary_writer = sess._hooks[1]._summary_writer for i in range(NEPOCHS): # Train for one epoch try: while not sess.should_stop(): sess.run(train) except tf.errors.OutOfRangeError: pass # Init testing and assess and log R-square score on test set sess.run(testing_init) r2_score = sess.run(r2) score_sum = tf.Summary(value=[ tf.Summary.Value(tag='r-square', simple_value=r2_score) ]) summary_writer.add_summary(score_sum, sess.run(global_step)) # Re-init training sess.run(training_init) # Prediction sess.run(testing_init) Ey = ab.predict_samples(phi, feed_dict=None, n_groups=NPREDICTSAMPLES, session=sess) sigma = sess.run(std) r2_score = sess.run(r2) # Score mean standardised log likelihood Eymean = Ey.mean(axis=0) Eyvar = Ey.var(axis=0) + sigma**2 # add sigma2 for obervation noise snlp = msll(Ys.flatten(), Eymean, Eyvar, Yr.flatten()) print("------------") print("r-square: {:.4f}, smse: {:.4f}, msll: {:.4f}." .format(r2_score, 1 - r2_score, snlp))
def main(): parser = argparse.ArgumentParser(description='Finetune Alexnet') parser.add_argument('--kfold', type=int, default=1) parser.add_argument('--images_dirname', type=str, default='images') parser.add_argument('--from_checkpoint', action='store_true', default=False) parser.add_argument('--no-from_checkpoint', action='store_false', dest='from_checkpoint') parser.add_argument('--prototype_run', action='store_true', default=False) parser.add_argument('--save_every', type=int, default=10) parser.add_argument('--display_step', type=int, default=20, help='how often to write the tf.summary to disk') parser.add_argument('--loss', type=str, default='features') parser.add_argument('--learning_rate', type=float, default=0.01) parser.add_argument('--dropout_rate', type=float, default=0.5) parser.add_argument('--num_epochs', type=int, default=1000) parser.add_argument('--batch_size', type=int, default=16) # TODO: make sure this still trains well (was 128) parser.add_argument('--patience', type=int, default=10) args = parser.parse_args() logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) logger.info('Running with args {:s}'.format(', '.join( ['{}={}'.format(key, value) for (key, value) in vars(args).items()]))) """ Configuration Part. """ # Path to the textfiles for the trainings and validation set data_path = os.path.join(os.path.dirname(__file__), args.images_dirname + ('' if not args.prototype_run else '.proto')) train_file = os.path.join(data_path, 'train{}.txt'.format(args.kfold)) val_file = os.path.join(data_path, 'val{}.txt'.format(args.kfold)) test_file = os.path.join(data_path, 'test{}.txt'.format(args.kfold)) predictions_file = os.path.join(data_path, "predictions{}.txt".format(args.kfold)) # Network params skip_layers = [] # train everything, not just ['fc8', 'fc7', 'fc6'] num_classes = 325 if args.loss == 'categorical' else None logger.debug("Num classes: {}".format(num_classes)) # Path for tf.summary.FileWriter and to store model checkpoints storage_path = os.path.join(os.path.dirname(__file__), "storage", "kfold{}".format(args.kfold)) filewriter_path = os.path.join(storage_path, "tensorboard") checkpoint_path = os.path.join(storage_path, "checkpoints") """ Main Part of the finetuning Script. """ # Create parent path if it doesn't exist if not os.path.isdir(checkpoint_path): os.makedirs(checkpoint_path, exist_ok=True) # Place data loading and preprocessing on the cpu with tf.device('/cpu:0'): logger.info('Loading data') tr_data = ImageDataGenerator(train_file, mode='training', num_classes=num_classes, batch_size=args.batch_size, shuffle=True) val_data = ImageDataGenerator(val_file, mode='inference', num_classes=num_classes, batch_size=args.batch_size, shuffle=False) test_data = ImageDataGenerator(test_file, mode='inference', num_classes=num_classes, batch_size=args.batch_size, shuffle=False) # create an reinitializable iterator given the dataset structure iterator = Iterator.from_structure(tr_data.data.output_types, tr_data.data.output_shapes) next_batch = iterator.get_next() # Ops for initializing the different iterators training_init_op = iterator.make_initializer(tr_data.data) validation_init_op = iterator.make_initializer(val_data.data) test_init_op = iterator.make_initializer(test_data.data) # TF placeholder for graph input and output x = tf.placeholder(tf.float32, [args.batch_size, 227, 227, 3], name='x') y = tf.placeholder(tf.float32, [args.batch_size, 4096 if args.loss == 'features' else num_classes], name='y') keep_prob = tf.placeholder(tf.float32) # Initialize model logger.info('Creating model') model = AlexNet(x, keep_prob, skip_layer=skip_layers + ['fc8'] if args.loss == 'categorical' else [], num_classes=num_classes) # Link variable to model output class_output = model.fc8 features_output = model.fc7 # List of trainable variables of the layers we want to train var_list = [v for v in tf.trainable_variables() if not skip_layers or v.name.split('/')[0] in skip_layers] # Op for calculating the loss with tf.name_scope("loss"): if args.loss == 'features': loss = tf.losses.mean_squared_error(predictions=features_output, labels=y) elif args.loss == 'categorical': loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=class_output, labels=y)) # loss = -tf.reduce_sum(y * tf.log(class_output + 1e-10)) else: raise ValueError('Invalid value for loss provided: {}'.format(args.loss)) # Train op with tf.name_scope("train"): # Get gradients of all trainable variables gradients = tf.gradients(loss, var_list) gradients = list(zip(gradients, var_list)) gradients = [(g, v) for g, v in gradients if g is not None] # Create optimizer and apply gradient descent to the trainable variables optimizer = tf.train.GradientDescentOptimizer(args.learning_rate) train_op = optimizer.apply_gradients(grads_and_vars=gradients) # Add gradients to summary for gradient, var in gradients: tf.summary.histogram(var.name + '/gradient', gradient) # Add the variables we train to the summary for var in var_list: tf.summary.histogram(var.name, var) # Add the loss to summary tf.summary.scalar('loss', loss) # Merge all summaries together merged_summary = tf.summary.merge_all() # Initialize the FileWriter writer = tf.summary.FileWriter(filewriter_path) # Initialize an saver for store model checkpoints saver = tf.train.Saver() # Get the number of training/validation steps per epoch train_batches_per_epoch = max(int(np.floor(tr_data.data_size / args.batch_size)), 1) val_batches_per_epoch = max(int(np.floor(val_data.data_size / args.batch_size)), 1) test_batches_per_epoch = max(int(np.floor(test_data.data_size / args.batch_size)), 1) # Start Tensorflow session with tf.Session() as sess: start_epoch = 0 if args.from_checkpoint: checkpoints = [f[:-6] for f in [os.path.join(checkpoint_path, f) for f in os.listdir(checkpoint_path) if f.endswith(".ckpt.index")] if os.path.isfile(f)] if len(checkpoints) == 0: logger.warning("No checkpoints found - starting from scratch") args.from_checkpoint = False else: last_checkpoint = sorted(checkpoints)[-1] try: saver.restore(sess, last_checkpoint) start_epoch = int( last_checkpoint[-6]) + 1 # TODO: fix hard-coded single-digit (doesn't work for 11) except Exception: logger.warning("Unable to recover checkpoint {} - starting from scratch".format(last_checkpoint)) args.from_checkpoint = False if not args.from_checkpoint: # Initialize all variables sess.run(tf.global_variables_initializer()) # Load the pretrained weights into the non-trainable layer model.load_initial_weights(sess) # Add the model graph to TensorBoard writer.add_graph(sess.graph) logger.info("{} Start training...".format(datetime.now())) logger.info("{} Open Tensorboard at --logdir {}".format(datetime.now(), filewriter_path)) # Loop over number of epochs best_val_epoch, best_val_loss = None, math.inf for epoch in range(start_epoch, args.num_epochs): logger.info("{} Epoch number: {}".format(datetime.now(), epoch + 1)) # Initialize iterator with the training dataset sess.run(training_init_op) for step in range(train_batches_per_epoch): # get next batch of data img_batch, label_batch = sess.run(next_batch) # And run the training op sess.run(train_op, feed_dict={x: img_batch, y: label_batch, keep_prob: args.dropout_rate}) # Generate summary with the current batch of data and write to file if True or step % args.display_step == 0: # TODO: remove True s = sess.run(merged_summary, feed_dict={x: img_batch, y: label_batch, keep_prob: 1.}) writer.add_summary(s, epoch * train_batches_per_epoch + step) # save checkpoint of the model if epoch % args.save_every == 0: logger.info("{} Saving checkpoint of model...".format(datetime.now())) checkpoint_name = os.path.join(checkpoint_path, 'model_epoch{:d}.ckpt'.format(epoch + 1)) save_path = saver.save(sess, checkpoint_name) logger.info("{} Model checkpoint saved at {}".format(datetime.now(), save_path)) # Validate the model on the entire validation set logger.info("{} Start validation".format(datetime.now())) sess.run(validation_init_op) val_loss = 0. val_count = 0 for _ in range(val_batches_per_epoch): img_batch, label_batch = sess.run(next_batch) _loss = sess.run(loss, feed_dict={x: img_batch, y: label_batch, keep_prob: 1.}) val_loss += _loss val_count += 1 val_loss /= val_count logger.info("{} Validation Loss = {:.4f}".format(datetime.now(), val_loss)) if val_loss < best_val_loss: best_val_loss, best_val_epoch = val_loss, epoch saver.save(sess, os.path.join(checkpoint_path, 'model_best.ckpt')) # Test the model logger.info("{} Start test".format(datetime.now())) test_predictions = np.empty((test_data.data_size, num_classes or 4096)) sess.run(test_init_op) for batch_num in range(test_batches_per_epoch): img_batch, label_batch = sess.run(next_batch) preds = sess.run(features_output, feed_dict={x: img_batch, keep_prob: 1.}) test_predictions[batch_num * args.batch_size:(batch_num + 1) * args.batch_size, :] = preds with open(predictions_file, 'w') as f: for img_path, prediction in zip(test_data._img_paths, test_predictions): f.write('{} {}\n'.format(img_path, ",".join([str(p) for p in prediction]))) logger.info("Wrote predictions to {}".format(predictions_file)) elif epoch - best_val_epoch > args.patience: logger.info("Validation loss has not decreased for {:d} epochs - stop (best epoch: {:d})".format( epoch + 1 - best_val_epoch, best_val_epoch)) break
num_classes=num_classes) val_text_k = TextualDataGenerator(valid_text_k,valid_k_label, batch_size_text=int(batch_size/3), num_classes=num_classes) tes_text_i = TextualDataGenerator(test_text_i,test_i_label, batch_size_text=int(batch_size/3), num_classes=num_classes) tes_text_j = TextualDataGenerator(test_text_j,test_j_label, batch_size_text=int(batch_size/3), num_classes=num_classes) tes_text_k = TextualDataGenerator(test_text_k,test_k_label, batch_size_text=int(batch_size/3), num_classes=num_classes) # create an reinitializable iterator given the dataset structure iterator = Iterator.from_structure(tr_data.data.output_types, tr_data.data.output_shapes) i_text_ite = Iterator.from_structure(tra_text_i.data.output_types, tra_text_i.data.output_shapes) j_text_ite = Iterator.from_structure(tra_text_j.data.output_types, tra_text_j.data.output_shapes) k_text_ite = Iterator.from_structure(tra_text_k.data.output_types, tra_text_k.data.output_shapes) next_batch = iterator.get_next() next_batchi = i_text_ite.get_next() next_batchj = j_text_ite.get_next() next_batchk = k_text_ite.get_next() # Ops for initializing the two different iterators training_init_op = iterator.make_initializer(tr_data.data)
from alexnet import AlexNet from dataprocess import ImageDataGenerator from datetime import datetime from tensorflow.contrib.data import Iterator from config import * os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" os.environ["CUDA_VISIBLE_DEVICES"] = '0' with tf.device('/cpu:0'): val_data = ImageDataGenerator(rest_file, mode='inference', batch_size=test_batch_size, num_classes=num_classes, shuffle=False) iterator = Iterator.from_structure(val_data.data.output_types, val_data.data.output_shapes) next_batch = iterator.get_next() validation_init_op = iterator.make_initializer(val_data.data) x = tf.placeholder(tf.float32, [test_batch_size, 227, 227, 3]) y = tf.placeholder(tf.float32, [test_batch_size, num_classes]) keep_prob = tf.placeholder(tf.float32) model = AlexNet(x, keep_prob, num_classes, train_layers) score = model.fc8_softmax with tf.name_scope("accuracy"): index=tf.argmax(score, 1) correct_pred = tf.equal(tf.argmax(score, 1), tf.argmax(y, 1)) accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32)) saver = tf.train.Saver()
batch_size = FLAGS.bs # Network params num_classes = FLAGS.num_class # Place data loading and preprocessing on the cpu with tf.device('/cpu:0'): test_data = ImageDataGenerator(val_file, mode='inference', batch_size=batch_size, num_classes=num_classes, shuffle=False) # create an reinitializable iterator given the dataset structure iterator_test = Iterator.from_structure(test_data.data.output_types, test_data.data.output_shapes) images_batch_test, labels_batch_test = iterator_test.get_next() # Ops for initializing the two different iterators test_init_op = iterator_test.make_initializer(test_data.data) images_batch = tf.concat([images_batch_test, images_batch_test], axis=0) y = tf.concat([labels_batch_test, labels_batch_test], axis=0) with slim.arg_scope(resnet.resnet_arg_scope()): Gap5, CONV5, net, _ = resnet.resnet_v2_50(images_batch, is_training=False) net = tf.nn.dropout(net, 1.0) net = slim.conv2d(net, num_classes, [1, 1], activation_fn=None, normalizer_fn=None,
val_labels = tf.constant([0, 0, 1]) tr_dataset = Dataset.from_tensor_slices((train_imgs, train_labels)) val_dataset = Dataset.from_tensor_slices((val_imgs, val_labels)) # iterator = Iterator.from_structure(tr_dataset.output_types, tr_dataset.output_shapes) tr_dataset = tr_dataset.map(input_parser) val_dataset = val_dataset.map(input_parser) batch_size = 2 # dataset = tr_dataset.batch(batch_size) batched_dataset = tr_dataset.batch(batch_size) iterator = Iterator.from_dataset(batched_dataset) next_elements = iterator.get_next() training_init_op = iterator.make_initializer(batched_dataset) # validation_init_op = iterator.make_initializer(val_dataset) with tf.Session() as sess: sess.run(training_init_op) while True: try: (source, label) = sess.run(next_elements) print(source) print(label) except tf.errors.OutOfRangeError: break
def build_loss(patch, patch_est): loss = tf.losses.mean_squared_error(labels=patch, predictions=patch_est) return loss # create tf dataset from input data training_dataset = tf.contrib.data.Dataset.from_tensor_slices((training_measurements, training_patches)) validation_dataset = tf.contrib.data.Dataset.from_tensor_slices((validation_measurements, validation_patches)) nEpochs = 10 training_dataset = training_dataset.batch(1000) validation_dataset = validation_dataset.batch(3000) # initialize iterator iterator = Iterator.from_structure(training_dataset.output_types, training_dataset.output_shapes) next_measurement, next_patch = iterator.get_next() training_init_op = iterator.make_initializer(training_dataset) validation_init_op = iterator.make_initializer(validation_dataset) patch_est = build_linear_mapping(next_measurement) loss = build_loss(patch_est, next_patch) # define optimization procedure learning_rate = 0.01 learning_op = tf.train.AdamOptimizer(learning_rate).minimize(loss) with tf.Session() as sess:
noise2 = tf.random_normal([32, 1024, 128]) cloud = generate_cloud(tf.expand_dims(feature, axis=1), noise2) return cloud ######################################### main ############################################# ######################################### main ############################################# ######################################### main ############################################# ######################################### main ############################################# ######################################### main ############################################# cloud_provider = tf.data.Dataset.from_generator(provide_data, output_types=(tf.float32, tf.float32), \ output_shapes=(tf.TensorShape([32, 1024, 3]), tf.TensorShape([32,40]))) point_clouds, cloud_labels = cloud_provider.make_one_shot_iterator().get_next() iterator = Iterator.from_structure(cloud_provider.output_types, cloud_provider.output_shapes) training_init_op = iterator.make_initializer(cloud_provider) noise = tf.random_normal([FLAGS.batch_size, FLAGS.noise_dims]) #with tf.variable_scope("my_scope", reuse=tf.AUTO_REUSE): # Build the generator and discriminator. gan_model = tfgan.gan_model( generator_fn=conditional_generator, # you define discriminator_fn=conditional_discriminator, # you define real_data=point_clouds, generator_inputs=(noise, cloud_labels)) # Build the GAN loss. gan_loss = tfgan.gan_loss( gan_model, #gradient_penalty_weight=1.0,
def train(hps, design): """Training loop.""" train_records = _get_tfrecord_files_from_dir( FLAGS.train_data_path) #get tfrecord files for train train_iterator = petct_input.build_input(train_records, hps.batch_size, hps.num_epochs, FLAGS.mode) train_iterator_handle = train_iterator.string_handle() if not FLAGS.val_data_path == '': # skip validation if no path val_records = _get_tfrecord_files_from_dir( FLAGS.val_data_path) # get tfrecord files for val val_iterator = petct_input.build_input(val_records, hps.batch_size, hps.num_epochs, 'valid') val_iterator_handle = val_iterator.string_handle() handle = tf.placeholder(tf.string, shape=[], name='data') iterator = Iterator.from_string_handle(handle, train_iterator.output_types, train_iterator.output_shapes) ct, pt, ctlb, ptlb, bglb = iterator.get_next() model = fuse_cnn_petct.FuseNet(hps, design, ct, pt, ctlb, ptlb, bglb, FLAGS.mode) model.build_cross_modal_model() # for use in loading later #tf.get_collection('model') #tf.add_to_collection('model',model) # put get metrics ops here for train and val with tf.variable_scope('metrics'): tr_summary_op, tr_precision_op, tr_recall_op, tr_accuracy_op, tr_rmse_op = get_metrics_ops( model, 'train') val_summary_op, val_precision_op, val_recall_op, val_accuracy_op, val_rmse_op = get_metrics_ops( model, 'valid') # needed for input handlers g_init_op = tf.global_variables_initializer() l_init_op = tf.local_variables_initializer() with tf.Session(config=tf.ConfigProto( allow_soft_placement=True, device_count={'GPU': 1})) as mon_sess: # Need a saver to save and restore all the variables. saver = tf.train.Saver() if FLAGS.DEBUG: print('ENABLING DEBUG') mon_sess = tf_debug.LocalCLIDebugWrapperSession(mon_sess) mon_sess.add_tensor_filter("has_inf_or_nan", tf_debug.has_inf_or_nan) training_handle = mon_sess.run(train_iterator_handle) if not FLAGS.val_data_path == '': # skip validation if no path validation_handle = mon_sess.run(val_iterator_handle) train_writer = tf.summary.FileWriter(FLAGS.log_root + '/train', mon_sess.graph) if not FLAGS.val_data_path == '': # skip validation if no path valid_writer = tf.summary.FileWriter(FLAGS.log_root + '/valid') mon_sess.run([g_init_op, l_init_op]) summary = None step = None val_summary = None #check = 1 while True: try: ## FIRST RUN TRAINING OP BASED ON OUTPUT STYLE if FLAGS.output_style == fuse_cnn_petct.STYLE_SPLIT: # get PET and CT recons separately _, summary, step, loss, p, r, a, e, cts, pts, trcts, trpts, trbgs, recon_cts, recon_pts, ct_preds, pt_preds = mon_sess.run( [ model.train_op, tr_summary_op, model.global_step, model.cost, tr_precision_op, tr_recall_op, tr_accuracy_op, tr_rmse_op, model.ct, model.pt, model.lbct, model.lbpt, model.lbbg, model.ct_pred, model.pt_pred, model.ct_probabilities, model.pt_probabilities ], feed_dict={ handle: training_handle, model.is_training: True }) elif FLAGS.output_style == fuse_cnn_petct.STYLE_SINGLE: # get PET and CT recons together _, summary, step, loss, p, r, a, e, cts, pts, trallpos, trbgs, recon_all, all_preds = mon_sess.run( [ model.train_op, tr_summary_op, model.global_step, model.cost, tr_precision_op, tr_recall_op, tr_accuracy_op, tr_rmse_op, model.ct, model.pt, model.lb_pos_gt, model.lbbg, model.all_pred, model.all_probabilities ], feed_dict={ handle: training_handle, model.is_training: True }) if step % FLAGS.train_iter == 0: print( '[TRAIN] STEP: %d, LOSS: %.5f, PRECISION: %.5f, RECALL: %.5f, ACCURACY: %.5f, RMSE: %.5f' % (step, loss, p, r, a, e)) train_writer.add_summary(summary, step) train_writer.flush() if FLAGS.IMSAVE > 0: if step % FLAGS.IMSAVE == 0: print('SAVING IMAGES') if FLAGS.output_style == fuse_cnn_petct.STYLE_SPLIT: _saveImages(hps.batch_size, step, cts, pts, trcts=trcts, trpts=trpts, trbgs=trbgs, recon_cts=recon_cts, recon_pts=recon_pts, ct_preds=ct_preds, pt_preds=pt_preds) elif FLAGS.output_style == fuse_cnn_petct.STYLE_SINGLE: _saveImages(hps.batch_size, step, cts, pts, trallpos=trallpos, trbgs=trbgs, recon_all=recon_all, all_preds=all_preds) if not FLAGS.val_data_path == '': # skip validation if no path if step % FLAGS.val_iter == 0: _, val_summary, loss, p, r, a, e = mon_sess.run( [ model.val_op, val_summary_op, model.cost, val_precision_op, val_recall_op, val_accuracy_op, val_rmse_op ], feed_dict={ handle: validation_handle, model.is_training: False }) val_step = step print( '[VALID] STEP: %d, LOSS: %.5f, PRECISION: %.5f, RECALL: %.5f, ACCURACY: %.5f, RMSE: %.5f' % (step, loss, p, r, a, e)) valid_writer.add_summary(val_summary, step) valid_writer.flush() if step % FLAGS.chkpt_iter == 0: save_loc = FLAGS.log_root + '/' + FLAGS.chkpt_file + str( step) + '.ckpt' save_path = saver.save(mon_sess, save_loc) print('Model saved in path: %s' % save_path) except tf.errors.OutOfRangeError: print('OUT OF DATA - ENDING') # now finished training (either train or validation has run out) train_writer.add_summary(summary, step) train_writer.flush() if not FLAGS.val_data_path == '': # skip validation if no path valid_writer.add_summary(val_summary, val_step) valid_writer.flush() save_loc = FLAGS.log_root + '/' + FLAGS.chkpt_file + str( step) + '-end.ckpt' save_path = saver.save(mon_sess, save_loc) print('Model saved in path: %s' % save_path) break
if __name__ == '__main__': batch_size = 5 attr_label_num = 30 img_loader = image_bleach.ImageLoader( [global_configs.pic2attr_tfrecord_train_path], batch_size, 100, 40000) train_dataset = img_loader.launch_tfrecord_dataset() train_iterator = train_dataset.make_one_shot_iterator() # ========================= 数据导入 ========================= # =================== 用handle导入,feedble =================== # 构造一个可导入(feedble)的句柄占位符,可以通过这个将训练集的句柄或者验证集的句柄传入 handle = tf.placeholder(tf.string, shape=[]) iterator = Iterator.from_string_handle(handle, train_dataset.output_types, train_dataset.output_shapes) pic_name_batch, pic_class_batch, attr_label_batch, img_batch = iterator.get_next( ) # 从迭代器中出来的是一个二维数组,而用到的id、effect_len和label是要一个一维数组,需要reshape以下 pic_name_batch = tf.reshape(pic_name_batch, [batch_size]) pic_class_batch = tf.reshape(pic_class_batch, [batch_size]) attr_label_batch = tf.reshape(attr_label_batch, [batch_size, attr_label_num]) # ==================/ 用handle导入,feedble /================== with tf.Session() as sess: coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(coord=coord) sess.run(tf.global_variables_initializer()) # 所有变量初始化 # 获得训练集和验证集的引用句柄,后面导入数据到模型用
def train(config_file): # 1, load configuration parameters config = parse_config(config_file) config_data = config['data'] config_net = config['network'] config_train = config['training'] random.seed(config_train.get('random_seed', 1)) assert(config_data['with_ground_truth']) net_type = config_net['net_type'] net_name = config_net['net_name'] class_num = config_net['class_num'] batch_size = config_data.get('batch_size', 5) # 2, construct graph full_data_shape = [batch_size] + config_data['data_shape'] full_label_shape = [batch_size] + config_data['label_shape'] x = tf.placeholder(tf.float32, shape = full_data_shape) w = tf.placeholder(tf.float32, shape = full_label_shape) y = tf.placeholder(tf.int64, shape = full_label_shape) w_regularizer = regularizers.l2_regularizer(config_train.get('decay', 1e-7)) b_regularizer = regularizers.l2_regularizer(config_train.get('decay', 1e-7)) net_class = NetFactory.create(net_type) net = net_class(num_classes = class_num, w_regularizer = w_regularizer, b_regularizer = b_regularizer, name = net_name) net.set_params(config_net) predicty = net(x, is_training = True) proby = tf.nn.softmax(predicty) loss_func = LossFunction(n_class=class_num) loss = loss_func(predicty, y, weight_map = w) print('size of predicty:',predicty) # 3, initialize session and saver lr = config_train.get('learning_rate', 1e-3) opt_step = tf.train.AdamOptimizer(lr).minimize(loss) sess = tf.InteractiveSession() sess.run(tf.global_variables_initializer()) saver = tf.train.Saver() loader = DataLoader(config_data) train_data = loader.get_dataset('train', shuffle = True) batch_per_epoch = loader.get_batch_per_epoch() train_iterator = Iterator.from_structure(train_data.output_types, train_data.output_shapes) next_train_batch = train_iterator.get_next() train_init_op = train_iterator.make_initializer(train_data) # 4, start to train loss_file = config_train['model_save_prefix'] + "_loss.txt" start_it = config_train.get('start_iteration', 0) if( start_it > 0): saver.restore(sess, config_train['model_pre_trained']) loss_list, temp_loss_list = [], [] for n in range(start_it, config_train['maximal_iteration']): if((n-start_it)%batch_per_epoch == 0): sess.run(train_init_op) one_batch = sess.run(next_train_batch) feed_dict = {x:one_batch['image'], w:one_batch['weight'], y:one_batch['label']} opt_step.run(session = sess, feed_dict=feed_dict) loss_value = loss.eval(feed_dict = feed_dict) temp_loss_list.append(loss_value) if((n+1)%config_train['loss_display_iteration'] == 0): avg_loss = np.asarray(temp_loss_list, np.float32).mean() t = time.strftime('%X %x %Z') print(t, 'iter', n+1,'loss', avg_loss) loss_list.append(avg_loss) np.savetxt(loss_file, np.asarray(loss_list)) temp_loss_list = [] if((n+1)%config_train['snapshot_iteration'] == 0): saver.save(sess, config_train['model_save_prefix']+"_{0:}.ckpt".format(n+1)) sess.close()
target_val_file, mode='inference', batch_size=batch_size, num_classes=num_classes, shuffle=False, mean='/home/camalab/caffe/models/vgg19/dog129sandt_mean.npy') tst_data = ImageDataGenerator( test_file, mode='inference', batch_size=batch_size, num_classes=num_classes, shuffle=False, mean='/home/camalab/caffe/models/vgg19/dog129sandt_mean.npy') # create an reinitializable iterator given the dataset structure source_iterator = Iterator.from_structure( source_tr_data.data.output_types, source_tr_data.data.output_shapes) target_iterator = Iterator.from_structure( target_tr_data.data.output_types, target_tr_data.data.output_shapes) source_next_batch = source_iterator.get_next() target_next_batch = target_iterator.get_next() # Ops for initializing the two different iterators source_training_init_op = source_iterator.make_initializer(source_tr_data.data) target_training_init_op = target_iterator.make_initializer(target_tr_data.data) source_validation_init_op = source_iterator.make_initializer( source_val_data.data) target_validation_init_op = target_iterator.make_initializer( target_val_data.data) testing_init_op = target_iterator.make_initializer(tst_data.data)
print("loading data [train and test] \n") filename_train = os.path.join(conf.DATA_DIR, "train.tfrecords") filename_test = os.path.join(conf.DATA_DIR, "test.tfrecords") #---------------read TFRecords data for training data_train = tf.data.TFRecordDataset(filename_train) data_train = data_train.map(data.parser_tfrecord) data_train = data_train.batch(conf.BATCH_SIZE) data_train = data_train.shuffle(conf.ESTIMATED_NUMBER_OF_BATCHES) #---------------read TFRecords data for validation data_test = tf.data.TFRecordDataset(filename_test) data_test = data_test.map(data.parser_tfrecord) data_test = data_test.batch(conf.BATCH_SIZE) data_test = data_test.shuffle(conf.ESTIMATED_NUMBER_OF_BATCHES_TEST) #defining saver to save snapshots #defining a reinitializable iterator iterator = Iterator.from_structure(data_train.output_types, data_train.output_shapes) iterator_test = Iterator.from_structure(data_test.output_types, data_test.output_shapes) next_batch = iterator.get_next() next_batch_test = iterator_test.get_next() #tensor that initialize the iterator: training_init_op = iterator.make_initializer(data_train) testing_init_op = iterator_test.make_initializer(data_test) print("OK") with tf.device(device_name): net = mnet.net() print("train") #to save snapshots saver = tf.train.Saver() #load mean
def main(): """ Configuration Part. """ # Path to the textfiles for the trainings and validation set train_file = './train.txt' val_file = './val.txt' # Learning params learning_rate = 0.01 num_epochs = 100 batch_size = 10 # Network params dropout_rate = 0.5 num_classes = 2 train_layers = ['fc8', 'fc7', 'fc6'] # How often we want to write the tf.summary data to disk display_step = 20 # Path for tf.summary.FileWriter and to store model checkpoints filewriter_path = "./tmp/tensorboard" checkpoint_path = "./tmp/checkpoints" """ Main Part of the finetuning Script. """ # Create parent path if it doesn't exist if not os.path.isdir(checkpoint_path): os.mkdir(checkpoint_path) # Place data loading and preprocessing on the cpu with tf.device('/cpu:0'): tr_data = ImageDataGenerator(train_file, mode='training', batch_size=batch_size, num_classes=num_classes, shuffle=True) val_data = ImageDataGenerator(val_file, mode='inference', batch_size=batch_size, num_classes=num_classes, shuffle=False) # create an reinitializable iterator given the dataset structure iterator = Iterator.from_structure(tr_data.data.output_types, tr_data.data.output_shapes) next_batch = iterator.get_next() # Ops for initializing the two different iterators training_init_op = iterator.make_initializer(tr_data.data) validation_init_op = iterator.make_initializer(val_data.data) # TF placeholder for graph input and output x = tf.placeholder(tf.float32, [None, 227, 227, 3]) y = tf.placeholder(tf.float32, [None, num_classes]) keep_prob = tf.placeholder(tf.float32) # Initialize model model = AlexNet(x, keep_prob, num_classes, train_layers) # Link variable to model output score = model.fc8 pred = tf.nn.softmax(score) # List of trainable variables of the layers we want to train var_list = [ v for v in tf.trainable_variables() if v.name.split('/')[0] in train_layers ] # Op for calculating the loss with tf.name_scope("cross_ent"): loss = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits(logits=score, labels=y)) # Train op with tf.name_scope("train"): # Get gradients of all trainable variables gradients = tf.gradients(loss, var_list) gradients = list(zip(gradients, var_list)) # Create optimizer and apply gradient descent to the trainable variables # optimizer = tf.train.GradientDescentOptimizer(learning_rate) optimizer = tf.train.AdamOptimizer(learning_rate) train_op = optimizer.apply_gradients(grads_and_vars=gradients) # Add gradients to summary for gradient, var in gradients: tf.summary.histogram(var.name + '/gradient', gradient) # Add the variables we train to the summary for var in var_list: tf.summary.histogram(var.name, var) # Add the loss to summary tf.summary.scalar('cross_entropy', loss) # Evaluation op: Accuracy of the model with tf.name_scope("accuracy"): correct_pred = tf.equal(tf.argmax(score, 1), tf.argmax(y, 1)) accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32)) # Add the accuracy to the summary tf.summary.scalar('accuracy', accuracy) # Merge all summaries together merged_summary = tf.summary.merge_all() # Initialize the FileWriter writer = tf.summary.FileWriter(filewriter_path) # Initialize an saver for store model checkpoints saver = tf.train.Saver() # Get the number of training/validation steps per epoch train_batches_per_epoch = int(np.floor(tr_data.data_size / batch_size)) val_batches_per_epoch = int(np.floor(val_data.data_size / batch_size)) # Start Tensorflow session with tf.Session() as sess: print("# isTrain : ", args.istrain) if not args.istrain: ckpt = tf.train.get_checkpoint_state(checkpoint_path) if ckpt and tf.train.checkpoint_exists(ckpt.model_checkpoint_path): print("Reading model parameters from %s" % ckpt.model_checkpoint_path) saver.restore(sess, ckpt.model_checkpoint_path) else: print("checkpoint path isn't correct") return file = tf.read_file(args.input_path) decoded_img = tf.image.decode_jpeg(file, channels=3) resized_img = tf.image.resize_images(decoded_img, [227, 227]) resized_img = tf.reshape(resized_img, [1, 227, 227, 3]) print("# decoded img : ", decoded_img.eval().shape) pred_ = sess.run(pred, feed_dict={ x: resized_img.eval(), y: [[1, 0]], keep_prob: np.array(1.0) }) print("P(man|data) : ", pred_[0][0]) print("P(woman|data) : ", pred_[0][1]) img = decoded_img.eval() plt.imshow(img) plt.show() if args.visualize: w1, w2, w3, w4, w5 = sess.run(model.weight, feed_dict={ x: resized_img.eval(), y: [[1, 0]], keep_prob: np.array(1.0) }) print("W1 : ", w1.shape) visualize(w1[:, :, 0, :25]) print("W2 : ", w2.shape) visualize(w2[:, :, 0, :25]) print("W3 : ", w3.shape) visualize(w3[:, :, 0, :25]) print("W4 : ", w4.shape) visualize(w4[:, :, 0, :25]) print("W5 : ", w5.shape) visualize(w5[:, :, 0, :25]) f1, f2, f3, f4, f5 = sess.run(model.fm, { x: resized_img.eval(), y: [[1, 0]], keep_prob: np.array(1.0) }) print("F1 : ", f1.shape) visualize(f1[0][:, :, :25]) print("F2 : ", f2.shape) visualize(f2[0][:, :, :25]) print("F3 : ", f3.shape) visualize(f3[0][:, :, :25]) print("F4 : ", f4.shape) visualize(f4[0][:, :, :25]) print("F5 : ", f5.shape) visualize(f5[0][:, :, :25]) return else: # Initialize all variables sess.run(tf.global_variables_initializer()) # Add the model graph to TensorBoard writer.add_graph(sess.graph) # Load the pretrained weights into the non-trainable layer model.load_initial_weights(sess) print("{} Start training...".format(datetime.now())) print("{} Open Tensorboard at --logdir {}".format( datetime.now(), filewriter_path)) # Loop over number of epochs for epoch in range(num_epochs): print("{} Epoch number: {}".format(datetime.now(), epoch + 1)) # Initialize iterator with the training dataset sess.run(training_init_op) train_acc = 0. train_count = 0. for step in range(train_batches_per_epoch): # get next batch of data img_batch, label_batch = sess.run(next_batch) # And run the training op acc, _ = sess.run([accuracy, train_op], feed_dict={ x: img_batch, y: label_batch, keep_prob: dropout_rate }) train_acc += acc train_count += 1 # Generate summary with the current batch of data and write to file # if step % display_step == 0: # s = sess.run(merged_summary, feed_dict={x: img_batch, # y: label_batch, # keep_prob: 1.}) # # writer.add_summary(s, epoch*train_batches_per_epoch + step) train_acc /= train_count print("{} Train Accuracy = {:.4f}".format(datetime.now(), train_acc)) # Validate the model on the entire validation set print("{} Start validation".format(datetime.now())) sess.run(validation_init_op) test_acc = 0. test_count = 0 for ind in range(val_batches_per_epoch): img_batch, label_batch = sess.run(next_batch) acc = sess.run(accuracy, feed_dict={ x: img_batch, y: label_batch, keep_prob: 1. }) test_acc += acc test_count += 1 if epoch is 2 and ind is 0: fm = sess.run(model.fm, feed_dict={ x: img_batch, y: label_batch, keep_prob: 1. }) weight = sess.run(model.weight, feed_dict={ x: img_batch, y: label_batch, keep_prob: 1. }) # print("fm0 : ", np.array(fm[0]).shape) # print("fm1 : ", np.array(fm[1]).shape) # print("fm2 : ", np.array(fm[2]).shape) # print("fm3 : ", np.array(fm[3]).shape) # print("fm4 : ", np.array(fm[4]).shape) # # print("weight0 : ", np.array(weight[0]).shape) # print("weight1 : ", np.array(weight[1]).shape) # print("weight2 : ", np.array(weight[2]).shape) # print("weight3 : ", np.array(weight[3]).shape) # print("weight4 : ", np.array(weight[4]).shape) test_acc /= test_count print("{} Validation Accuracy = {:.4f}".format( datetime.now(), test_acc)) print("{} Saving checkpoint of model...".format(datetime.now())) # save checkpoint of the model checkpoint_name = os.path.join( checkpoint_path, 'model_epoch' + str(epoch + 1) + '.ckpt') save_path = saver.save(sess, checkpoint_name) print("{} Model checkpoint saved at {}".format( datetime.now(), checkpoint_name))
return img_decoded, label_decoded # training & validation 경로 train_imgs = tf.constant([("./dataset/train_in/%d_bi.png" % i) for i in range(1, 40001)]) label_imgs = tf.constant([("./dataset/train_out/%d_gt.png" % i) for i in range(1, 40001)]) dataset = tf.data.Dataset.from_tensor_slices((train_imgs, label_imgs)) dataset = dataset.map(input_parser) dataset = dataset.shuffle(buffer_size=20000) dataset = dataset.repeat() dataset = dataset.batch(batch_size) iterator = Iterator.from_structure(dataset.output_types, dataset.output_shapes) training_init_op = iterator.make_initializer(dataset) next_element = iterator.get_next() #valid_imgs = tf.constant([("./dataset/validation_bp/%d_bi.png" % i) for i in range(1,501)]) #valid_label_imgs = tf.constant([("./dataset/validation_gp/%d_gt.png" % i) for i in range(1,501)]) #dataset2 = Dataset.from_tensor_slices((valid_imgs,valid_label_imgs)) #dataset2= dataset2.map(input_parser) #dataset2 = dataset2.shuffle(buffer_size=20000) #dataset2 = dataset2.repeat() #dataset2 = dataset2.batch(batch_size) #iterator2 = Iterator.from_structure(dataset2.output_types,dataset2.output_shapes) #training_init_op2 = iterator2.make_initializer(dataset2) #next_element2 = iterator2.get_next()
# Place data loading and preprocessing on the cpu with tf.device('/cpu:0'): tr_data = ImageDataGenerator(train_file, mode='training', batch_size=batch_size, num_classes=num_classes, shuffle=True) val_data = ImageDataGenerator(val_file, mode='inference', batch_size=batch_size, num_classes=num_classes, shuffle=False) # create an reinitializable iterator given the dataset structure iterator = Iterator.from_structure(tr_data.data.output_types, tr_data.data.output_shapes) next_batch = iterator.get_next() # Ops for initializing the two different iterators training_init_op = iterator.make_initializer(tr_data.data) validation_init_op = iterator.make_initializer(val_data.data) # TF placeholder for graph input and output x = tf.placeholder(tf.float32, [batch_size, 227, 227, 3]) y = tf.placeholder(tf.float32, [batch_size, num_classes]) keep_prob = tf.placeholder(tf.float32) # Initialize model model = AlexNet(x, keep_prob, num_classes, train_layers) # Link variable to model output