def next_batch_train(self, initial_step): """ returns: a tuple(image, depths) where: image is a float tensor with shape [batch size] + input_size depth is a float tensor with shape [batch size] + depth_size """ filenames = self.train_file dataset = tf.contrib.data.TFRecordDataset(filenames) dataset = dataset.map( self._parse_function) # Parse the record into tensors. dataset = dataset.shuffle(buffer_size=3000) dataset = dataset.batch(self.config_dict["batch_size"]) dataset = dataset.repeat( self.config_dict["num_epochs"]) # Repeat the input. dataset = dataset.skip(initial_step) iterator = dataset.make_one_shot_iterator() images, depths = iterator.get_next() #depths = tf.reshape(depths, [None] + self.output_size) #images = tf.reshape(images, [None] + self.input_size) images = simulator.applyTurbidity(images, depths, self.c, self.binf, self.range_array) tf.summary.image("depth", depths) tf.summary.image("image", images) return images, depths
def input_fn(filenames, train=True, batch_size=16, buffer_size=512): # Args: # filenames: Filenames for the TFRecords files. # train: Boolean whether training (True) or testing (False). # batch_size: Return batches of this size. # buffer_size: Read buffers of this size. The random shuffling # is done on the buffer, so it must be big enough. # Create a TensorFlow Dataset-object which has functionality # for reading and shuffling data from TFRecords files. dataset = tf.data.TFRecordDataset(filenames=filenames) # Parse the serialized data in the TFRecords files. # This returns TensorFlow tensors for the image and labels. dataset = dataset.map(parse) if train: # If training then read a buffer of the given size and # randomly shuffle it. dataset = dataset.shuffle(buffer_size=buffer_size) # Allow infinite reading of the data. num_repeat = None else: # If testing then don't shuffle the data. num_repeat = 1 # Repeat the dataset the given number of times. dataset = dataset.repeat(num_repeat) # Get a batch of data with the given size. dataset = dataset.batch(batch_size) # Create an iterator for the dataset and the above modifications. iterator = dataset.make_one_shot_iterator() # Get the next batch of images and labels, may take dimensionality info later but for now we set to _ images_batch, labels_batch, \ xdim_batch, ydim_batch, channels_batch = iterator.get_next() if train: images_batch = distort_batch(images_batch) # The input-function must return a dict wrapping the images. x = {'x': images_batch} y = labels_batch return x, y
def next_batch_train(self, initial_step): """ args: batch_size: number of examples per returned batch num_epochs: number of time to read the input data returns: a tuple(image, transmissions) where: image is a float tensor with shape [batch size] + patch_size transmissions is a float tensor with shape [batch size] """ filenames = self.train_file dataset = tf.contrib.data.TFRecordDataset(filenames) dataset = dataset.map( self._parse_function) # Parse the record into tensors. dataset = dataset.shuffle(buffer_size=3000) dataset = dataset.batch(self.config_dict["batch_size"]) dataset = dataset.repeat( self.config_dict["num_epochs"]) # Repeat the input. dataset = dataset.skip(initial_step) iterator = dataset.make_one_shot_iterator() images = iterator.get_next() size_x = self.config_dict['patch_size'][0] size_y = self.config_dict['patch_size'][1] offset_x = random.randint(0, self.input_size[0] - size_x - 1) offset_y = random.randint(0, self.input_size[0] - size_y - 1) images = images[:, offset_x:offset_x + size_x, offset_y:offset_y + size_y] transmissions = self.random_transmissions(self.batch_size) images = simulator.applyTurbidityTransmission(images, self.binf, transmissions) tf.summary.image("image", images) return images, transmissions
def next_batch_train(self, initial_step, sess): """ returns: a tuple(image, depths) where: image is a float tensor with shape [batch size] + input_size depth is a float tensor with shape [batch size] + depth_size """ filenames = self.train_file dataset = tf.contrib.data.TFRecordDataset(filenames) dataset = dataset.map( self._parse_function) # Parse the record into tensors. dataset = dataset.shuffle(buffer_size=3000) dataset = dataset.batch(self.config_dict["batch_size"]) dataset = dataset.repeat( self.config_dict["num_epochs"]) # Repeat the input. dataset = dataset.skip(initial_step) iterator = dataset.make_one_shot_iterator() images_gt, depths = iterator.get_next() #print (depths.eval(session=sess)) #depths = tf.reshape(depths, [None] + self.output_size) #images = tf.reshape(images, [None] + self.input_size) images = simulator.applyTurbidity(images_gt, depths, self.c, self.binf, self.range_array) tf.summary.image("image_gt", images_gt) tf.summary.image("depth", depths) tf.summary.image("image", images) # sess = tf.Session() # d, im = sess.run([depths[0], images_gt[0]]) # print (im.dtype) # pic = Image.fromarray(img_as_ubyte(im)) # print("depths[0]", np.min(d), np.max(d), np.mean(d)) # pic.show() return images, images_gt
train_location = tf.placeholder(tf.float32, shape=(None, 4)) training_flag = tf.placeholder(tf.bool) dense_ssd = Densenet_SSD(4, 12, training_flag) anchors = dense_ssd.anchors gclasses, glocations, gscores = dense_ssd.bboxes_encode( train_y, train_location, anchors) predictions, locations = dense_ssd.densenet_ssd(train_x) #predictions=tf.nn.softmax(predictions) #tf.cast(predictions,tf.int32) loss = dense_ssd.loss(predictions, locations, gclasses, glocations, gscores) optimizer = tf.train.AdagradOptimizer(learning_rate=1e-4) train = optimizer.minimize(loss) dataset = dataset.get_dataset('./train.tfrecords') dataset = dataset.shuffle(2) dataset = dataset.batch(1) dataset = dataset.repeat(2) iterator = dataset.make_one_shot_iterator() initializer = tf.global_variables_initializer() with tf.Session() as sess: sess.run(initializer) for i in range(iteration): data_x, data_y, data_location = iterator.get_next() data_x = tf.decode_raw(data_x, tf.uint8) data_x = tf.reshape(data_x, [-1, 4096, 2048, 3]) data_y = tf.reshape(data_y, [-1, 1]) data_location = tf.reshape(data_location, [-1, 4]) #print(data_x) #dic={train_x:data_x,train_y:data_y,train_location:data_location} data_x, data_y, data_location = sess.run(
from __future__ import print_function import tensorflow as tf import dataset import matplotlib.pyplot as plt import numpy as np import os import time import squeezenet dataset = dataset.Data_set() dataset.open('./face_photos',8) dataset.shuffle() #dataset.read(30) #[None,224,224,3] # x = tf.placeholder(tf.float32,shape=[None,224,224,3]) # y = tf.placeholder(tf.float32,shape=[None,5]) x, y = dataset.read() # print(x) x = tf.reshape(x,shape=[-1,224,224,3]) # x = tf.transpose(x,[0,3,1,2]) print(x) class netInit(object): num_classes=5 weight_decay=0.1
def train(): training_filename = [ "/home/szaman5/Phytoplankton_Classifier/test_data/train.tfrecords" ] validation_filename = [ "/home/szaman5/Phytoplankton_Classifier/test_data/validation.tfrecords" ] filename = tf.placeholder(tf.string, shape=[None]) dataset = tf.data.TFRecordDataset(filename) dataset = dataset.map(_parser, num_parallel_calls=40) dataset = dataset.shuffle(buffer_size=int(sys.argv[1])) dataset = dataset.batch(int(sys.argv[2])) dataset = dataset.prefetch(buffer_size=int(sys.argv[2]) * 100) iterator = dataset.make_initializable_iterator() next_element = iterator.get_next() counter = 0 NUM_EPOCHS = 40 val_acc = 0 acc = 0 count = 0 for epoch in range(NUM_EPOCHS): session.run(iterator.initializer, feed_dict={filename: training_filename}) #x_batch, y_true_batch = data.train.next_batch(batch_size) while True: try: t = time() x_batch, y_true_batch = session.run(next_element) #print("Load time is",time()-t) feed_dict_tr = { x: x_batch, y_true: y_true_batch, keep_prob: 0.3 } _, acc = session.run([optimizer, accuracy], feed_dict=feed_dict_tr) #acc = session.run(accuracy,feed_dict= feed_dict_tr) #print(time()-t) count += 1 #print("Batch %d complete",count) except tf.errors.OutOfRangeError as e: break #print("Training complete. Starting Test set") session.run(iterator.initializer, feed_dict={filename: validation_filename}) val_b = 0 while True: try: x_valid_batch, y_valid_batch = session.run(next_element) feed_dict_val = { x: x_valid_batch, y_true: y_valid_batch, keep_prob: 1 } #val_loss = session.run(cost,feed_dict=feed_dict_val) val_loss, val_acc, valacc3 = session.run( [cost, accuracy2, accuracy3], feed_dict=feed_dict_val) #summary,val_acc = session.run([merged,accuracy],feed_dict=feed_dict_val) #print("Batch Accuracy: ",epoch,val_acc) val_acc3 = session.run(accuracy3, feed_dict=feed_dict_val) print(val_acc, val_acc3) val_b += 1 #writer.add_summary(summary,epoch) #print("Validation bath:",val_b) except tf.errors.OutOfRangeError as e: show_progress(epoch, acc, val_acc, val_loss) saver.save( session, "/home/szaman5/Phytoplankton_Classifier/trained_model/") break
})) return ( np.concatenate(rotated_xs, axis=0), np.concatenate(zoomed_xs, axis=0), ) with tf.Session() as session: training_file = f"{config.DATA_DIR}/train.p" with open(training_file, mode='rb') as f: train = pickle.load(f) X_train, y_train = train["features"], train["labels"] # Need to shuffle because some perturbations will be constant per # batch. X_train, y_train = dataset.shuffle(X_train, y_train) rotated_x, zoomed_x = perturb_x(session, X_train, y_train) print(rotated_x.shape) print(zoomed_x.shape) X_train_augmented = np.concatenate([X_train, rotated_x, zoomed_x], axis=0) y_train_augmented = np.concatenate([y_train, y_train, y_train], axis=0) augmented_training_file = f"/{config.DATA_DIR}/train_augmented.p" with open(augmented_training_file, "wb") as f: pickle.dump( { "features": X_train_augmented, "labels": y_train_augmented, }, f)
"2019-03-24_18-07-29_res_128", "2019-03-24_19-05-22_res_128", ] train_sessions = sess_names[:50] valid_sessions = sess_names[50:] with tf.Session() as sess: sess.run(tf.global_variables_initializer()) prev_sess = None for epoch in range(5000): for data in shuffle( iterate_sequences(iterate_dataset('data-good'), window_size=1)): x0_val = data[0]['density'] x1_val = data[2]['density'] y_val = data[1]['density'] if sess_names in train_sessions: _, summary = sess.run([optimizer, summaries], feed_dict={ x0: x0_val, x1: x1_val, y: y_val, }) writer_train.add_summary(summary, epoch)