def testNestedOutputs(self): ds = Dataset.zip((Dataset.range(4), Dataset.zip((Dataset.range(4), Dataset.range(4))))) total = 0 # The Iterator will return a nested structure of Tensor objects. # Some funkiness to compare against simple integers. for (i, x) in enumerate(datasets.Iterator(ds)): want = (i, (i, i)) got = (x[0].numpy(), (x[1][0].numpy(), x[1][1].numpy())) self.assertEqual(got, want) total += 1 self.assertEqual(4, total)
def testMultipleIteratorsOnADatasetThatUsesFunctions(self): ds = Dataset.from_tensor_slices([1, 2, 3, 4, 5, 6]).map(math_ops.square) got1 = [x.numpy() for x in datasets.Iterator(ds)] self.assertAllEqual([1, 4, 9, 16, 25, 36], got1) got2 = [x.numpy() for x in datasets.Iterator(ds)] self.assertAllEqual(got1, got2)
def testMapAndFilter(self): def even(x): return math_ops.equal(math_ops.mod(x, 2), 0) it = datasets.Iterator(Dataset.range(8).map(math_ops.square).filter(even)) got = [x.numpy() for x in it] self.assertAllEqual([0, 4, 16, 36], got)
def testMultipleIteratorsOnTheSameDataset(self): ds = Dataset.range(4) it1 = datasets.Iterator(ds) it2 = datasets.Iterator(ds) got = [x.numpy() for x in it1] self.assertAllEqual([0, 1, 2, 3], got) got = [x.numpy() for x in it2] self.assertAllEqual([0, 1, 2, 3], got)
def testPyFunc(self): def my_map(inp): return [[x + 1 for x in inp]] ds = Dataset.range(4).map( lambda x: script_ops.py_func(my_map, [[x]], dtypes.int64)) got = [x.numpy() for x in datasets.Iterator(ds)] self.assertAllEqual([[1], [2], [3], [4]], got)
def main(argv=None): ''' ''' main.__doc__ = __doc__ argv = sys.argv if argv is None else sys.argv.extend(argv) desc = main.__doc__ # .format(os.path.basename(__file__)) # CLI parser args = parser_(desc) nranks_per_gpu = args.nranks_per_gpu local_rank = hvd.local_rank() gpu_local_rank = local_rank // nranks_per_gpu print('local_rank, GPU_LOCAL_RANK: {}, {}'.format( local_rank, gpu_local_rank)) # Pin GPU to be used to process local rank (one GPU per process) config = tf.ConfigProto() config.gpu_options.allow_growth = True # config.gpu_options.visible_device_list = str(hvd.local_rank()) config.gpu_options.visible_device_list = str(gpu_local_rank) K.set_session(tf.Session(config=config)) # input image dimensions img_rows, img_cols, img_chns = 28, 28, 1 # number of convolutional filters to use filters = 64 # convolution kernel size num_conv = 3 hvdsize = hvd.size() batch_size = 128 # 100 if K.image_data_format() == 'channels_first': original_img_size = (img_chns, img_rows, img_cols) else: original_img_size = (img_rows, img_cols, img_chns) latent_dim = 2 intermediate_dim = 128 epsilon_std = 1.0 epochs = args.epochs # 5 # train the VAE on MNIST digits (x_train, _), (x_test, y_test) = mnist.load_data() x_train = x_train.astype('float32') / 255. x_train = x_train.reshape((x_train.shape[0],) + original_img_size) x_test = x_test.astype('float32') / 255. x_test = x_test.reshape((x_test.shape[0],) + original_img_size) if hvd.rank() == 0: print('x_train.shape:', x_train.shape) train_samples = x_train.shape[0] # steps_per_epoch = train_samples // batch_size // hvdsize speedupopt = args.speedup if speedupopt == SpeedupOpts.imgspersec: steps_per_epoch = train_samples // batch_size else: steps_per_epoch = int(round( float(train_samples) / batch_size / hvdsize + 0.5)) # Create the dataset and its associated one-shot iterator. buffer_size = 10000 dataset = Dataset.from_tensor_slices(x_train) dataset = dataset.repeat() dataset = dataset.shuffle(buffer_size) dataset = dataset.batch(batch_size) iterator = dataset.make_one_shot_iterator() x_train_batch = iterator.get_next() ldict = make_shared_layers_dict( img_chns, img_rows, img_cols, batch_size, filters, num_conv, intermediate_dim, latent_dim, epsilon_std) # ldict is a dictionary that holds all layers. Since these layers are # instantiated once, they are shared amongs vae, encoder, and generator. x = Input(tensor=x_train_batch) vae = make_vae(ldict, x) # : :type vae: Model lr = 0.001 # * hvdsize opt = tf.train.RMSPropOptimizer(lr) # Add Horovod Distributed Optimizer. opt = hvd.DistributedOptimizer(opt) # , use_locking=True) opt = TFOptimizer(opt) # opt = RMSprop(lr) # Add Horovod Distributed Optimizer. # opt = hvd_keras.DistributedOptimizer(opt) # , use_locking=True) vae.compile(optimizer=opt, loss=None) if hvd.rank() == 0: vae.summary() callbacks = [] if hvd.rank() == 0: callbacks += [BatchTiming(), SamplesPerSec(batch_size * hvdsize)] sess = K.get_session() sess.run(hvd.broadcast_global_variables(0)) # Fit the model using data from the TF data tensors. vae.fit(steps_per_epoch=steps_per_epoch, epochs=epochs, callbacks=callbacks) if hvd.rank() == 0: x = Input(shape=original_img_size) vae_val = make_vae(ldict, x) vae_val.compile(optimizer=opt, loss=None) loss = vae_val.evaluate(x=x_test, y=None, batch_size=batch_size) print('\n\nVAE VALIDATION LOSS: {}'.format(loss)) x = Input(shape=original_img_size) z_mean, _ = get_encoded(ldict, x) encoder = Model(x, z_mean) # : :type encoder: Model decoder_input = Input(shape=(latent_dim,)) x_decoded_mean_squash = get_decoded(ldict, decoder_input) generator = Model(decoder_input, x_decoded_mean_squash) # : :type generator: Model # display a 2D plot of the digit classes in the latent space x_test_encoded = encoder.predict(x_test, batch_size=batch_size) plt.figure(figsize=(6, 6)) plt.scatter(x_test_encoded[:, 0], x_test_encoded[:, 1], c=y_test) plt.colorbar() # plt.show() plt.savefig('vae_scatter.ps') plt.close() # display a 2D manifold of the digits n = 15 # figure with 15x15 digits digit_size = 28 figure = np.zeros((digit_size * n, digit_size * n)) # Linearly spaced coordinates on the unit square were transformed # through the inverse CDF (ppf) of the Gaussian # To produce values of the latent variables z, since the prior of the # latent space is Gaussian grid_x = norm.ppf(np.linspace(0.05, 0.95, n)) grid_y = norm.ppf(np.linspace(0.05, 0.95, n)) for i, yi in enumerate(grid_x): for j, xi in enumerate(grid_y): z_sample = np.array([[xi, yi]]) z_sample = np.tile(z_sample, batch_size).reshape(batch_size, 2) x_decoded = generator.predict(z_sample, batch_size=batch_size) digit = x_decoded[0].reshape(digit_size, digit_size) figure[i * digit_size: (i + 1) * digit_size, j * digit_size: (j + 1) * digit_size] = digit plt.figure(figsize=(10, 10)) plt.imshow(figure, cmap='Greys_r') # plt.show() plt.savefig('vae_digit.ps') plt.close() K.clear_session()
def __init__(self, txt_file, mode, batch_size, num_classes, shuffle=True, buffer_size=1000): """Create a new ImageDataGenerator. Recieves a path string to a text file, which consists of many lines, where each line has first a path string to an image and seperated by a space an integer, referring to the class number. Using this data, this class will create TensrFlow datasets, that can be used to train e.g. a convolutional neural network. Args: txt_file: Path to the text file. mode: Either 'training' or 'validation'. Depending on this value, different parsing functions will be used. batch_size: Number of images per batch. num_classes: Number of classes in the dataset. shuffle: Wether or not to shuffle the data in the dataset and the initial file list. buffer_size: Number of images used as buffer for TensorFlows shuffling of the dataset. Raises: ValueError: If an invalid mode is passed. """ self.txt_file = txt_file self.num_classes = num_classes # retrieve the data from the text file self._read_txt_file() # number of samples in the dataset self.data_size = len(self.labels) # initial shuffling of the file and label lists (together!) if shuffle: self._shuffle_lists() # convert lists to TF tensor self.img_paths = convert_to_tensor(self.img_paths, dtype=dtypes.string) self.labels = convert_to_tensor(self.labels, dtype=dtypes.int32) # create dataset data = Dataset.from_tensor_slices((self.img_paths, self.labels)) # distinguish between train/infer. when calling the parsing functions if mode == 'training': data = data.map(self._parse_function_train, num_threads=8, output_buffer_size=100 * batch_size) elif mode == 'inference': data = data.map(self._parse_function_inference, num_threads=8, output_buffer_size=100 * batch_size) else: raise ValueError("Invalid mode '%s'." % (mode)) # shuffle the first `buffer_size` elements of the dataset if shuffle: data = data.shuffle(buffer_size=buffer_size) # create a new dataset with batches of images data = data.batch(batch_size) self.data = data
def main(add_negative): mnist_data = input_data.read_data_sets('MNIST_data', one_hot=True) if add_negative: train_set = gen_noise_dataset(mnist_data.train) class_n = 11 else: train_set = Dataset.from_tensor_slices( (mnist_data.train.images, mnist_data.train.labels)) class_n = 10 print('Done generating dataset') # Input placeholder X = tf.placeholder(tf.float32, shape=(None, 28 * 28), name='X') keep_prob = tf.placeholder(tf.float32, name='keep_prob') y = tf.placeholder(tf.float32, shape=(None, class_n), name='y') # Model model_var_dict = build_cnn_graph(X, keep_prob=keep_prob, class_n=class_n) y_conv = model_var_dict['y_conv'] h_conv2 = model_var_dict['h_conv2'] # metrics cross_entropy = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits(logits=y_conv, labels=y)) correct_prediction = tf.equal(tf.argmax(y_conv, 1), tf.argmax(y, 1)) accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) # Train OP train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy) # Tensorboard summary tf.summary.scalar('cross entropy', cross_entropy) tf.summary.scalar('training accuracy', accuracy) tf.summary.histogram( 'L2 activation degree by filters', tf.reduce_sum(tf.reduce_mean(h_conv2, axis=0), axis=[0, 1])) merged_summary = tf.summary.merge_all() # RUN! dataset = train_set dataset = dataset.shuffle(buffer_size=8192) dataset = dataset.batch(BATCH_SIZE) iterator = dataset.make_initializable_iterator() next_batch = iterator.get_next() with tf.Session() as sess: sess.run(tf.global_variables_initializer()) summary_writer = tf.summary.FileWriter('mnist/logs/', sess.graph) step_i = 0 for epoch_i in range(EPOCH_N): sess.run(iterator.initializer) while True: try: batch_X, batch_y = sess.run(next_batch) sess.run(train_step, feed_dict={ X: batch_X, y: batch_y, keep_prob: 0.5 }) # monitor if step_i % 100 == 0: train_accuracy, summary = sess.run( [accuracy, merged_summary], feed_dict={ X: batch_X, y: batch_y, keep_prob: 1.0 }) print('step {}, training accuracy {}'.format( step_i, train_accuracy)) summary_writer.add_summary(summary, step_i) step_i += 1 except tf.errors.OutOfRangeError: break # predict predict_data(mnist_data.test, add_negative, accuracy) # Visualize: what input images lead to high activation? debug_W1, debug_b1, debug_W2, debug_b2 = sess.run( ['conv1/W:0', 'conv1/b:0', 'conv2/W:0', 'conv2/b:0']) fn_prfx = '11_' if add_negative else '10_' # reduce mean axis=0 for eliminate mini-batch dimension h_conv1 = model_var_dict['h_conv1'] degrees = tf.reduce_sum(tf.reduce_mean(h_conv1, axis=0), axis=[0, 1]) get_highact_images(degrees, X, keep_prob, fn_prfx + 'max_conv1', debug_b_filters=debug_b1, debug_W_filters=debug_W1.reshape( -1, CONV_1_CHANNEL_N)) h_conv2 = model_var_dict['h_conv2'] degrees = tf.reduce_sum(tf.reduce_mean(h_conv2, axis=0), axis=[0, 1]) # average L2 filter over input (L1) depth when showing in 2D debug_W2 = np.mean(debug_W2, axis=2) get_highact_images(degrees, X, keep_prob, fn_prfx + 'max_conv2', debug_b_filters=debug_b2, debug_W_filters=debug_W2.reshape( -1, CONV_2_CHANNEL_N), first_n=30) h_fc1 = model_var_dict['h_fc1'] degrees = tf.reduce_mean(h_fc1, axis=0) get_highact_images(degrees, X, keep_prob, fn_prfx + 'max_fc1', first_n=30) y_conv = model_var_dict['y_conv'] degrees = tf.reduce_mean(y_conv, axis=0) img_and_scores = get_highact_images(degrees, X, keep_prob, fn_prfx + 'max_y', sort_by_degree=False) for i, (im, sc, w, b) in enumerate(img_and_scores): y_out = sess.run(y_conv, feed_dict={ X: im.reshape(1, 28 * 28), keep_prob: 1.0 }) print('{}: {}'.format(i, np.exp(y_out) / np.sum(np.exp(y_out))))
def dataset_with_label(self, label_int, src_pattern): label = tf.constant(label_int, tf.int32, name="label") lines = Dataset.list_files(src_pattern).flat_map( lambda fn: TextLineDataset(fn)) labels = Dataset.from_tensors(label).repeat() return Dataset.zip((lines, labels))
def validate(model_def): """ Validate my alexnet implementation Args: model_def: the model class/definition """ img_dir = os.path.join('.', 'images') images = [] print "loading images ..." files = fnmatch.filter(os.listdir(img_dir), '*.jpeg') for f in files: print "> " + f img_file = tf.read_file(os.path.join(img_dir, f)) img_decoded = tf.image.decode_jpeg(img_file, channels=3) img_processed = model_def.image_prep.preprocess_image( image=img_decoded, output_height=model_def.image_size, output_width=model_def.image_size, is_training=False ) images.append(img_processed) # create TensorFlow Iterator object images = Dataset.from_tensors(images) iterator = Iterator.from_structure(images.output_types, images.output_shapes) next_element = iterator.get_next() iterator_init_op = iterator.make_initializer(images) # create the model and get scores (pipe to softmax) model = model_def(next_element) scores = model.get_final_op() softmax = tf.nn.softmax(scores) print 'start validation ...' with tf.Session() as sess: # Initialize all variables and the iterator sess.run(tf.global_variables_initializer()) sess.run(iterator_init_op) # Load the pretrained weights into the model model.load_initial_weights(sess) # run the graph probs = sess.run(softmax) # sometime we have an offset if model_def is ResNetV2: offset = len(class_names) - len(probs[0][0][0]) else: offset = len(class_names) - len(probs[0]) # print the results for prob in probs: if model_def is ResNetV2: prob = prob[0][0] best_index = np.argmax(prob) print "> " + class_names[best_index+offset] + " -> %.4f" %prob[best_index]
def testBasic(self): got = [] for t in datasets.Iterator(Dataset.range(4)): got.append(t.numpy()) self.assertAllEqual([0, 1, 2, 3], got)
def random_images(): return None, Dataset.from_tensor_slices( np.random.random((32, 784,)).astype(np.float32))
# convert the label to one-hot encoding one_hot = tf.one_hot(label, NUM_CLASSES) # read the img from file img_file = tf.read_file(img_path) img_decoded = tf.image.decode_image(img_file) return img_decoded, one_hot # Define tensorflow constants for each dataset train_imgs = tf.constant(img_arrary) train_labels = tf.constant(lbl_arrary) # create TensorFlow Dataset objects tf_dataset = Dataset.from_tensor_slices((train_imgs, train_labels)) # Parse dataset tf_dataset = tf_dataset.map(map_func=input_parser, num_threads=None) # Repeats the dataset n times (optional) #tf_dataset = tf_dataset.repeat(2) # Shuffle dataset (optional) tf_dataset = tf_dataset.shuffle(buffer_size=len(img_arrary) * 2) # Set batch size tf_dataset = tf_dataset.batch(BATCH_SIZE) # Define an iterator iterator = tf_dataset.make_initializable_iterator()
def main(mode): data_dir = "data/challenger.ai" bin_size = 14 with open(os.path.join(data_dir, 'word_to_idx.pkl'), 'rb') as f: word_to_idx = pickle.load(f) with open( os.path.join( data_dir, "annotations/caption_%s_annotations_20170902.json" % mode)) as f: annotations = json.load(f) image_ids = [ann['image_id'] for ann in annotations] caps = [ann['caption'] for ann in annotations] def my_split(text): text = text.decode("utf-8") # todo: take care of the unknown character. idx = [word_to_idx.get(ch, 0) for ch in text] idx.insert(0, word_to_idx['<START>']) idx.append(word_to_idx['<END>']) return np.array(idx, dtype=np.int32) def parse(img_id, caps): filename = os.path.join(data_dir, "image/%s" % mode) + "/" + img_id image = tf.image.decode_jpeg(tf.read_file(filename), channels=3) image = tf.image.convert_image_dtype(image, dtype=tf.float32) image = tf.subtract(image, 0.5) image = tf.multiply(image, 2.0) splitted_caps = tuple( map(lambda c: tf.py_func(my_split, [c], tf.int32, stateful=False), tf.unstack(caps))) return { 'img_id': img_id, 'raw_img': image, 'raw_caps': caps, 'cap_idx': splitted_caps } # return img_id, image, caps, splitted_caps it = Dataset.from_tensor_slices( (image_ids, caps)).map(parse).make_one_shot_iterator() feat_tensor_dict = it.get_next() arg_scope = inception_v4_arg_scope() with slim.arg_scope(arg_scope): final_conv_layer, end_points = inception_v4_base( tf.expand_dims(feat_tensor_dict['raw_img'], 0)) feats_tensor = spatial_pyramid_pooling(final_conv_layer, [bin_size], mode='avg') feats_tensor = tf.reshape(feats_tensor, shape=(-1, bin_size * bin_size, 1536)) sess = tf.Session() variables_to_restore = slim.get_variables_to_restore( exclude=['global_step']) init_fn = assign_from_checkpoint_fn("data/model/inception_v4.ckpt", variables_to_restore) init_fn(sess) tfrecord_filename_base = 'data/challenger.ai/tfrecords/%s_feat_14x14x1536_inception_v4' % mode writer = tf.python_io.TFRecordWriter(tfrecord_filename_base + "-0.tfrecords") i = 0 while True: try: feature_dict, feats = sess.run((feat_tensor_dict, feats_tensor)) example = tf.train.Example(features=tf.train.Features( feature={ 'img_id': tf.train.Feature(bytes_list=tf.train.BytesList( value=[feature_dict['img_id']])), # 'raw_img': tf.train.Feature( # bytes_list=tf.train.BytesList(value=[feature_dict['raw_img'].tostring()])), 'img_feats': tf.train.Feature(bytes_list=tf.train.BytesList( value=[feats.tostring()])), 'raw_caps': tf.train.Feature(bytes_list=tf.train.BytesList( value=feature_dict['raw_caps'])), 'cap_idx': tf.train.Feature(bytes_list=tf.train.BytesList(value=[ idx.tostring() for idx in feature_dict['cap_idx'] ])), })) writer.write(example.SerializeToString()) print(i) i += 1 if i % 10000 == 0: writer.close() writer = tf.python_io.TFRecordWriter(tfrecord_filename_base + "-%d.tfrecords" % i) except OutOfRangeError as e: print(e) break writer.close()
def transform_test_dataset(): data_dir = "data/challenger.ai" bin_size = 14 filenames = [ fn.split('/')[-1] for fn in glob.glob(os.path.join(data_dir, "image/test/*")) ] def parse(img_id): filename = os.path.join(data_dir, "image/test/") + img_id image = tf.image.decode_jpeg(tf.read_file(filename), channels=3) image = tf.image.convert_image_dtype(image, dtype=tf.float32) image = tf.subtract(image, 0.5) image = tf.multiply(image, 2.0) return { 'img_id': img_id, 'raw_img': image, } it = Dataset.from_tensor_slices(filenames).map( parse).make_one_shot_iterator() feat_tensor_dict = it.get_next() arg_scope = inception_v4_arg_scope() with slim.arg_scope(arg_scope): final_conv_layer, end_points = inception_v4_base( tf.expand_dims(feat_tensor_dict['raw_img'], 0)) feats_tensor = spatial_pyramid_pooling(final_conv_layer, [bin_size], mode='avg') feats_tensor = tf.reshape(feats_tensor, shape=(-1, bin_size * bin_size, 1536)) sess = tf.Session() variables_to_restore = slim.get_variables_to_restore( exclude=['global_step']) init_fn = assign_from_checkpoint_fn("data/model/inception_v4.ckpt", variables_to_restore) init_fn(sess) tfrecord_filename_base = 'data/challenger.ai/tfrecords/test_feat_14x14x1536_inception_v4' writer = tf.python_io.TFRecordWriter(tfrecord_filename_base + "-0.tfrecords") i = 0 while True: try: feature_dict, feats = sess.run((feat_tensor_dict, feats_tensor)) example = tf.train.Example(features=tf.train.Features( feature={ 'img_id': tf.train.Feature(bytes_list=tf.train.BytesList( value=[feature_dict['img_id']])), # 'raw_img': tf.train.Feature( # bytes_list=tf.train.BytesList(value=[feature_dict['raw_img'].tostring()])), 'img_feats': tf.train.Feature(bytes_list=tf.train.BytesList( value=[feats.tostring()])), })) writer.write(example.SerializeToString()) print(i) i += 1 if i % 10000 == 0: writer.close() writer = tf.python_io.TFRecordWriter(tfrecord_filename_base + "-%d.tfrecords" % i) except OutOfRangeError as e: print(e) break writer.close()
DIR = '/home/mtb/test.npy' # a = np.asarray([1,2,3,4,5,6]) # np.save(DIR, a) # b =np.load(DIR) # print b.shape filename = DIR # input_=np.load(DIR + 'input.npy') output_ = np.load(DIR + 'output.npy') input_placeholder = tf.placeholder(tf.float32, shape=None) # output_placeholder = tf.placeholder(tf.float32, shape = output_.shape) a = Dataset.from_tensors(input_placeholder) b = Dataset.from_tensors(output_placeholder) # c = Dataset.zip((a,b)) # batch_ = a.batch(100) iterator = a.make_initializable_iterator() next_elem = iterator.get_next() # # b = Dataset.from_tensor_slices(input_placeholder) # # tf.Session().run(tf.global_variables_initializer(),) # # b = Dataset.from_tensor_slices(tf.Session().run(v,{input_placeholder: input_})) with tf.Session() as sess: sess.run(tf.global_variables_initializer()) # print a.shape # print sess.run(next_elem, {input_placeholder:input_}) input_dict = OrderedDict()
def train_deeper_better(train_data, train_labels, test_data, test_labels, params): """Same as 'train_deeper', but now with tf.contrib.data.Dataset input pipeline.""" default_params = { 'regularization_coeff': 0.00001, 'keep_prob': 0.5, 'batch_size': 128, 'fc1_size': 2048, 'fc2_size': 1024, 'fc3_size': 1024, 'fc4_size': 1024, 'fc5_size': 512, 'activation': 'relu', } activation_funcs = { 'relu': tf.nn.relu, 'tanh': tf.nn.tanh, } def get_param(name): if name in params: return params[name] logger.warning('%s not found in param, use default value %r', name, default_params[name]) return default_params[name] regularization_coeff = get_param('regularization_coeff') keep_prob_param = get_param('keep_prob') batch_size = int(get_param('batch_size')) fc1_size = int(get_param('fc1_size')) fc2_size = int(get_param('fc2_size')) fc3_size = int(get_param('fc3_size')) fc4_size = int(get_param('fc4_size')) fc5_size = int(get_param('fc5_size')) activation_func = activation_funcs[get_param('activation')] save_restore = False time_limit_seconds = 3600 saver_path = join(SAVER_FOLDER, train_deeper_better.__name__) graph = tf.Graph() with graph.as_default(): tf.set_random_seed(52) global_step_tensor = tf.contrib.framework.get_or_create_global_step() epoch_tensor = tf.Variable(0, trainable=False, name='epoch') next_epoch = tf.assign_add(epoch_tensor, 1) # dataset definition dataset = Dataset.from_tensor_slices({ 'x': train_data, 'y': train_labels }) dataset = dataset.shuffle(buffer_size=10000) dataset = dataset.batch(batch_size) iterator = dataset.make_initializable_iterator() sample = iterator.get_next() x = sample['x'] y = sample['y'] # actual computation graph keep_prob = tf.placeholder(tf.float32) is_training = tf.placeholder(tf.bool, name='is_training') regularizer = tf.contrib.layers.l2_regularizer( scale=regularization_coeff) def fully_connected(x, size, name): return dense_regularized( x, size, is_training, keep_prob, regularizer, name, activation_func, ) fc1 = fully_connected(x, fc1_size, 'fc1') fc2 = fully_connected(fc1, fc2_size, 'fc2') fc3 = fully_connected(fc2, fc3_size, 'fc3') fc4 = fully_connected(fc3, fc4_size, 'fc4') fc5 = fully_connected(fc4, fc5_size, 'fc5') logits = dense(fc5, NUM_CLASSES, regularizer, 'logits') layer_summaries(logits, 'logits_summaries') with tf.name_scope('accuracy'): accuracy = tf.reduce_mean( tf.cast(tf.equal(tf.argmax(y, 1), tf.argmax(logits, 1)), tf.float32), ) accuracy_percent = 100 * accuracy tf.summary.scalar('accuracy_percent', accuracy_percent) with tf.name_scope('loss'): regularization_losses = tf.get_collection( tf.GraphKeys.REGULARIZATION_LOSSES) regularization_loss = tf.reduce_sum(regularization_losses) cross_entropy_loss = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=y), ) loss = cross_entropy_loss + regularization_loss tf.summary.scalar('regularization_loss', regularization_loss) tf.summary.scalar('cross_entropy_loss', cross_entropy_loss) tf.summary.scalar('loss', loss) update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) with tf.control_dependencies(update_ops): # ensures that we execute the update_ops before performing the train_op # needed for batch normalization (apparently) optimizer = tf.train.AdamOptimizer(learning_rate=(1e-4), epsilon=1e-3) train_op = optimizer.minimize(loss, global_step=global_step_tensor) all_summaries = tf.summary.merge_all() train_writer = tf.summary.FileWriter(join(SUMMARY_FOLDER, 'train')) batch_writer = tf.summary.FileWriter(join(SUMMARY_FOLDER, 'batch')) test_writer = tf.summary.FileWriter(join(SUMMARY_FOLDER, 'test')) saver = tf.train.Saver(max_to_keep=3) test_accuracy = 0 best_accuracy = 0 with tf.Session(graph=graph) as sess: restored = False if save_restore: try: saver.restore( sess, tf.train.latest_checkpoint(checkpoint_dir=SAVER_FOLDER)) restored = True except ValueError as exc: logger.info('Could not restore previous session! %r', exc) logger.info('Starting from scratch!') if not restored: tf.global_variables_initializer().run() logger.info('Starting training...') start_time = time.time() def enough(): if time_limit_seconds is None: return False elapsed = time.time() - start_time return elapsed > time_limit_seconds epoch = epoch_tensor.eval() new_epoch = True while not enough(): logger.info('Starting new epoch #%d!', epoch) sess.run(iterator.initializer, feed_dict={}) while not enough(): step = tf.train.global_step(sess, tf.train.get_global_step()) try: sess.run(train_op, feed_dict={ keep_prob: keep_prob_param, is_training: True }) if new_epoch: new_epoch = False l, reg_l, ac, summaries = sess.run( [ loss, regularization_loss, accuracy_percent, all_summaries ], feed_dict={ keep_prob: keep_prob_param, is_training: False }, ) batch_writer.add_summary(summaries, global_step=step) logger.info( 'Minibatch loss: %f, reg loss: %f, accuracy: %.2f%%', l, reg_l, ac, ) except tf.errors.OutOfRangeError: logger.info('End of epoch #%d', epoch) break # end of epoch previous_epoch = epoch epoch = next_epoch.eval() new_epoch = True if previous_epoch % 5 == 0 and save_restore: saver.save(sess, saver_path, global_step=previous_epoch) def get_eval_dict(data, labels): """Data for evaluation.""" return {x: data, y: labels, keep_prob: 1, is_training: False} train_l, train_ac, summaries = sess.run( [loss, accuracy_percent, all_summaries], feed_dict=get_eval_dict(train_data[:10000], train_labels[:10000]), ) train_writer.add_summary(summaries, global_step=step) test_l, test_accuracy, summaries = sess.run( [loss, accuracy_percent, all_summaries], feed_dict=get_eval_dict(test_data, test_labels), ) test_writer.add_summary(summaries, global_step=step) best_accuracy = max(best_accuracy, test_accuracy) logger.info('Train loss: %f, train accuracy: %.2f%%', train_l, train_ac) logger.info( 'Test loss: %f, TEST ACCURACY: %.2f%% BEST ACCURACY %.2f%% <<<<<<<', test_l, test_accuracy, best_accuracy, ) return best_accuracy
def testBasic(self): got = [] for t in datasets.Iterator(Dataset.range(4)): got.append(t.numpy()) self.assertAllEqual([0, 1, 2, 3], got)
def get_inputs(paras_file, titles_file, embedding_file, params): ''' This function returns a dictionary of input texts, inputs required for tensorflow operation and input placeholder and operations :param paras_file: This the file for the input text :param titles_file: This is the file for the input summaries :param embedding_file: This is the embedding file like Glove, word2vec, etc :param params: These are the flags required for the input text batch iterator :return: inputs: This is the dictionary corresponding to the input training and validation texts and summaries inputs_tf: This is the dictionary with the tensorflow variables which will be used as inputs to the RNN summarizer inputs_ph_op: This is the dictionary with the tensorflow placeholders and operations which we will use to declare the session ''' def _input_parse_function(para, title): ''' This function is used to parse the input :param para: This is the input paragraph :param title: This is the input summary :return: A tuple for para as well as for title. This tuple consists of following two things: i) A list consisting of all the input words ii) Number of words in the input ''' def parse_input(text, src=None): words = tf.string_split([text]).values size = tf.size(words) words = vocab.lookup(words) if src == 'Target': words = tf.concat([tf.constant(SOS_INDEX, dtype=tf.int64, shape=[1, ]), words], axis=0) return (words, size) return (parse_input(para), parse_input(title, src='Target')) if paras_file.endswith('.pickle') or paras_file.endswith('pkl'): input_paras = pickle.load(open(paras_file,'rb')) input_titles = pickle.load(open(titles_file, 'rb')) print("Data is loaded. It has {} rows".format(len(input_paras))) input_paras, val_paras, input_titles, val_titles = train_test_split(input_paras, input_titles, test_size=params.test_size, train_size=params.train_size, shuffle=False) ################### ## Getting VOCAB ## ################### vocab, embedding = utils.loadGlove(embedding_file, params) embedding_W = tf.Variable(tf.constant(0.0, shape=embedding.shape), trainable=False, name='embedding_w') embedding_ph = tf.placeholder(tf.float32, embedding.shape) embedding_init = embedding_W.assign(embedding_ph) # embedding_W = tf.Variable(embedding, trainable=False, name='embedding') vocab = tf.contrib.lookup.index_table_from_tensor(mapping=vocab, default_value=UNK_INDEX) ####################### ## Data manipulation ## ####################### paras_ph = tf.placeholder(tf.string, shape=(None,)) titles_ph = tf.placeholder(tf.string, shape=(None,)) batch_size = tf.placeholder(tf.int32, shape=()) data = Dataset.from_tensor_slices((paras_ph, titles_ph)) data = data.map(_input_parse_function, num_parallel_calls=8).prefetch(params.batch_size * 10) data = data.padded_batch(tf.cast(batch_size, dtype=tf.int64), padded_shapes=((tf.TensorShape([None]), tf.TensorShape([])), (tf.TensorShape([None]), tf.TensorShape([]))), padding_values=((tf.to_int64(EOS_INDEX), 0), (tf.to_int64(EOS_INDEX), 0))) iterator = data.make_initializable_iterator() (para_batch, para_length), (title_batch, title_length) = iterator.get_next() para_embedding = tf.nn.embedding_lookup(embedding_W, para_batch) title_embedding = tf.nn.embedding_lookup(embedding_W, title_batch) inputs = dict() inputs['input_paras'] = input_paras inputs['val_paras'] = val_paras inputs['input_titles'] = input_titles inputs['val_titles'] = val_titles inputs['embedding'] = embedding inputs_tf = dict() inputs_tf['para_embedding'] = para_embedding inputs_tf['title_embedding'] = title_embedding inputs_tf['para_batch'], inputs_tf['para_length'] = para_batch, para_length inputs_tf['title_batch'], inputs_tf['title_length'] = title_batch, title_length inputs_tf['embedding_W'] = embedding_W inputs_tf['batch_size'] = batch_size # Repeating batch_size as it is also required for the model building inputs_tf['iterator'] = iterator inputs_ph_op = dict() inputs_ph_op['paras_ph'], inputs_ph_op['titles_ph'] = paras_ph, titles_ph inputs_ph_op['embedding_ph'], inputs_ph_op['embedding_init'] = embedding_ph, embedding_init inputs_ph_op['batch_size'] = batch_size return inputs, inputs_tf, inputs_ph_op
def load_tfrecord(filename): #Create a dataset from the file. return Dataset.from_tensor_slices(np.arange(1, 16, 1))
def __init__(self, csv_file, data_dir, datatype, mode, batch_size, shuffle, preprocess_conditions=False, buffer_size=1000): self.preprocess_conditions = preprocess_conditions self.data_dir = data_dir self.datatype = datatype self.condition_dir = os.path.join(self.data_dir, csv_file) self.condition_names = list(datatype['names'][1:]) self.mode = mode self._read_csv_file() self.data_size_train = len(self.train_filename) self.data_size_test = len(self.test_filename) if shuffle: self._shuffle_lists() self.train_x = ["../Postprocess/x_train/" + s for s in self.train_filename] self.train_y = ["../Postprocess/y_train/" + s for s in self.train_filename] self.train_gen = ["../Postprocess/gen_train/" + s for s in self.train_filename] self.train_dis = ["../Postprocess/dis_train/" + s for s in self.train_filename] self.test_x = ["../Postprocess/x_test/" + s for s in self.test_filename] self.test_y = ["../Postprocess/y_test/" + s for s in self.test_filename] self.test_gen = ["../Postprocess/gen_test/" + s for s in self.test_filename] self.test_dis = ["../Postprocess/dis_test/" + s for s in self.test_filename] # convert lists to TF tensor self.train_x = convert_to_tensor(self.train_x, dtype=dtypes.string) self.train_y = convert_to_tensor(self.train_y, dtype=dtypes.string) self.train_gen = convert_to_tensor(self.train_gen, dtype=dtypes.string) self.train_dis = convert_to_tensor(self.train_dis, dtype=dtypes.string) self.train_filename = convert_to_tensor(self.train_filename, dtype=dtypes.string) self.test_x = convert_to_tensor(self.test_x, dtype=dtypes.string) self.test_y = convert_to_tensor(self.test_y, dtype=dtypes.string) self.test_gen = convert_to_tensor(self.test_gen, dtype=dtypes.string) self.test_dis = convert_to_tensor(self.test_dis, dtype=dtypes.string) self.test_filename = convert_to_tensor(self.test_filename, dtype=dtypes.string) # create dataset train_data = Dataset.from_tensor_slices((self.train_x, self.train_y, self.train_gen, self.train_dis, self.train_filename)) test_data = Dataset.from_tensor_slices((self.test_x, self.test_y, self.test_gen, self.test_dis, self.test_filename)) if mode == 'training': data = train_data.map(self._parse_function_train, num_threads=2, output_buffer_size=2*batch_size) elif mode == 'inference': data = test_data.map(self._parse_function_inference, num_threads=2, output_buffer_size=2*batch_size) else: raise ValueError("Invalid mode '%s'." % (mode)) if shuffle: data = data.shuffle(buffer_size=buffer_size) data = data.batch(batch_size) self.data = data
def main(argv=None): ''' ''' main.__doc__ = __doc__ argv = sys.argv if argv is None else sys.argv.extend(argv) desc = main.__doc__ # .format(os.path.basename(__file__)) # CLI parser args = parser_(desc) mgpu = 1 if getattr(args, 'mgpu', None) is None else args.mgpu # input image dimensions img_rows, img_cols, img_chns = 28, 28, 1 # number of convolutional filters to use filters = 64 # convolution kernel size num_conv = 3 gpus_list = get_available_gpus(mgpu) ngpus = len(gpus_list) batch_size = 128 * ngpus if K.image_data_format() == 'channels_first': original_img_size = (img_chns, img_rows, img_cols) else: original_img_size = (img_rows, img_cols, img_chns) latent_dim = 2 intermediate_dim = 128 epsilon_std = 1.0 epochs = args.epochs # 5 # train the VAE on MNIST digits (x_train, _), (x_test, y_test) = mnist.load_data() x_train = x_train.astype('float32') / 255. x_train = x_train.reshape((x_train.shape[0],) + original_img_size) x_test = x_test.astype('float32') / 255. x_test = x_test.reshape((x_test.shape[0],) + original_img_size) print('x_train.shape:', x_train.shape) train_samples = x_train.shape[0] steps_per_epoch = int(round(float(train_samples) / batch_size + 0.5)) # Create the dataset and its associated one-shot iterator. buffer_size = 10000 dataset = Dataset.from_tensor_slices(x_train) dataset = dataset.repeat() dataset = dataset.shuffle(buffer_size) dataset = dataset.batch(batch_size) iterator = dataset.make_one_shot_iterator() x_train_batch = iterator.get_next() ldict = make_shared_layers_dict( img_chns, img_rows, img_cols, batch_size, filters, num_conv, intermediate_dim, latent_dim, epsilon_std) # ldict is a dictionary that holds all layers. Since these layers are # instantiated once, they are shared amongs vae, encoder, and generator. x = Input(tensor=x_train_batch) vae_serial = make_vae(ldict, x) # : :type vae: Model vae = make_parallel(vae_serial, gpus_list) lr = 0.001 * ngpus opt = RMSprop(lr) # 'rmsprop' # opt = tf.train.RMSPropOptimizer(lr) # opt = TFOptimizer(opt) vae.compile(optimizer=opt, loss=None) # vae.summary() print_mgpu_modelsummary(vae) callbacks = [BatchTiming(), SamplesPerSec(batch_size)] # Fit the model using data from the TF data tensors. vae.fit(steps_per_epoch=steps_per_epoch, epochs=epochs, callbacks=callbacks) x = Input(shape=original_img_size) vae_val = make_vae(ldict, x) vae_val.compile(optimizer=opt, loss=None) loss = vae_val.evaluate(x=x_test, y=None, batch_size=batch_size // ngpus) print('\n\nVAE VALIDATION LOSS: {}'.format(loss)) x = Input(shape=original_img_size) z_mean, _ = get_encoded(ldict, x) encoder = Model(x, z_mean) # : :type encoder: Model decoder_input = Input(shape=(latent_dim,)) x_decoded_mean_squash = get_decoded(ldict, decoder_input) generator = Model(decoder_input, x_decoded_mean_squash) # : :type generator: Model # display a 2D plot of the digit classes in the latent space x_test_encoded = encoder.predict(x_test, batch_size=batch_size) plt.figure(figsize=(6, 6)) plt.scatter(x_test_encoded[:, 0], x_test_encoded[:, 1], c=y_test) plt.colorbar() # plt.show() plt.savefig('vae_scatter.ps') plt.close() # display a 2D manifold of the digits n = 15 # figure with 15x15 digits digit_size = 28 figure = np.zeros((digit_size * n, digit_size * n)) # Linearly spaced coordinates on the unit square were transformed through # the inverse CDF (ppf) of the Gaussian # To produce values of the latent variables z, since the prior of the # latent space is Gaussian grid_x = norm.ppf(np.linspace(0.05, 0.95, n)) grid_y = norm.ppf(np.linspace(0.05, 0.95, n)) for i, yi in enumerate(grid_x): for j, xi in enumerate(grid_y): z_sample = np.array([[xi, yi]]) z_sample = np.tile(z_sample, batch_size).reshape(batch_size, 2) x_decoded = generator.predict(z_sample, batch_size=batch_size) digit = x_decoded[0].reshape(digit_size, digit_size) figure[i * digit_size: (i + 1) * digit_size, j * digit_size: (j + 1) * digit_size] = digit plt.figure(figsize=(10, 10)) plt.imshow(figure, cmap='Greys_r') # plt.show() plt.savefig('vae_digit.ps') plt.close()
x_shuffled, y_shuffled = shuffleDataset(x, y) # Split 60 / 40 split_index = int(len(x_shuffled) * (40 / 100.0)) x_train, x_val_test = x_shuffled[split_index:], x_shuffled[:split_index] y_train, y_val_test = y_shuffled[split_index:], y_shuffled[:split_index] # Split 40 into dev and test sets split_index_val_test = int(len(x_val_test) * (50 / 100.0)) x_val, x_test = x_val_test[split_index_val_test:], x_val_test[:split_index_val_test] y_val, y_test = y_val_test[split_index_val_test:], y_val_test[:split_index_val_test] training_data = (Dataset.from_tensor_slices((x_train, y_train)) .shuffle(buffer_size=10) .batch(args.batch_size) .make_initializable_iterator()) validation_data = (Dataset.from_tensor_slices((x_val, y_val)) .shuffle(buffer_size=10) .batch(args.batch_size) .make_initializable_iterator()) next_element_training = training_data.get_next() next_element_validation = validation_data.get_next() del x, y words_per_document = get_size(x_train) num_classes = get_size(y_train) vocabulary_size = len(vocab_dset) filter_sizes = list(map(int, args.filter_sizes.split(",")))
training_measurements = f['measurements'] training_measurements = np.squeeze(training_measurements).transpose([1,0]) training_patches = f['patches_vec'] training_patches = np.squeeze(training_patches).transpose([1,0]) f = sio.loadmat(os.path.dirname(os.path.abspath(__file__)) + '/dataset/validation_dataset.mat') validation_measurements = f['measurements'] validation_measurements = np.squeeze(validation_measurements).transpose([1,0]) validation_patches = f['patches_vec'] validation_patches = np.squeeze(validation_patches).transpose([1,0]) measurements_placeholder = tf.placeholder(training_measurements.dtype, [None, np.ceil(measurement_rate*(blockSize**2))], name='measurements') patches_placeholder = tf.placeholder(training_patches.dtype, [None, blockSize**2], name='patches') training_dataset = Dataset.from_tensor_slices((measurements_placeholder, patches_placeholder)).batch(50) validation_dataset = Dataset.from_tensor_slices((measurements_placeholder, patches_placeholder)).batch(1000) nEpochs = 20 iterator = Iterator.from_structure(training_dataset.output_types, training_dataset.output_shapes) next_measurement, next_patch = iterator.get_next() training_init_op = iterator.make_initializer(training_dataset) validation_init_op = iterator.make_initializer(validation_dataset) def build_phi(patch): measurement = tf.layers.dense(patch, np.ceil(measurement_rate * blockSize ** 2), activation=tf.nn.relu, use_bias=False, kernel_initializer=tf.random_normal_initializer(mean=0.0, stddev=0.001, dtype=tf.float64), name='fc_phi') return measurement
def model_fn(features, labels, mode, params, config): feat_tensor = caption_tensor = cap_idx_tensor = cap_len_tensor = None scaffold = None bin_size = 8 if mode == ModeKeys.TRAIN or mode == ModeKeys.EVAL: cap_lens = labels["index"].map(lambda t: tf.size(t)) # todo: cannot utilize GPU to accelerate input pipeline, so train 1 by 1 # def extract_feats(image): # with tf.device("/gpu:0"): # _, end_points = vgg.vgg_16(tf.expand_dims(image, 0), # is_training=(mode == ModeKeys.TRAIN), # spatial_squeeze=False) # final_conv_layer = end_points['vgg_16/conv5/conv5_3'] # feats = spatial_pyramid_pooling(final_conv_layer, [bin_size], mode='avg') # return tf.reshape(feats, shape=(bin_size * bin_size, tf.shape(final_conv_layer)[-1])) # features = features.map(extract_feats) datasets = (features, labels["raw"], labels["index"], cap_lens) # todo: 512 is the feature depth, should not hard code here # pad_size = ((bin_size * bin_size, 512), (), (None,), ()) pad_size = ((None, None, 3), (), (None, ), ()) # todo: cannot utilize GPU to accelerate input pipeline, so train 1 by 1 batches = Dataset.zip(datasets) \ .shuffle(buffer_size=200 * params.batch_size) \ .padded_batch(1, pad_size) if mode == ModeKeys.TRAIN: train_iterator = batches \ .repeat() \ .make_initializable_iterator() feat_tensor, caption_tensor, cap_idx_tensor, cap_len_tensor = \ train_iterator.get_next() tf.add_to_collection("train_initializer", train_iterator.initializer) if mode == ModeKeys.EVAL: val_iterator = batches \ .make_initializable_iterator() feat_tensor, caption_tensor, cap_idx_tensor, cap_len_tensor = \ val_iterator.get_next() tf.add_to_collection("val_initializer", val_iterator.initializer) scaffold = tf.train.Scaffold(init_op=val_iterator.initializer) if mode == ModeKeys.INFER: batches = features.batch(params.batch_size) infer_iterator = batches.make_initializable_iterator() feat_tensor = infer_iterator.get_next() tf.add_to_collection("infer_initializer", infer_iterator.initializer) feat_tensor = _extract_feats(bin_size, feat_tensor, mode) if mode == ModeKeys.TRAIN: variables_to_restore = slim.get_variables_to_restore( exclude=['global_step']) init_fn = assign_from_checkpoint_fn(params.vgg_model_path, variables_to_restore) # signature of sc scaffold = tf.train.Scaffold(init_fn=lambda _, sess: init_fn(sess)) loss_op = None train_op = None predictions = None model = AttendTell(vocab_size=params.vocab_size, selector=params.selector, dropout=params.dropout, ctx2out=params.ctx2out, prev2out=params.prev2out, hard_attention=params.hard_attention, mode=mode) if mode != ModeKeys.INFER: if params.use_sampler: outputs = model.build_train(feat_tensor, cap_idx_tensor, use_generated_inputs=True) else: outputs = model.build_train(feat_tensor, cap_idx_tensor, use_generated_inputs=False) loss_op = create_loss(outputs, cap_idx_tensor, cap_len_tensor) train_op = _get_train_op(loss_op, params.learning_rate, params.hard_attention) else: outputs = model.build_infer(feat_tensor) predictions = tf.argmax(outputs, axis=-1) return EstimatorSpec(mode=mode, predictions=predictions, loss=loss_op, train_op=train_op, scaffold=scaffold)
import tensorflow as tf from tensorflow.contrib.data import Dataset, Iterator # Toy data train_imgs = tf.constant([ 'train/img1.png', 'train/img2.png', 'train/img3.png', 'train/img4.png', 'train/img5.png', 'train/img6.png' ]) train_labels = tf.constant([0, 0, 0, 1, 1, 1]) val_imgs = tf.constant( ['val/img1.png', 'val/img2.png', 'val/img3.png', 'val/img4.png']) val_labels = tf.constant([0, 0, 1, 1]) # create TensorFlow Dataset objects tr_data = Dataset.from_tensor_slices((train_imgs, train_labels)) val_data = Dataset.from_tensor_slices((val_imgs, val_labels)) # create TensorFlow Iterator object iterator = Iterator.from_structure(tr_data.output_types, tr_data.output_shapes) next_element = iterator.get_next() # create two initialization ops to switch between the datasets training_init_op = iterator.make_initializer(tr_data) validation_init_op = iterator.make_initializer(val_data) with tf.Session() as sess: # initialize the iterator on the training data sess.run(training_init_op)
def model_fn_inner(features, labels, mode, params, config): feat_tensor = cap_idx_tensor = cap_len_tensor = None scaffold = None if mode == ModeKeys.TRAIN or mode == ModeKeys.EVAL: cap_lens = labels.map(lambda t: tf.size(t)) pad_size = ((params.bin_size * params.bin_size, 1536), (None, ), ()) batches = Dataset.zip((features, labels, cap_lens)) \ .shuffle(buffer_size=200 * params.batch_size) \ .padded_batch(params.batch_size, pad_size) if mode == ModeKeys.TRAIN: train_iterator = batches \ .repeat() \ .make_initializable_iterator() feat_tensor, cap_idx_tensor, cap_len_tensor = \ train_iterator.get_next() tf.add_to_collection("train_initializer", train_iterator.initializer) if mode == ModeKeys.EVAL: val_iterator = batches \ .make_initializable_iterator() feat_tensor, cap_idx_tensor, cap_len_tensor = \ val_iterator.get_next() tf.add_to_collection("val_initializer", val_iterator.initializer) scaffold = tf.train.Scaffold(init_op=val_iterator.initializer) if mode == ModeKeys.INFER: # for infer, we need to get image id. batches = features.padded_batch( params.batch_size, ((), (params.bin_size * params.bin_size, 1536))) infer_iterator = batches.make_initializable_iterator() image_id, feat_tensor = infer_iterator.get_next() tf.add_to_collection("infer_initializer", infer_iterator.initializer) loss_op = None train_op = None predictions = None model = AttendTell(vocab_size=params.vocab_size, dim_feature=(params.bin_size * params.bin_size, 1536), selector=params.selector, dropout=params.dropout, ctx2out=params.ctx2out, prev2out=params.prev2out, hard_attention=params.hard_attention, mode=mode) if mode != ModeKeys.INFER: if params.use_sampler: outputs = model.build_train(feat_tensor, cap_idx_tensor, use_generated_inputs=True) else: outputs = model.build_train(feat_tensor, cap_idx_tensor, use_generated_inputs=False) loss_op = create_loss(outputs, cap_idx_tensor, cap_len_tensor) train_op = _get_train_op(loss_op, params.learning_rate, params.hard_attention) else: outputs = model.build_infer(feat_tensor) predictions = tf.argmax(outputs, axis=-1) if mode != ModeKeys.INFER: return EstimatorSpec(mode=mode, predictions=predictions, loss=loss_op, train_op=train_op, scaffold=scaffold) else: return EstimatorSpec(mode=mode, predictions={ "image_id": image_id, "predictions": predictions }, loss=loss_op, train_op=train_op, scaffold=scaffold)
def task(task_name): s = tf.constant(task_name, tf.string) return Dataset.from_tensors(s).repeat()
X_train_ges = tf.constant(ges[train_idx]) X_train_obj = tf.constant(obj[train_idx]) X_train_head_path = tf.constant(X_head[train_idx]) X_valid_path = tf.constant(X[valid_idx]) X_valid_FA = tf.constant(FA[valid_idx]) X_valid_ges = tf.constant(ges[valid_idx]) X_valid_obj = tf.constant(obj[valid_idx]) X_valid_head_path = tf.constant(X_head[valid_idx]) X_test_path = tf.constant(X_test_path) X_test_FA = tf.constant(X_test_FA) X_test_ges = tf.constant(X_test_ges) X_test_obj = tf.constant(X_test_obj) X_test_head_path = tf.constant(X_test_head_path) # create TensorFlow Dataset objects dataset = Dataset.from_tensor_slices( (X_train_path, X_train_FA, X_train_ges, X_train_obj, X_train_head_path)) test_dataset = Dataset.from_tensor_slices( (X_test_path, X_test_FA, X_test_ges, X_test_obj, X_test_head_path)) valid_dataset = Dataset.from_tensor_slices( (X_valid_path, X_valid_FA, X_valid_ges, X_valid_obj, X_valid_head_path)) def data_generator(X_train_path, X_train_FA, X_train_ges, X_train_obj, X_train_head_path): # read the img from file img_file = tf.read_file(X_train_path) img = tf.image.decode_image(img_file, channels=3) img = tf.image.convert_image_dtype(img, tf.float64) img.set_shape([1080, 1920, 3]) img = tf.image.resize_images(img, size=[height, width]) img = tf.image.random_flip_left_right(img)
def main(_): FLAGS.eval_interval = 1000 # todo remove if FLAGS.logdir is not None: if FLAGS.taskid is not None: FLAGS.logdir = FLAGS.logdir + '/t_' + str(FLAGS.taskid) else: FLAGS.logdir = FLAGS.logdir + '/t_' + str(random.randint(0,99999)) dataset_tools = import_module('tools.' + FLAGS.dataset) NUM_LABELS = dataset_tools.NUM_LABELS num_labels = NUM_LABELS IMAGE_SHAPE = dataset_tools.IMAGE_SHAPE image_shape = IMAGE_SHAPE train_images, train_labels_svm = dataset_tools.get_data('train') # no train labels nowhere test_images, test_labels = dataset_tools.get_data('test') if FLAGS.zero_fact < 1: # exclude a random set of zeros (not at the end, then there would be many batches without zeros) keep = np.ones(len(train_labels_svm), dtype=bool) zero_indices = np.where((train_labels_svm == 0))[0] remove = np.random.uniform(0, 1, len(zero_indices)) zero_indices_to_remove = zero_indices[remove > FLAGS.zero_fact] keep[zero_indices_to_remove] = False train_images = train_images[keep] train_labels_svm = train_labels_svm[keep] print('using only a fraction of zeros, resulting in the following shape:', train_images.shape) if FLAGS.num_unlabeled_images > 0: unlabeled_train_images, _ = dataset_tools.get_data('unlabeled', max_num=np.min([FLAGS.num_unlabeled_images, 50000])) train_images = np.vstack([train_images, unlabeled_train_images]) if FLAGS.normalize_input: train_images = (train_images - 128.) / 128. test_images = (test_images - 128.) / 128. if FLAGS.use_test: train_images = np.vstack([train_images, test_images]) train_labels_svm = np.hstack([train_labels_svm, test_labels]) #if FLAGS.dataset == 'svhn' and FLAGS.architecture == 'resnet_cifar_model': # FLAGS.emb_size = 64 image_shape_crop = image_shape c_test_imgs = test_images c_train_imgs = train_images # crop images to some random region. Intuitively, images should belong to the same cluster, # even if a part of the image is missing # (no padding, because the net could detect padding easily, and match it to other augmented samples that have # padding) if FLAGS.dataset == 'stl10': image_shape_crop = [64, 64, 3] c_test_imgs = test_images[:, 16:80, 16:80] c_train_imgs = train_images[:, 16:80, 16:80] def aug(image): return apply_augmentation(image, target_shape=image_shape_crop, params=dataset_tools.augmentation_params) def random_crop(image): image_size = image_shape_crop[0] image = tf.random_crop(image, [image_size, image_size, image_shape[2]]) return image graph = tf.Graph() with graph.as_default(): t_images = tf.placeholder("float", shape=[None] + image_shape) dataset = Dataset.from_tensor_slices(t_images) dataset = dataset.shuffle(buffer_size=10000, seed=47) # important, so that we have the same images in both sets # parameters for buffering during augmentation. Only influence training speed. nt = 8 if FLAGS.volta else 4 # that's not even enough, but there are no more CPUs b = 10000 rf = FLAGS.num_augmented_samples augmented_set = dataset if FLAGS.shuffle_augmented_samples: augmented_set = augmented_set.shuffle(buffer_size=10000, seed=47) # get multiple augmented versions of the same image - they should later have similar embeddings augmented_set = augmented_set.flat_map(lambda x: Dataset.from_tensors(x).repeat(rf)) augmented_set = augmented_set.map(aug, num_threads=nt, output_buffer_size=b) dataset = dataset.map(random_crop, num_threads=1, output_buffer_size=b) dataset = dataset.repeat().batch(FLAGS.unsup_batch_size) augmented_set = augmented_set.repeat().batch(FLAGS.unsup_batch_size * rf) iterator = dataset.make_initializable_iterator() reg_iterator = augmented_set.make_initializable_iterator() t_unsup_images = iterator.get_next() t_reg_unsup_images = reg_iterator.get_next() model_func = getattr(semisup.architectures, FLAGS.architecture) model = semisup.SemisupModel(model_func, num_labels, image_shape_crop, optimizer='adam', emb_size=FLAGS.emb_size, dropout_keep_prob=FLAGS.dropout_keep_prob, num_blocks=FLAGS.num_blocks, normalize_embeddings=FLAGS.normalize_embeddings, beta1=FLAGS.beta1, beta2=FLAGS.beta2) init_virt = [] for c in range(num_labels): center = np.random.normal(0, 0.3, size=[1, FLAGS.emb_size]) noise = np.random.uniform(-0.01, 0.01, size=[FLAGS.virtual_embeddings_per_class, FLAGS.emb_size]) centroids = noise + center init_virt.extend(centroids) t_sup_emb = tf.Variable(tf.cast(np.array(init_virt), tf.float32), name="virtual_centroids") t_sup_labels = tf.constant( np.concatenate([[i] * FLAGS.virtual_embeddings_per_class for i in range(num_labels)])) visit_weight = tf.placeholder("float", shape=[]) walker_weight = tf.placeholder("float", shape=[]) t_logit_weight = tf.placeholder("float", shape=[]) t_trafo_weight = tf.placeholder("float", shape=[]) t_l1_weight = tf.placeholder("float", shape=[]) t_norm_weight = tf.placeholder("float", shape=[]) t_learning_rate = tf.placeholder("float", shape=[]) t_sat_loss_weight = tf.placeholder("float", shape=[]) t_unsup_emb = model.image_to_embedding(t_unsup_images) t_reg_unsup_emb = model.image_to_embedding(t_reg_unsup_images) t_all_unsup_emb = tf.concat([t_unsup_emb, t_reg_unsup_emb], axis=0) t_rsup_labels = tf.constant(np.concatenate([[i] * rf for i in range(FLAGS.unsup_batch_size)])) rwalker_weight = tf.placeholder("float", shape=[]) rvisit_weight = tf.placeholder("float", shape=[]) if FLAGS.normalize_embeddings: t_sup_logit = model.embedding_to_logit(tf.nn.l2_normalize(t_sup_emb, dim=1)) model.add_semisup_loss( tf.nn.l2_normalize(t_sup_emb, dim=1), tf.nn.l2_normalize(t_unsup_emb, dim=1), t_sup_labels, walker_weight=walker_weight, visit_weight=visit_weight, match_scale=FLAGS.scale_match_ab) model.reg_loss_aba = model.add_semisup_loss( tf.nn.l2_normalize(t_reg_unsup_emb, dim=1), tf.nn.l2_normalize(t_unsup_emb, dim=1), t_rsup_labels, walker_weight=rwalker_weight, visit_weight=rvisit_weight, match_scale=FLAGS.scale_match_ab, est_err=False) else: t_sup_logit = model.embedding_to_logit(t_sup_emb) model.add_semisup_loss( t_sup_emb, t_unsup_emb, t_sup_labels, walker_weight=walker_weight, visit_weight=visit_weight, match_scale=FLAGS.scale_match_ab, est_err=True, name='c_association') model.reg_loss_aba = model.add_semisup_loss( t_reg_unsup_emb, t_unsup_emb, t_rsup_labels, walker_weight=rwalker_weight, visit_weight=rvisit_weight, match_scale=FLAGS.scale_match_ab, est_err=False, name='aug_association') model.add_logit_loss(t_sup_logit, t_sup_labels, weight=t_logit_weight) t_reg_unsup_emb_singled = t_reg_unsup_emb[::FLAGS.num_augmented_samples] t_unsup_logit = model.embedding_to_logit(t_unsup_emb) t_reg_unsup_logit = model.embedding_to_logit(t_reg_unsup_emb_singled) model.add_sat_loss(t_unsup_logit, t_reg_unsup_logit, weight=t_sat_loss_weight) trafo_lc = semisup.NO_FC_COLLECTION if FLAGS.trafo_separate_loss_collection else semisup.LOSSES_COLLECTION if FLAGS.trafo_weight > 0: # only use a single augmented sample per sample t_trafo_loss = model.add_transformation_loss(t_unsup_emb, t_reg_unsup_emb_singled, t_unsup_logit, t_reg_unsup_logit, FLAGS.unsup_batch_size, weight=t_trafo_weight, label_smoothing=0, loss_collection=trafo_lc) else: t_trafo_loss = tf.constant(0) model.add_emb_regularization(t_all_unsup_emb, weight=t_l1_weight) model.add_emb_regularization(t_sup_emb, weight=t_l1_weight) # make l2 norm = 3 model.add_emb_normalization(t_sup_emb, weight=t_norm_weight, target=FLAGS.norm_target) model.add_emb_normalization(t_all_unsup_emb, weight=t_norm_weight, target=FLAGS.norm_target) gradient_multipliers = {t_sup_emb: 1 } [train_op, train_op_sat] = model.create_train_op(t_learning_rate, gradient_multipliers=gradient_multipliers) summary_op = tf.summary.merge_all() if FLAGS.logdir is not None: summary_writer = tf.summary.FileWriter(FLAGS.logdir, graph) saver = tf.train.Saver() with tf.Session(graph=graph) as sess: tf.global_variables_initializer().run() sess.run(iterator.initializer, feed_dict={t_images: train_images}) sess.run(reg_iterator.initializer, feed_dict={t_images: train_images}) # optional: init from autoencoder if FLAGS.restore_checkpoint is not None: # logit fc layer cannot be restored def is_main_net(x): return 'logit_fc' not in x.name and 'Adam' not in x.name variables = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='net') variables = list(filter(is_main_net, variables)) restorer = tf.train.Saver(var_list=variables) restorer.restore(sess, FLAGS.restore_checkpoint) extra_feed_dict = {} from numpy.linalg import norm reg_warmup_steps = FLAGS.reg_warmup_steps logit_weight_ = FLAGS.logit_weight rwalker_weight_ = FLAGS.rwalker_weight rvisit_weight_ = FLAGS.rvisit_weight learning_rate_ = FLAGS.learning_rate trafo_weight = FLAGS.trafo_weight kmeans_initialized = False for step in range(FLAGS.max_steps): import time start = time.time() if FLAGS.init_with_kmeans: if FLAGS.kmeans_sat_thresh is not None and not kmeans_initialized or \ FLAGS.kmeans_sat_thresh is None and step <= reg_warmup_steps: walker_weight_ = 0 visit_weight_ = 0 logit_weight_ = 0 trafo_weight = 0 else: walker_weight_ = FLAGS.walker_weight visit_weight_ = FLAGS.visit_weight_base logit_weight_ = FLAGS.logit_weight trafo_weight = FLAGS.trafo_weight else: walker_weight_ = apply_envelope("log", step, FLAGS.walker_weight, reg_warmup_steps, 0) visit_weight_ = apply_envelope("log", step, FLAGS.visit_weight_base, reg_warmup_steps, 0) feed_dict = {rwalker_weight: rwalker_weight_ * FLAGS.reg_association_weight, rvisit_weight: rvisit_weight_ * FLAGS.reg_association_weight, walker_weight: walker_weight_ * FLAGS.cluster_association_weight, visit_weight: visit_weight_ * FLAGS.cluster_association_weight, t_l1_weight: FLAGS.l1_weight, t_norm_weight: FLAGS.norm_weight, t_logit_weight: logit_weight_, t_trafo_weight: trafo_weight, t_sat_loss_weight: 0, t_learning_rate: 1e-6 + apply_envelope("log", step, learning_rate_, FLAGS.warmup_steps, 0) } _, sat_loss, train_loss, summaries, centroids, unsup_emb, reg_unsup_emb, estimated_error, p_ab, p_ba, p_aba, \ reg_loss, trafo_loss = sess.run( [train_op, train_op_sat, model.train_loss, summary_op, t_sup_emb, t_unsup_emb, t_reg_unsup_emb, model.estimate_error, model.p_ab, model.p_ba, model.p_aba, model.reg_loss_aba, t_trafo_loss], {**extra_feed_dict, **feed_dict}) if FLAGS.kmeans_sat_thresh is not None and step % 200 == 0 and not kmeans_initialized: sat_score = semisup.calc_sat_score(unsup_emb, reg_unsup_emb) if sat_score > FLAGS.kmeans_sat_thresh: print('initializing with kmeans', step, sat_score) FLAGS.init_with_kmeans = True kmeans_initialized = True reg_warmup_steps = step # -> jump to next if clause if FLAGS.init_with_kmeans and step == reg_warmup_steps: # do kmeans, initialize with kmeans embs = model.calc_embedding(c_train_imgs, model.test_emb, sess, extra_feed_dict) kmeans = semisup.KMeans(n_clusters=num_labels, random_state=0).fit(embs) init_virt = [] noise = 0.0001 for c in range(num_labels): center = kmeans.cluster_centers_[c] noise = np.random.uniform(-noise, noise, size=[FLAGS.virtual_embeddings_per_class, FLAGS.emb_size]) centroids = noise + center init_virt.extend(centroids) # init with K-Means assign_op = t_sup_emb.assign(np.array(init_virt)) sess.run(assign_op) model.reset_optimizer(sess) rwalker_weight_ *= FLAGS.reg_decay_factor rvisit_weight_ *= FLAGS.reg_decay_factor if FLAGS.svm_test_interval is not None and step % FLAGS.svm_test_interval == 0 and step > 0: svm_test_score, _ = model.train_and_eval_svm(c_train_imgs, train_labels_svm, c_test_imgs, test_labels, sess, num_samples=5000) print('svm score:', svm_test_score) test_pred = model.classify(c_test_imgs, sess) train_pred = model.classify(c_train_imgs, sess) svm_test_score, _ = model.train_and_eval_svm_on_preds(train_pred, train_labels_svm, test_pred, test_labels, sess, num_samples=5000) print('svm score on logits:', svm_test_score) if step % FLAGS.decay_steps == 0 and step > 0: learning_rate_ = learning_rate_ * FLAGS.decay_factor if step == 0 or (step + 1) % FLAGS.eval_interval == 0 or step == 99: print('Step: %d' % step) print('trafo loss', trafo_loss) print('reg loss' , reg_loss) print('Time for step', time.time() - start) test_pred = model.classify(c_test_imgs, sess, extra_feed_dict).argmax(-1) nmi = semisup.calc_nmi(test_pred, test_labels) conf_mtx, score = semisup.calc_correct_logit_score(test_pred, test_labels, num_labels) print(conf_mtx) print('Test error: %.2f %%' % (100 - score * 100)) print('Test NMI: %.2f %%' % (nmi * 100)) print('Train loss: %.2f ' % train_loss) print('Train loss no fc: %.2f ' % sat_loss) print('Reg loss aba: %.2f ' % reg_loss) print('Estimated Accuracy: %.2f ' % estimated_error) sat_score = semisup.calc_sat_score(unsup_emb, reg_unsup_emb) print('sat accuracy', sat_score) embs = model.calc_embedding(c_test_imgs, model.test_emb, sess, extra_feed_dict) c_n = norm(centroids, axis=1, ord=2) e_n = norm(embs[0:100], axis=1, ord=2) print('centroid norm', np.mean(c_n)) print('embedding norm', np.mean(e_n)) k_conf_mtx, k_score = semisup.do_kmeans(embs, test_labels, num_labels) print(k_conf_mtx) print('k means score:', k_score) # sometimes that kmeans is better than the logits if FLAGS.logdir is not None: sum_values = { 'test score': score, 'reg loss': reg_loss, 'centroid norm': np.mean(c_n), 'embedding norm': np.mean(c_n), 'k means score': k_score } summary_writer.add_summary(summaries, step) for key, value in sum_values.items(): summary = tf.Summary( value=[tf.Summary.Value(tag=key, simple_value=value)]) summary_writer.add_summary(summary, step) # early stopping to save some time if step == 34999 and score < 0.45: break if step == 14999 and score < 0.225: break if dataset == 'mnist' and step == 6999 and score < 0.25: break svm_test_score, _ = model.train_and_eval_svm(c_train_imgs, train_labels_svm, c_test_imgs, test_labels, sess, num_samples=10000) if FLAGS.logdir is not None: path = saver.save(sess, FLAGS.logdir, model.step) print('@@model_path:%s' % path) print('FINAL RESULTS:') print(conf_mtx) print('Test error: %.2f %%' % (100 - score * 100)) print('final_score', score) print('@@test_error:%.4f' % score) print('@@train_loss:%.4f' % train_loss) print('@@reg_loss:%.4f' % reg_loss) print('@@estimated_error:%.4f' % estimated_error) print('@@centroid_norm:%.4f' % np.mean(c_n)) print('@@emb_norm:%.4f' % np.mean(e_n)) print('@@k_score:%.4f' % k_score) print('@@svm_score:%.4f' % svm_test_score)
eos = '-1' padding = '0' string1 = 'a b a d e f a g e a d g g h r e f g j' string2 = 'a g r c g h r h r h r f j k a e' all_words = set(string1.split() + string2.split()) all_words = list(all_words) + [eos, padding] test_batch = ['a b c c c', 'd e c', 'd e c c', 'd e e', 'd e e e e e', 'd'] test_batch = sorted(test_batch, key=lambda x: len(x)) src_strings = tf.placeholder(shape=[None], dtype=tf.string, name='source') src_dataset = Dataset.from_tensor_slices(src_strings) src_dataset = src_dataset.map(lambda src: tf.string_split([src]).values) src_dataset = src_dataset.map(lambda src: (src, tf.size(src))) batch_size = 2 batched_dataset = src_dataset.padded_batch(batch_size, padded_shapes=(tf.TensorShape( [None]), tf.TensorShape([])), padding_values=(eos, 0)) batch_iterator = batched_dataset.make_initializable_iterator() next_element = batch_iterator.get_next() with tf.Session() as sess:
def test_model(self, x_test, y_test, return_accuracy=False, print_out=True, voting=True): with self.graph.as_default(): test_dataset = Dataset.from_tensor_slices( (self.x_test_placeholder, self.y_test_placeholder)).map( view_merging, num_threads=8).batch(self.batch_sz) my_test_iterator = test_dataset.make_initializable_iterator() test_next = my_test_iterator.get_next() test_n = x_test.shape[0] self.sess.run(my_test_iterator.initializer, feed_dict={ self.x_test_placeholder: x_test, self.y_test_placeholder: y_test }) test_correct = 0 test_predictions = np.zeros((test_n)) m = 0 while True: try: x_test_now, y_test_now = self.sess.run(test_next) test_correct_m, test_prediction_m = self.sess.run( [self.correct, self.predictions], feed_dict={ self.x: x_test_now, self.y: y_test_now, self.is_training: False }) test_correct += test_correct_m test_predictions[m * self.batch_sz:(m + 1) * self.batch_sz] = test_prediction_m m += 1 except tf.errors.OutOfRangeError: break test_accuracy = test_correct / test_n if print_out: print("Testing the model") print('Test accuracy of {0:.3g}'.format(test_accuracy)) sensitivity, specificity, PPV, NPV, _, _false_positives_idx, _false_negatives_idx, total_test_cases = model_metrics( test_predictions, y_test) print( 'For the test set: sensitivity/specificity are {0:.3g} / {1:.3g}. The PPV/NPV are {2:.3g} / {3:.3g}.' .format(sensitivity, specificity, PPV, NPV)) print('total test cases: ', total_test_cases) if voting: print('Voting now') self.predictions = test_predictions if return_accuracy: print('Test accuracy of {0:.3g}'.format(test_accuracy)) return test_accuracy
def read(self, batch_size, num_epochs=1, shuffle=False, task_spec=None): """ Reads the data and return a tuple of (inputs,outputs) :param batch_size: the batch size of the returned inputs/outputs :param num_epochs: the number of epochs to read the dataset :param shuffle: whether to shuffle the data or not :param task_spec: the task spec of the training. I will help to know whether it is distributed training or not :return: The result of calling dataset.make_one_shot_iterator().get_next() """ # create the dataset of files with the data # TODO in TF 1.3 use: dataset = Dataset.list_files(self.data_files_pattern) from tensorflow.python.ops import gen_io_ops dataset = Dataset.from_tensor_slices( gen_io_ops.matching_files(self.data_files_pattern)) if shuffle: # read one sample per file # TODO in TF 1.3 use: # dataset = dataset.interleave(self.dataset_class, # # number of readers the same as number of CPUs # cycle_length=multiprocessing.cpu_count() + 1, # # block size is 1 to get directly a flat map # block_length=1) files = self._read_files_once(dataset) import random random.shuffle(files) dataset = self.dataset_class(files) else: # reads files sequentially files = self._read_files_once(dataset) dataset = self.dataset_class(files) # set the number of epochs dataset = dataset.repeat(num_epochs) if task_spec and task_spec.num_workers > 1: # split the dataset in shards # TODO in TF 1.4 use: dataset = dataset.shard(task_spec.num_workers, task_spec.index) from tensorflow.python.ops import math_ops def filter_fn(elem_index, _): mod_result = math_ops.mod(elem_index, task_spec.num_workers) return math_ops.equal(mod_result, task_spec.index) dataset = dataset.enumerate().filter(filter_fn).map( lambda _, elem: elem) if shuffle: # shuffle the samples if self.shuffle_size is None: raise ValueError('shuffle_size has not been set') dataset = dataset.shuffle(buffer_size=self.shuffle_size) # process each example. We check the method is defined in the child class: if self._flat_map.__func__ not in TFDataSet.__dict__.values(): dataset = dataset.flat_map(self._flat_map) if self._map.__func__ not in TFDataSet.__dict__.values(): dataset = dataset.map( self._map, # use as many threads as CPUs + 1 # TODO in TF 1.4 use: num_parallel_calls=multiprocessing.cpu_count() + 1, num_threads=multiprocessing.cpu_count() + 1, # buffer the data as CPUs * batch_size + minimum_size output_buffer_size=batch_size * multiprocessing.cpu_count() + self.min_queue_examples) if self.padded_shapes is not None: dataset = dataset.padded_batch(batch_size, self.padded_shapes, self.padded_values) else: dataset = dataset.batch(batch_size) return dataset.make_one_shot_iterator().get_next()
return predictions batch_size = 128 buffer_size = 10000 steps_per_epoch = int(np.ceil(60000 / float(batch_size))) # = 469 epochs = 5 num_classes = 10 (x_train, y_train), (x_test, y_test) = mnist.load_data() x_train = x_train.astype(np.float32) / 255 x_train = np.expand_dims(x_train, -1) y_train = tf.one_hot(y_train, num_classes) # Create the dataset and its associated one-shot iterator. dataset = Dataset.from_tensor_slices((x_train, y_train)) dataset = dataset.repeat() dataset = dataset.shuffle(buffer_size) dataset = dataset.batch(batch_size) iterator = dataset.make_one_shot_iterator() # Model creation using tensors from the get_next() graph node. inputs, targets = iterator.get_next() model_input = layers.Input(tensor=inputs) model_output = cnn_layers(model_input) train_model = keras.models.Model(inputs=model_input, outputs=model_output) train_model.compile(optimizer=keras.optimizers.RMSprop(lr=2e-3, decay=1e-5), loss='categorical_crossentropy', metrics=['accuracy'], target_tensors=[targets])
def input_fn(self, num_threads=1): """ Receives sequences from defined data path and processes it to feed it into the seq2seq system. :param num_threads: Number of parallel operations :return: training iterator for iterating over training data and validation iterator for iterating over validation data """ # Get subject sequences from file sequences_subject = TextLineDataset(self.data_path + "sequences_subject.txt") sequences_subject = sequences_subject.map(self.split_string, num_threads=num_threads) # Get content sequences from file sequences_content = TextLineDataset(self.data_path + "sequences_content.txt") sequences_content = sequences_content.map(self.split_string, num_threads=num_threads) # Get n best answer sequences from file sequences_n_best_answers = TextLineDataset(self.data_path + "sequences_n_best_answers.txt") sequences_n_best_answers = sequences_n_best_answers.flat_map(self.split_multi_string) # Merge sequences into dataset all_data = Dataset.zip((sequences_subject, sequences_content, sequences_n_best_answers)) # Get length for all sequences all_data = all_data.flat_map(self.get_seq_len_and_join_ba) # Filter sequence by maximum sequence length all_data = all_data.filter(self.filter_by_sequence_length) # Process target sequences by setting GO and EOS symbols all_data = all_data.flat_map(self.process_target) # Pad all sequences to a fixed length all_data = all_data.map(self.process_pad) # Count of validation data. In the thesis, the value 10000 was used. To work with little data count, the value # is set to 100 for now n_val_data = 100 # Make validation data by defining count, repeat data after count validation_data = all_data.take(n_val_data).repeat() # Process a padding for validation data validation_data = validation_data.padded_batch(self.batch_size, padded_shapes=( [None], [], [None], [], [None, self.max_seq_len], [None, self.max_seq_len], [None])) # Make training data by defining count, repeat data after count all_data = all_data.skip(n_val_data) all_data = all_data.repeat() # Shuffle training data after 10000 iterations all_data = all_data.shuffle(10000) # Process a padding for training data all_data = all_data.padded_batch(self.batch_size, padded_shapes=( [None], [], [None], [], [None, self.max_seq_len], [None, self.max_seq_len], [None])) # Make iterators for iterating over training and validation data training_iterator = all_data.make_one_shot_iterator() validation_iterator = validation_data.make_one_shot_iterator() return training_iterator, validation_iterator
import tensorflow as tf import numpy as np from tensorflow.contrib.data import Dataset # load your data or create your data in here npx = np.random.uniform(-1, 1, (1000, 1)) # x data npy = np.power(npx, 2) + np.random.normal(0, 0.1, size=npx.shape) # y data npx_train, npx_test = np.split(npx, [800]) # training and test data npy_train, npy_test = np.split(npy, [800]) # use placeholder, later you may need different data, pass the different data into placeholder tfx = tf.placeholder(npx_train.dtype, npx_train.shape) tfy = tf.placeholder(npy_train.dtype, npy_train.shape) # create dataloader dataset = Dataset.from_tensor_slices((tfx, tfy)) dataset = dataset.shuffle( buffer_size=1000) # choose data randomly from this buffer dataset = dataset.batch(32) # batch size you will use dataset = dataset.repeat(3) # repeat for 3 epochs iterator = dataset.make_initializable_iterator( ) # later we have to initialize this one # your network bx, by = iterator.get_next() # use batch to update l1 = tf.layers.dense(bx, 10, tf.nn.relu) out = tf.layers.dense(l1, npy.shape[1]) loss = tf.losses.mean_squared_error(by, out) train = tf.train.GradientDescentOptimizer(0.1).minimize(loss) sess = tf.Session()
def pad_front(frames, dataset): #Create `frames-1` empty inputs and prepend them to the dataset. #This simulates starting from the first frame in a real game. pad = Dataset.from_tensor_slices(np.zeros(frames - 1, dtype=np.int32)) return pad.concatenate(dataset)
def main(_): num_epochs = 10 batch_size = 64 with tf.Graph().as_default(): # Import data train_images, train_labels, validation_images, validation_labels, test_images, test_labels = load_mnist( FLAGS.data_dir) num_steps = int(np.floor(train_images.shape[0] / batch_size)) #make placeholders for dataset features_placeholder_train = tf.placeholder(dtype=tf.float32, shape=train_images.shape) labels_placeholder_train = tf.placeholder(dtype=train_labels.dtype, shape=train_labels.shape) #make data set from placeholders dataset = Dataset.from_tensor_slices( (features_placeholder_train, labels_placeholder_train)) #add batching parameter dataset = dataset.shuffle(buffer_size=10000) dataset = dataset.batch(batch_size) dataset = dataset.repeat(num_epochs) # # confirm that dataset is producing expected batched data for training # print(train_dataset.output_types) # print(train_dataset.output_shapes) #add noise to training images #make iterator iterator = dataset.make_initializable_iterator() (next_x_test, next_y_test) = iterator.get_next() # Build the graph for the deep net y_conv, keep_prob = deepnn(next_x_test) print(y_conv) print(next_y_test) with tf.name_scope('loss'): cross_entropy = tf.nn.softmax_cross_entropy_with_logits( labels=next_y_test, logits=y_conv) cross_entropy = tf.reduce_mean(cross_entropy) with tf.name_scope('adam_optimizer'): train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy) with tf.name_scope('accuracy'): correct_prediction = tf.equal(tf.argmax(y_conv, 1), tf.argmax(next_y_test, 1)) correct_prediction = tf.cast(correct_prediction, tf.float32) accuracy = tf.reduce_mean(correct_prediction) graph_location = tempfile.mkdtemp() print('Saving graph to: %s' % graph_location) train_writer = tf.summary.FileWriter(graph_location) train_writer.add_graph(tf.get_default_graph()) with tf.Session() as sess: sess.run(tf.global_variables_initializer()) #feed images to create datasets sess.run(iterator.initializer, feed_dict={ features_placeholder_train: train_images, labels_placeholder_train: train_labels }) for epoch_iter in range(num_epochs): print("epoch %d" % (epoch_iter)) start_time = time.time() for step in range(num_steps): try: if (step % 50 == 0): train_accuracy = accuracy.eval( feed_dict={keep_prob: 1.0}) time_so_far = time.time() - start_time print( " accuracy at step %d: %f . Time elapsed: %f" % (step, train_accuracy, time_so_far)) else: _ = sess.run([train_step], feed_dict={keep_prob: .8}) except tf.errors.OutOfRangeError: break print('test accuracy %g' % accuracy.eval())
def random_image(): return None, Dataset.from_tensors( np.random.random((784,)).astype(np.float32))
def __init__(self, txt_file, mode, batch_size, num_classes, shuffle=True, buffer_size=1000): """Create a new ImageDataGenerator. Recieves a path string to a text file, which consists of many lines, where each line has first a path string to an image and seperated by a space an integer, referring to the class number. Using this data, this class will create TensrFlow datasets, that can be used to train e.g. a convolutional neural network. Args: txt_file: Path to the text file. mode: Either 'training' or 'validation'. Depending on this value, different parsing functions will be used. batch_size: Number of images per batch. num_classes: Number of classes in the dataset. shuffle: Wether or not to shuffle the data in the dataset and the initial file list. buffer_size: Number of images used as buffer for TensorFlows shuffling of the dataset. Raises: ValueError: If an invalid mode is passed. """ self.txt_file = txt_file self.num_classes = num_classes # retrieve the data from the text file self._read_txt_file() # number of samples in the dataset self.data_size = len(self.labels) # initial shuffling of the file and label lists (together!) if shuffle: self._shuffle_lists() # convert lists to TF tensor self.img_paths = convert_to_tensor(self.img_paths, dtype=dtypes.string) self.labels = convert_to_tensor(self.labels, dtype=dtypes.int32) # create dataset data = Dataset.from_tensor_slices((self.img_paths, self.labels)) # distinguish between train/infer. when calling the parsing functions if mode == 'training': data = data.map(self._parse_function_train, num_threads=8, output_buffer_size=100*batch_size) elif mode == 'inference': data = data.map(self._parse_function_inference, num_threads=8, output_buffer_size=100*batch_size) else: raise ValueError("Invalid mode '%s'." % (mode)) # shuffle the first `buffer_size` elements of the dataset if shuffle: data = data.shuffle(buffer_size=buffer_size) # create a new dataset with batches of images data = data.batch(batch_size) self.data = data
import numpy as np from tensorflow.contrib.data import Dataset # load your data or create your data in here npx = np.random.uniform(-1, 1, (1000, 1)) # x data npy = np.power(npx, 2) + np.random.normal(0, 0.1, size=npx.shape) # y data npx_train, npx_test = np.split(npx, [800]) # training and test data npy_train, npy_test = np.split(npy, [800]) # use placeholder, later you may need different data, pass the different data into placeholder tfx = tf.placeholder(npx_train.dtype, npx_train.shape) tfy = tf.placeholder(npy_train.dtype, npy_train.shape) # create dataloader dataset = Dataset.from_tensor_slices((tfx, tfy)) dataset = dataset.shuffle(buffer_size=1000) # choose data randomly from this buffer dataset = dataset.batch(32) # batch size you will use dataset = dataset.repeat(3) # repeat for 3 epochs iterator = dataset.make_initializable_iterator() # later we have to initialize this one # your network bx, by = iterator.get_next() # use batch to update l1 = tf.layers.dense(bx, 10, tf.nn.relu) out = tf.layers.dense(l1, npy.shape[1]) loss = tf.losses.mean_squared_error(by, out) train = tf.train.GradientDescentOptimizer(0.1).minimize(loss) sess = tf.Session() # need to initialize the iterator in this case sess.run([iterator.initializer, tf.global_variables_initializer()], feed_dict={tfx: npx_train, tfy: npy_train})