def testNestedOutputs(self):
ds = Dataset.zip((Dataset.range(4), Dataset.zip((Dataset.range(4),
Dataset.range(4)))))
total = 0
# The Iterator will return a nested structure of Tensor objects.
# Some funkiness to compare against simple integers.
for (i, x) in enumerate(datasets.Iterator(ds)):
want = (i, (i, i))
got = (x[0].numpy(), (x[1][0].numpy(), x[1][1].numpy()))
self.assertEqual(got, want)
total += 1
self.assertEqual(4, total)
def testMultipleIteratorsOnADatasetThatUsesFunctions(self):
ds = Dataset.from_tensor_slices([1, 2, 3, 4, 5, 6]).map(math_ops.square)
got1 = [x.numpy() for x in datasets.Iterator(ds)]
self.assertAllEqual([1, 4, 9, 16, 25, 36], got1)
got2 = [x.numpy() for x in datasets.Iterator(ds)]
self.assertAllEqual(got1, got2)
def testMapAndFilter(self):
def even(x):
return math_ops.equal(math_ops.mod(x, 2), 0)
it = datasets.Iterator(Dataset.range(8).map(math_ops.square).filter(even))
got = [x.numpy() for x in it]
self.assertAllEqual([0, 4, 16, 36], got)
def testMultipleIteratorsOnTheSameDataset(self):
ds = Dataset.range(4)
it1 = datasets.Iterator(ds)
it2 = datasets.Iterator(ds)
got = [x.numpy() for x in it1]
self.assertAllEqual([0, 1, 2, 3], got)
got = [x.numpy() for x in it2]
self.assertAllEqual([0, 1, 2, 3], got)
def testPyFunc(self):
def my_map(inp):
return [[x + 1 for x in inp]]
ds = Dataset.range(4).map(
lambda x: script_ops.py_func(my_map, [[x]], dtypes.int64))
got = [x.numpy() for x in datasets.Iterator(ds)]
self.assertAllEqual([[1], [2], [3], [4]], got)
def main(argv=None):
'''
'''
main.__doc__ = __doc__
argv = sys.argv if argv is None else sys.argv.extend(argv)
desc = main.__doc__ # .format(os.path.basename(__file__))
# CLI parser
args = parser_(desc)
nranks_per_gpu = args.nranks_per_gpu
local_rank = hvd.local_rank()
gpu_local_rank = local_rank // nranks_per_gpu
print('local_rank, GPU_LOCAL_RANK: {}, {}'.format(
local_rank, gpu_local_rank))
# Pin GPU to be used to process local rank (one GPU per process)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
# config.gpu_options.visible_device_list = str(hvd.local_rank())
config.gpu_options.visible_device_list = str(gpu_local_rank)
K.set_session(tf.Session(config=config))
# input image dimensions
img_rows, img_cols, img_chns = 28, 28, 1
# number of convolutional filters to use
filters = 64
# convolution kernel size
num_conv = 3
hvdsize = hvd.size()
batch_size = 128 # 100
if K.image_data_format() == 'channels_first':
original_img_size = (img_chns, img_rows, img_cols)
else:
original_img_size = (img_rows, img_cols, img_chns)
latent_dim = 2
intermediate_dim = 128
epsilon_std = 1.0
epochs = args.epochs # 5
# train the VAE on MNIST digits
(x_train, _), (x_test, y_test) = mnist.load_data()
x_train = x_train.astype('float32') / 255.
x_train = x_train.reshape((x_train.shape[0],) + original_img_size)
x_test = x_test.astype('float32') / 255.
x_test = x_test.reshape((x_test.shape[0],) + original_img_size)
if hvd.rank() == 0:
print('x_train.shape:', x_train.shape)
train_samples = x_train.shape[0]
# steps_per_epoch = train_samples // batch_size // hvdsize
speedupopt = args.speedup
if speedupopt == SpeedupOpts.imgspersec:
steps_per_epoch = train_samples // batch_size
else:
steps_per_epoch = int(round(
float(train_samples) / batch_size / hvdsize + 0.5))
# Create the dataset and its associated one-shot iterator.
buffer_size = 10000
dataset = Dataset.from_tensor_slices(x_train)
dataset = dataset.repeat()
dataset = dataset.shuffle(buffer_size)
dataset = dataset.batch(batch_size)
iterator = dataset.make_one_shot_iterator()
x_train_batch = iterator.get_next()
ldict = make_shared_layers_dict(
img_chns, img_rows, img_cols, batch_size, filters,
num_conv, intermediate_dim, latent_dim, epsilon_std)
# ldict is a dictionary that holds all layers. Since these layers are
# instantiated once, they are shared amongs vae, encoder, and generator.
x = Input(tensor=x_train_batch)
vae = make_vae(ldict, x)
# : :type vae: Model
lr = 0.001 # * hvdsize
opt = tf.train.RMSPropOptimizer(lr)
# Add Horovod Distributed Optimizer.
opt = hvd.DistributedOptimizer(opt) # , use_locking=True)
opt = TFOptimizer(opt)
# opt = RMSprop(lr)
# Add Horovod Distributed Optimizer.
# opt = hvd_keras.DistributedOptimizer(opt) # , use_locking=True)
vae.compile(optimizer=opt, loss=None)
if hvd.rank() == 0:
vae.summary()
callbacks = []
if hvd.rank() == 0:
callbacks += [BatchTiming(), SamplesPerSec(batch_size * hvdsize)]
sess = K.get_session()
sess.run(hvd.broadcast_global_variables(0))
# Fit the model using data from the TF data tensors.
vae.fit(steps_per_epoch=steps_per_epoch, epochs=epochs,
callbacks=callbacks)
if hvd.rank() == 0:
x = Input(shape=original_img_size)
vae_val = make_vae(ldict, x)
vae_val.compile(optimizer=opt, loss=None)
loss = vae_val.evaluate(x=x_test, y=None, batch_size=batch_size)
print('\n\nVAE VALIDATION LOSS: {}'.format(loss))
x = Input(shape=original_img_size)
z_mean, _ = get_encoded(ldict, x)
encoder = Model(x, z_mean)
# : :type encoder: Model
decoder_input = Input(shape=(latent_dim,))
x_decoded_mean_squash = get_decoded(ldict, decoder_input)
generator = Model(decoder_input, x_decoded_mean_squash)
# : :type generator: Model
# display a 2D plot of the digit classes in the latent space
x_test_encoded = encoder.predict(x_test, batch_size=batch_size)
plt.figure(figsize=(6, 6))
plt.scatter(x_test_encoded[:, 0], x_test_encoded[:, 1], c=y_test)
plt.colorbar()
# plt.show()
plt.savefig('vae_scatter.ps')
plt.close()
# display a 2D manifold of the digits
n = 15 # figure with 15x15 digits
digit_size = 28
figure = np.zeros((digit_size * n, digit_size * n))
# Linearly spaced coordinates on the unit square were transformed
# through the inverse CDF (ppf) of the Gaussian
# To produce values of the latent variables z, since the prior of the
# latent space is Gaussian
grid_x = norm.ppf(np.linspace(0.05, 0.95, n))
grid_y = norm.ppf(np.linspace(0.05, 0.95, n))
for i, yi in enumerate(grid_x):
for j, xi in enumerate(grid_y):
z_sample = np.array([[xi, yi]])
z_sample = np.tile(z_sample, batch_size).reshape(batch_size, 2)
x_decoded = generator.predict(z_sample, batch_size=batch_size)
digit = x_decoded[0].reshape(digit_size, digit_size)
figure[i * digit_size: (i + 1) * digit_size,
j * digit_size: (j + 1) * digit_size] = digit
plt.figure(figsize=(10, 10))
plt.imshow(figure, cmap='Greys_r')
# plt.show()
plt.savefig('vae_digit.ps')
plt.close()
K.clear_session()
def __init__(self,
txt_file,
mode,
batch_size,
num_classes,
shuffle=True,
buffer_size=1000):
"""Create a new ImageDataGenerator.
Recieves a path string to a text file, which consists of many lines,
where each line has first a path string to an image and seperated by
a space an integer, referring to the class number. Using this data,
this class will create TensrFlow datasets, that can be used to train
e.g. a convolutional neural network.
Args:
txt_file: Path to the text file.
mode: Either 'training' or 'validation'. Depending on this value,
different parsing functions will be used.
batch_size: Number of images per batch.
num_classes: Number of classes in the dataset.
shuffle: Wether or not to shuffle the data in the dataset and the
initial file list.
buffer_size: Number of images used as buffer for TensorFlows
shuffling of the dataset.
Raises:
ValueError: If an invalid mode is passed.
"""
self.txt_file = txt_file
self.num_classes = num_classes
# retrieve the data from the text file
self._read_txt_file()
# number of samples in the dataset
self.data_size = len(self.labels)
# initial shuffling of the file and label lists (together!)
if shuffle:
self._shuffle_lists()
# convert lists to TF tensor
self.img_paths = convert_to_tensor(self.img_paths, dtype=dtypes.string)
self.labels = convert_to_tensor(self.labels, dtype=dtypes.int32)
# create dataset
data = Dataset.from_tensor_slices((self.img_paths, self.labels))
# distinguish between train/infer. when calling the parsing functions
if mode == 'training':
data = data.map(self._parse_function_train,
num_threads=8,
output_buffer_size=100 * batch_size)
elif mode == 'inference':
data = data.map(self._parse_function_inference,
num_threads=8,
output_buffer_size=100 * batch_size)
else:
raise ValueError("Invalid mode '%s'." % (mode))
# shuffle the first `buffer_size` elements of the dataset
if shuffle:
data = data.shuffle(buffer_size=buffer_size)
# create a new dataset with batches of images
data = data.batch(batch_size)
self.data = data
def main(add_negative):
mnist_data = input_data.read_data_sets('MNIST_data', one_hot=True)
if add_negative:
train_set = gen_noise_dataset(mnist_data.train)
class_n = 11
else:
train_set = Dataset.from_tensor_slices(
(mnist_data.train.images, mnist_data.train.labels))
class_n = 10
print('Done generating dataset')
# Input placeholder
X = tf.placeholder(tf.float32, shape=(None, 28 * 28), name='X')
keep_prob = tf.placeholder(tf.float32, name='keep_prob')
y = tf.placeholder(tf.float32, shape=(None, class_n), name='y')
# Model
model_var_dict = build_cnn_graph(X, keep_prob=keep_prob, class_n=class_n)
y_conv = model_var_dict['y_conv']
h_conv2 = model_var_dict['h_conv2']
# metrics
cross_entropy = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(logits=y_conv, labels=y))
correct_prediction = tf.equal(tf.argmax(y_conv, 1), tf.argmax(y, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
# Train OP
train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy)
# Tensorboard summary
tf.summary.scalar('cross entropy', cross_entropy)
tf.summary.scalar('training accuracy', accuracy)
tf.summary.histogram(
'L2 activation degree by filters',
tf.reduce_sum(tf.reduce_mean(h_conv2, axis=0), axis=[0, 1]))
merged_summary = tf.summary.merge_all()
# RUN!
dataset = train_set
dataset = dataset.shuffle(buffer_size=8192)
dataset = dataset.batch(BATCH_SIZE)
iterator = dataset.make_initializable_iterator()
next_batch = iterator.get_next()
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
summary_writer = tf.summary.FileWriter('mnist/logs/', sess.graph)
step_i = 0
for epoch_i in range(EPOCH_N):
sess.run(iterator.initializer)
while True:
try:
batch_X, batch_y = sess.run(next_batch)
sess.run(train_step,
feed_dict={
X: batch_X,
y: batch_y,
keep_prob: 0.5
})
# monitor
if step_i % 100 == 0:
train_accuracy, summary = sess.run(
[accuracy, merged_summary],
feed_dict={
X: batch_X,
y: batch_y,
keep_prob: 1.0
})
print('step {}, training accuracy {}'.format(
step_i, train_accuracy))
summary_writer.add_summary(summary, step_i)
step_i += 1
except tf.errors.OutOfRangeError:
break
# predict
predict_data(mnist_data.test, add_negative, accuracy)
# Visualize: what input images lead to high activation?
debug_W1, debug_b1, debug_W2, debug_b2 = sess.run(
['conv1/W:0', 'conv1/b:0', 'conv2/W:0', 'conv2/b:0'])
fn_prfx = '11_' if add_negative else '10_'
# reduce mean axis=0 for eliminate mini-batch dimension
h_conv1 = model_var_dict['h_conv1']
degrees = tf.reduce_sum(tf.reduce_mean(h_conv1, axis=0), axis=[0, 1])
get_highact_images(degrees,
X,
keep_prob,
fn_prfx + 'max_conv1',
debug_b_filters=debug_b1,
debug_W_filters=debug_W1.reshape(
-1, CONV_1_CHANNEL_N))
h_conv2 = model_var_dict['h_conv2']
degrees = tf.reduce_sum(tf.reduce_mean(h_conv2, axis=0), axis=[0, 1])
# average L2 filter over input (L1) depth when showing in 2D
debug_W2 = np.mean(debug_W2, axis=2)
get_highact_images(degrees,
X,
keep_prob,
fn_prfx + 'max_conv2',
debug_b_filters=debug_b2,
debug_W_filters=debug_W2.reshape(
-1, CONV_2_CHANNEL_N),
first_n=30)
h_fc1 = model_var_dict['h_fc1']
degrees = tf.reduce_mean(h_fc1, axis=0)
get_highact_images(degrees,
X,
keep_prob,
fn_prfx + 'max_fc1',
first_n=30)
y_conv = model_var_dict['y_conv']
degrees = tf.reduce_mean(y_conv, axis=0)
img_and_scores = get_highact_images(degrees,
X,
keep_prob,
fn_prfx + 'max_y',
sort_by_degree=False)
for i, (im, sc, w, b) in enumerate(img_and_scores):
y_out = sess.run(y_conv,
feed_dict={
X: im.reshape(1, 28 * 28),
keep_prob: 1.0
})
print('{}: {}'.format(i, np.exp(y_out) / np.sum(np.exp(y_out))))
def dataset_with_label(self, label_int, src_pattern):
label = tf.constant(label_int, tf.int32, name="label")
lines = Dataset.list_files(src_pattern).flat_map(
lambda fn: TextLineDataset(fn))
labels = Dataset.from_tensors(label).repeat()
return Dataset.zip((lines, labels))
def validate(model_def):
"""
Validate my alexnet implementation
Args:
model_def: the model class/definition
"""
img_dir = os.path.join('.', 'images')
images = []
print "loading images ..."
files = fnmatch.filter(os.listdir(img_dir), '*.jpeg')
for f in files:
print "> " + f
img_file = tf.read_file(os.path.join(img_dir, f))
img_decoded = tf.image.decode_jpeg(img_file, channels=3)
img_processed = model_def.image_prep.preprocess_image(
image=img_decoded,
output_height=model_def.image_size,
output_width=model_def.image_size,
is_training=False
)
images.append(img_processed)
# create TensorFlow Iterator object
images = Dataset.from_tensors(images)
iterator = Iterator.from_structure(images.output_types, images.output_shapes)
next_element = iterator.get_next()
iterator_init_op = iterator.make_initializer(images)
# create the model and get scores (pipe to softmax)
model = model_def(next_element)
scores = model.get_final_op()
softmax = tf.nn.softmax(scores)
print 'start validation ...'
with tf.Session() as sess:
# Initialize all variables and the iterator
sess.run(tf.global_variables_initializer())
sess.run(iterator_init_op)
# Load the pretrained weights into the model
model.load_initial_weights(sess)
# run the graph
probs = sess.run(softmax)
# sometime we have an offset
if model_def is ResNetV2:
offset = len(class_names) - len(probs[0][0][0])
else:
offset = len(class_names) - len(probs[0])
# print the results
for prob in probs:
if model_def is ResNetV2:
prob = prob[0][0]
best_index = np.argmax(prob)
print "> " + class_names[best_index+offset] + " -> %.4f" %prob[best_index]
def testBasic(self):
got = []
for t in datasets.Iterator(Dataset.range(4)):
got.append(t.numpy())
self.assertAllEqual([0, 1, 2, 3], got)
def random_images():
return None, Dataset.from_tensor_slices(
np.random.random((32, 784,)).astype(np.float32))
# convert the label to one-hot encoding
one_hot = tf.one_hot(label, NUM_CLASSES)
# read the img from file
img_file = tf.read_file(img_path)
img_decoded = tf.image.decode_image(img_file)
return img_decoded, one_hot
# Define tensorflow constants for each dataset
train_imgs = tf.constant(img_arrary)
train_labels = tf.constant(lbl_arrary)
# create TensorFlow Dataset objects
tf_dataset = Dataset.from_tensor_slices((train_imgs, train_labels))
# Parse dataset
tf_dataset = tf_dataset.map(map_func=input_parser, num_threads=None)
# Repeats the dataset n times (optional)
#tf_dataset = tf_dataset.repeat(2)
# Shuffle dataset (optional)
tf_dataset = tf_dataset.shuffle(buffer_size=len(img_arrary) * 2)
# Set batch size
tf_dataset = tf_dataset.batch(BATCH_SIZE)
# Define an iterator
iterator = tf_dataset.make_initializable_iterator()
def main(mode):
data_dir = "data/challenger.ai"
bin_size = 14
with open(os.path.join(data_dir, 'word_to_idx.pkl'), 'rb') as f:
word_to_idx = pickle.load(f)
with open(
os.path.join(
data_dir, "annotations/caption_%s_annotations_20170902.json" %
mode)) as f:
annotations = json.load(f)
image_ids = [ann['image_id'] for ann in annotations]
caps = [ann['caption'] for ann in annotations]
def my_split(text):
text = text.decode("utf-8")
# todo: take care of the unknown character.
idx = [word_to_idx.get(ch, 0) for ch in text]
idx.insert(0, word_to_idx['<START>'])
idx.append(word_to_idx['<END>'])
return np.array(idx, dtype=np.int32)
def parse(img_id, caps):
filename = os.path.join(data_dir, "image/%s" % mode) + "/" + img_id
image = tf.image.decode_jpeg(tf.read_file(filename), channels=3)
image = tf.image.convert_image_dtype(image, dtype=tf.float32)
image = tf.subtract(image, 0.5)
image = tf.multiply(image, 2.0)
splitted_caps = tuple(
map(lambda c: tf.py_func(my_split, [c], tf.int32, stateful=False),
tf.unstack(caps)))
return {
'img_id': img_id,
'raw_img': image,
'raw_caps': caps,
'cap_idx': splitted_caps
}
# return img_id, image, caps, splitted_caps
it = Dataset.from_tensor_slices(
(image_ids, caps)).map(parse).make_one_shot_iterator()
feat_tensor_dict = it.get_next()
arg_scope = inception_v4_arg_scope()
with slim.arg_scope(arg_scope):
final_conv_layer, end_points = inception_v4_base(
tf.expand_dims(feat_tensor_dict['raw_img'], 0))
feats_tensor = spatial_pyramid_pooling(final_conv_layer, [bin_size],
mode='avg')
feats_tensor = tf.reshape(feats_tensor,
shape=(-1, bin_size * bin_size, 1536))
sess = tf.Session()
variables_to_restore = slim.get_variables_to_restore(
exclude=['global_step'])
init_fn = assign_from_checkpoint_fn("data/model/inception_v4.ckpt",
variables_to_restore)
init_fn(sess)
tfrecord_filename_base = 'data/challenger.ai/tfrecords/%s_feat_14x14x1536_inception_v4' % mode
writer = tf.python_io.TFRecordWriter(tfrecord_filename_base +
"-0.tfrecords")
i = 0
while True:
try:
feature_dict, feats = sess.run((feat_tensor_dict, feats_tensor))
example = tf.train.Example(features=tf.train.Features(
feature={
'img_id':
tf.train.Feature(bytes_list=tf.train.BytesList(
value=[feature_dict['img_id']])),
# 'raw_img': tf.train.Feature(
# bytes_list=tf.train.BytesList(value=[feature_dict['raw_img'].tostring()])),
'img_feats':
tf.train.Feature(bytes_list=tf.train.BytesList(
value=[feats.tostring()])),
'raw_caps':
tf.train.Feature(bytes_list=tf.train.BytesList(
value=feature_dict['raw_caps'])),
'cap_idx':
tf.train.Feature(bytes_list=tf.train.BytesList(value=[
idx.tostring() for idx in feature_dict['cap_idx']
])),
}))
writer.write(example.SerializeToString())
print(i)
i += 1
if i % 10000 == 0:
writer.close()
writer = tf.python_io.TFRecordWriter(tfrecord_filename_base +
"-%d.tfrecords" % i)
except OutOfRangeError as e:
print(e)
break
writer.close()
def transform_test_dataset():
data_dir = "data/challenger.ai"
bin_size = 14
filenames = [
fn.split('/')[-1]
for fn in glob.glob(os.path.join(data_dir, "image/test/*"))
]
def parse(img_id):
filename = os.path.join(data_dir, "image/test/") + img_id
image = tf.image.decode_jpeg(tf.read_file(filename), channels=3)
image = tf.image.convert_image_dtype(image, dtype=tf.float32)
image = tf.subtract(image, 0.5)
image = tf.multiply(image, 2.0)
return {
'img_id': img_id,
'raw_img': image,
}
it = Dataset.from_tensor_slices(filenames).map(
parse).make_one_shot_iterator()
feat_tensor_dict = it.get_next()
arg_scope = inception_v4_arg_scope()
with slim.arg_scope(arg_scope):
final_conv_layer, end_points = inception_v4_base(
tf.expand_dims(feat_tensor_dict['raw_img'], 0))
feats_tensor = spatial_pyramid_pooling(final_conv_layer, [bin_size],
mode='avg')
feats_tensor = tf.reshape(feats_tensor,
shape=(-1, bin_size * bin_size, 1536))
sess = tf.Session()
variables_to_restore = slim.get_variables_to_restore(
exclude=['global_step'])
init_fn = assign_from_checkpoint_fn("data/model/inception_v4.ckpt",
variables_to_restore)
init_fn(sess)
tfrecord_filename_base = 'data/challenger.ai/tfrecords/test_feat_14x14x1536_inception_v4'
writer = tf.python_io.TFRecordWriter(tfrecord_filename_base +
"-0.tfrecords")
i = 0
while True:
try:
feature_dict, feats = sess.run((feat_tensor_dict, feats_tensor))
example = tf.train.Example(features=tf.train.Features(
feature={
'img_id':
tf.train.Feature(bytes_list=tf.train.BytesList(
value=[feature_dict['img_id']])),
# 'raw_img': tf.train.Feature(
# bytes_list=tf.train.BytesList(value=[feature_dict['raw_img'].tostring()])),
'img_feats':
tf.train.Feature(bytes_list=tf.train.BytesList(
value=[feats.tostring()])),
}))
writer.write(example.SerializeToString())
print(i)
i += 1
if i % 10000 == 0:
writer.close()
writer = tf.python_io.TFRecordWriter(tfrecord_filename_base +
"-%d.tfrecords" % i)
except OutOfRangeError as e:
print(e)
break
writer.close()
DIR = '/home/mtb/test.npy'
# a = np.asarray([1,2,3,4,5,6])
# np.save(DIR, a)
# b =np.load(DIR)
# print b.shape
filename = DIR
# input_=np.load(DIR + 'input.npy')
output_ = np.load(DIR + 'output.npy')
input_placeholder = tf.placeholder(tf.float32, shape=None)
# output_placeholder = tf.placeholder(tf.float32, shape = output_.shape)
a = Dataset.from_tensors(input_placeholder)
b = Dataset.from_tensors(output_placeholder)
# c = Dataset.zip((a,b))
# batch_ = a.batch(100)
iterator = a.make_initializable_iterator()
next_elem = iterator.get_next()
# # b = Dataset.from_tensor_slices(input_placeholder)
# # tf.Session().run(tf.global_variables_initializer(),)
# # b = Dataset.from_tensor_slices(tf.Session().run(v,{input_placeholder: input_}))
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
# print a.shape
# print sess.run(next_elem, {input_placeholder:input_})
input_dict = OrderedDict()
def train_deeper_better(train_data, train_labels, test_data, test_labels,
params):
"""Same as 'train_deeper', but now with tf.contrib.data.Dataset input pipeline."""
default_params = {
'regularization_coeff': 0.00001,
'keep_prob': 0.5,
'batch_size': 128,
'fc1_size': 2048,
'fc2_size': 1024,
'fc3_size': 1024,
'fc4_size': 1024,
'fc5_size': 512,
'activation': 'relu',
}
activation_funcs = {
'relu': tf.nn.relu,
'tanh': tf.nn.tanh,
}
def get_param(name):
if name in params:
return params[name]
logger.warning('%s not found in param, use default value %r', name,
default_params[name])
return default_params[name]
regularization_coeff = get_param('regularization_coeff')
keep_prob_param = get_param('keep_prob')
batch_size = int(get_param('batch_size'))
fc1_size = int(get_param('fc1_size'))
fc2_size = int(get_param('fc2_size'))
fc3_size = int(get_param('fc3_size'))
fc4_size = int(get_param('fc4_size'))
fc5_size = int(get_param('fc5_size'))
activation_func = activation_funcs[get_param('activation')]
save_restore = False
time_limit_seconds = 3600
saver_path = join(SAVER_FOLDER, train_deeper_better.__name__)
graph = tf.Graph()
with graph.as_default():
tf.set_random_seed(52)
global_step_tensor = tf.contrib.framework.get_or_create_global_step()
epoch_tensor = tf.Variable(0, trainable=False, name='epoch')
next_epoch = tf.assign_add(epoch_tensor, 1)
# dataset definition
dataset = Dataset.from_tensor_slices({
'x': train_data,
'y': train_labels
})
dataset = dataset.shuffle(buffer_size=10000)
dataset = dataset.batch(batch_size)
iterator = dataset.make_initializable_iterator()
sample = iterator.get_next()
x = sample['x']
y = sample['y']
# actual computation graph
keep_prob = tf.placeholder(tf.float32)
is_training = tf.placeholder(tf.bool, name='is_training')
regularizer = tf.contrib.layers.l2_regularizer(
scale=regularization_coeff)
def fully_connected(x, size, name):
return dense_regularized(
x,
size,
is_training,
keep_prob,
regularizer,
name,
activation_func,
)
fc1 = fully_connected(x, fc1_size, 'fc1')
fc2 = fully_connected(fc1, fc2_size, 'fc2')
fc3 = fully_connected(fc2, fc3_size, 'fc3')
fc4 = fully_connected(fc3, fc4_size, 'fc4')
fc5 = fully_connected(fc4, fc5_size, 'fc5')
logits = dense(fc5, NUM_CLASSES, regularizer, 'logits')
layer_summaries(logits, 'logits_summaries')
with tf.name_scope('accuracy'):
accuracy = tf.reduce_mean(
tf.cast(tf.equal(tf.argmax(y, 1), tf.argmax(logits, 1)),
tf.float32), )
accuracy_percent = 100 * accuracy
tf.summary.scalar('accuracy_percent', accuracy_percent)
with tf.name_scope('loss'):
regularization_losses = tf.get_collection(
tf.GraphKeys.REGULARIZATION_LOSSES)
regularization_loss = tf.reduce_sum(regularization_losses)
cross_entropy_loss = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(logits=logits,
labels=y), )
loss = cross_entropy_loss + regularization_loss
tf.summary.scalar('regularization_loss', regularization_loss)
tf.summary.scalar('cross_entropy_loss', cross_entropy_loss)
tf.summary.scalar('loss', loss)
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
# ensures that we execute the update_ops before performing the train_op
# needed for batch normalization (apparently)
optimizer = tf.train.AdamOptimizer(learning_rate=(1e-4),
epsilon=1e-3)
train_op = optimizer.minimize(loss, global_step=global_step_tensor)
all_summaries = tf.summary.merge_all()
train_writer = tf.summary.FileWriter(join(SUMMARY_FOLDER, 'train'))
batch_writer = tf.summary.FileWriter(join(SUMMARY_FOLDER, 'batch'))
test_writer = tf.summary.FileWriter(join(SUMMARY_FOLDER, 'test'))
saver = tf.train.Saver(max_to_keep=3)
test_accuracy = 0
best_accuracy = 0
with tf.Session(graph=graph) as sess:
restored = False
if save_restore:
try:
saver.restore(
sess,
tf.train.latest_checkpoint(checkpoint_dir=SAVER_FOLDER))
restored = True
except ValueError as exc:
logger.info('Could not restore previous session! %r', exc)
logger.info('Starting from scratch!')
if not restored:
tf.global_variables_initializer().run()
logger.info('Starting training...')
start_time = time.time()
def enough():
if time_limit_seconds is None:
return False
elapsed = time.time() - start_time
return elapsed > time_limit_seconds
epoch = epoch_tensor.eval()
new_epoch = True
while not enough():
logger.info('Starting new epoch #%d!', epoch)
sess.run(iterator.initializer, feed_dict={})
while not enough():
step = tf.train.global_step(sess, tf.train.get_global_step())
try:
sess.run(train_op,
feed_dict={
keep_prob: keep_prob_param,
is_training: True
})
if new_epoch:
new_epoch = False
l, reg_l, ac, summaries = sess.run(
[
loss, regularization_loss, accuracy_percent,
all_summaries
],
feed_dict={
keep_prob: keep_prob_param,
is_training: False
},
)
batch_writer.add_summary(summaries, global_step=step)
logger.info(
'Minibatch loss: %f, reg loss: %f, accuracy: %.2f%%',
l,
reg_l,
ac,
)
except tf.errors.OutOfRangeError:
logger.info('End of epoch #%d', epoch)
break
# end of epoch
previous_epoch = epoch
epoch = next_epoch.eval()
new_epoch = True
if previous_epoch % 5 == 0 and save_restore:
saver.save(sess, saver_path, global_step=previous_epoch)
def get_eval_dict(data, labels):
"""Data for evaluation."""
return {x: data, y: labels, keep_prob: 1, is_training: False}
train_l, train_ac, summaries = sess.run(
[loss, accuracy_percent, all_summaries],
feed_dict=get_eval_dict(train_data[:10000],
train_labels[:10000]),
)
train_writer.add_summary(summaries, global_step=step)
test_l, test_accuracy, summaries = sess.run(
[loss, accuracy_percent, all_summaries],
feed_dict=get_eval_dict(test_data, test_labels),
)
test_writer.add_summary(summaries, global_step=step)
best_accuracy = max(best_accuracy, test_accuracy)
logger.info('Train loss: %f, train accuracy: %.2f%%', train_l,
train_ac)
logger.info(
'Test loss: %f, TEST ACCURACY: %.2f%% BEST ACCURACY %.2f%% <<<<<<<',
test_l,
test_accuracy,
best_accuracy,
)
return best_accuracy
def testBasic(self):
got = []
for t in datasets.Iterator(Dataset.range(4)):
got.append(t.numpy())
self.assertAllEqual([0, 1, 2, 3], got)
def get_inputs(paras_file, titles_file, embedding_file, params):
'''
This function returns a dictionary of input texts, inputs required for tensorflow operation and input
placeholder and operations
:param paras_file: This the file for the input text
:param titles_file: This is the file for the input summaries
:param embedding_file: This is the embedding file like Glove, word2vec, etc
:param params: These are the flags required for the input text batch iterator
:return:
inputs: This is the dictionary corresponding to the input training and validation texts and summaries
inputs_tf: This is the dictionary with the tensorflow variables which will be used as inputs to the RNN
summarizer
inputs_ph_op: This is the dictionary with the tensorflow placeholders and operations which we will use to
declare the session
'''
def _input_parse_function(para, title):
'''
This function is used to parse the input
:param para: This is the input paragraph
:param title: This is the input summary
:return:
A tuple for para as well as for title. This tuple consists of following two things:
i) A list consisting of all the input words
ii) Number of words in the input
'''
def parse_input(text, src=None):
words = tf.string_split([text]).values
size = tf.size(words)
words = vocab.lookup(words)
if src == 'Target':
words = tf.concat([tf.constant(SOS_INDEX, dtype=tf.int64, shape=[1, ]), words], axis=0)
return (words, size)
return (parse_input(para), parse_input(title, src='Target'))
if paras_file.endswith('.pickle') or paras_file.endswith('pkl'):
input_paras = pickle.load(open(paras_file,'rb'))
input_titles = pickle.load(open(titles_file, 'rb'))
print("Data is loaded. It has {} rows".format(len(input_paras)))
input_paras, val_paras, input_titles, val_titles = train_test_split(input_paras, input_titles,
test_size=params.test_size,
train_size=params.train_size, shuffle=False)
###################
## Getting VOCAB ##
###################
vocab, embedding = utils.loadGlove(embedding_file, params)
embedding_W = tf.Variable(tf.constant(0.0, shape=embedding.shape), trainable=False, name='embedding_w')
embedding_ph = tf.placeholder(tf.float32, embedding.shape)
embedding_init = embedding_W.assign(embedding_ph)
# embedding_W = tf.Variable(embedding, trainable=False, name='embedding')
vocab = tf.contrib.lookup.index_table_from_tensor(mapping=vocab, default_value=UNK_INDEX)
#######################
## Data manipulation ##
#######################
paras_ph = tf.placeholder(tf.string, shape=(None,))
titles_ph = tf.placeholder(tf.string, shape=(None,))
batch_size = tf.placeholder(tf.int32, shape=())
data = Dataset.from_tensor_slices((paras_ph, titles_ph))
data = data.map(_input_parse_function, num_parallel_calls=8).prefetch(params.batch_size * 10)
data = data.padded_batch(tf.cast(batch_size, dtype=tf.int64),
padded_shapes=((tf.TensorShape([None]),
tf.TensorShape([])),
(tf.TensorShape([None]),
tf.TensorShape([]))),
padding_values=((tf.to_int64(EOS_INDEX), 0),
(tf.to_int64(EOS_INDEX), 0)))
iterator = data.make_initializable_iterator()
(para_batch, para_length), (title_batch, title_length) = iterator.get_next()
para_embedding = tf.nn.embedding_lookup(embedding_W, para_batch)
title_embedding = tf.nn.embedding_lookup(embedding_W, title_batch)
inputs = dict()
inputs['input_paras'] = input_paras
inputs['val_paras'] = val_paras
inputs['input_titles'] = input_titles
inputs['val_titles'] = val_titles
inputs['embedding'] = embedding
inputs_tf = dict()
inputs_tf['para_embedding'] = para_embedding
inputs_tf['title_embedding'] = title_embedding
inputs_tf['para_batch'], inputs_tf['para_length'] = para_batch, para_length
inputs_tf['title_batch'], inputs_tf['title_length'] = title_batch, title_length
inputs_tf['embedding_W'] = embedding_W
inputs_tf['batch_size'] = batch_size # Repeating batch_size as it is also required for the model building
inputs_tf['iterator'] = iterator
inputs_ph_op = dict()
inputs_ph_op['paras_ph'], inputs_ph_op['titles_ph'] = paras_ph, titles_ph
inputs_ph_op['embedding_ph'], inputs_ph_op['embedding_init'] = embedding_ph, embedding_init
inputs_ph_op['batch_size'] = batch_size
return inputs, inputs_tf, inputs_ph_op
def load_tfrecord(filename):
#Create a dataset from the file.
return Dataset.from_tensor_slices(np.arange(1, 16, 1))
def __init__(self, csv_file, data_dir, datatype, mode, batch_size, shuffle,
preprocess_conditions=False, buffer_size=1000):
self.preprocess_conditions = preprocess_conditions
self.data_dir = data_dir
self.datatype = datatype
self.condition_dir = os.path.join(self.data_dir, csv_file)
self.condition_names = list(datatype['names'][1:])
self.mode = mode
self._read_csv_file()
self.data_size_train = len(self.train_filename)
self.data_size_test = len(self.test_filename)
if shuffle:
self._shuffle_lists()
self.train_x = ["../Postprocess/x_train/" + s for s in self.train_filename]
self.train_y = ["../Postprocess/y_train/" + s for s in self.train_filename]
self.train_gen = ["../Postprocess/gen_train/" + s for s in self.train_filename]
self.train_dis = ["../Postprocess/dis_train/" + s for s in self.train_filename]
self.test_x = ["../Postprocess/x_test/" + s for s in self.test_filename]
self.test_y = ["../Postprocess/y_test/" + s for s in self.test_filename]
self.test_gen = ["../Postprocess/gen_test/" + s for s in self.test_filename]
self.test_dis = ["../Postprocess/dis_test/" + s for s in self.test_filename]
# convert lists to TF tensor
self.train_x = convert_to_tensor(self.train_x, dtype=dtypes.string)
self.train_y = convert_to_tensor(self.train_y, dtype=dtypes.string)
self.train_gen = convert_to_tensor(self.train_gen, dtype=dtypes.string)
self.train_dis = convert_to_tensor(self.train_dis, dtype=dtypes.string)
self.train_filename = convert_to_tensor(self.train_filename, dtype=dtypes.string)
self.test_x = convert_to_tensor(self.test_x, dtype=dtypes.string)
self.test_y = convert_to_tensor(self.test_y, dtype=dtypes.string)
self.test_gen = convert_to_tensor(self.test_gen, dtype=dtypes.string)
self.test_dis = convert_to_tensor(self.test_dis, dtype=dtypes.string)
self.test_filename = convert_to_tensor(self.test_filename, dtype=dtypes.string)
# create dataset
train_data = Dataset.from_tensor_slices((self.train_x, self.train_y, self.train_gen,
self.train_dis,
self.train_filename))
test_data = Dataset.from_tensor_slices((self.test_x, self.test_y, self.test_gen, self.test_dis,
self.test_filename))
if mode == 'training':
data = train_data.map(self._parse_function_train, num_threads=2,
output_buffer_size=2*batch_size)
elif mode == 'inference':
data = test_data.map(self._parse_function_inference, num_threads=2,
output_buffer_size=2*batch_size)
else:
raise ValueError("Invalid mode '%s'." % (mode))
if shuffle:
data = data.shuffle(buffer_size=buffer_size)
data = data.batch(batch_size)
self.data = data
def main(argv=None):
'''
'''
main.__doc__ = __doc__
argv = sys.argv if argv is None else sys.argv.extend(argv)
desc = main.__doc__ # .format(os.path.basename(__file__))
# CLI parser
args = parser_(desc)
mgpu = 1 if getattr(args, 'mgpu', None) is None else args.mgpu
# input image dimensions
img_rows, img_cols, img_chns = 28, 28, 1
# number of convolutional filters to use
filters = 64
# convolution kernel size
num_conv = 3
gpus_list = get_available_gpus(mgpu)
ngpus = len(gpus_list)
batch_size = 128 * ngpus
if K.image_data_format() == 'channels_first':
original_img_size = (img_chns, img_rows, img_cols)
else:
original_img_size = (img_rows, img_cols, img_chns)
latent_dim = 2
intermediate_dim = 128
epsilon_std = 1.0
epochs = args.epochs # 5
# train the VAE on MNIST digits
(x_train, _), (x_test, y_test) = mnist.load_data()
x_train = x_train.astype('float32') / 255.
x_train = x_train.reshape((x_train.shape[0],) + original_img_size)
x_test = x_test.astype('float32') / 255.
x_test = x_test.reshape((x_test.shape[0],) + original_img_size)
print('x_train.shape:', x_train.shape)
train_samples = x_train.shape[0]
steps_per_epoch = int(round(float(train_samples) / batch_size + 0.5))
# Create the dataset and its associated one-shot iterator.
buffer_size = 10000
dataset = Dataset.from_tensor_slices(x_train)
dataset = dataset.repeat()
dataset = dataset.shuffle(buffer_size)
dataset = dataset.batch(batch_size)
iterator = dataset.make_one_shot_iterator()
x_train_batch = iterator.get_next()
ldict = make_shared_layers_dict(
img_chns, img_rows, img_cols, batch_size, filters,
num_conv, intermediate_dim, latent_dim, epsilon_std)
# ldict is a dictionary that holds all layers. Since these layers are
# instantiated once, they are shared amongs vae, encoder, and generator.
x = Input(tensor=x_train_batch)
vae_serial = make_vae(ldict, x)
# : :type vae: Model
vae = make_parallel(vae_serial, gpus_list)
lr = 0.001 * ngpus
opt = RMSprop(lr) # 'rmsprop'
# opt = tf.train.RMSPropOptimizer(lr)
# opt = TFOptimizer(opt)
vae.compile(optimizer=opt, loss=None)
# vae.summary()
print_mgpu_modelsummary(vae)
callbacks = [BatchTiming(), SamplesPerSec(batch_size)]
# Fit the model using data from the TF data tensors.
vae.fit(steps_per_epoch=steps_per_epoch, epochs=epochs,
callbacks=callbacks)
x = Input(shape=original_img_size)
vae_val = make_vae(ldict, x)
vae_val.compile(optimizer=opt, loss=None)
loss = vae_val.evaluate(x=x_test, y=None, batch_size=batch_size // ngpus)
print('\n\nVAE VALIDATION LOSS: {}'.format(loss))
x = Input(shape=original_img_size)
z_mean, _ = get_encoded(ldict, x)
encoder = Model(x, z_mean)
# : :type encoder: Model
decoder_input = Input(shape=(latent_dim,))
x_decoded_mean_squash = get_decoded(ldict, decoder_input)
generator = Model(decoder_input, x_decoded_mean_squash)
# : :type generator: Model
# display a 2D plot of the digit classes in the latent space
x_test_encoded = encoder.predict(x_test, batch_size=batch_size)
plt.figure(figsize=(6, 6))
plt.scatter(x_test_encoded[:, 0], x_test_encoded[:, 1], c=y_test)
plt.colorbar()
# plt.show()
plt.savefig('vae_scatter.ps')
plt.close()
# display a 2D manifold of the digits
n = 15 # figure with 15x15 digits
digit_size = 28
figure = np.zeros((digit_size * n, digit_size * n))
# Linearly spaced coordinates on the unit square were transformed through
# the inverse CDF (ppf) of the Gaussian
# To produce values of the latent variables z, since the prior of the
# latent space is Gaussian
grid_x = norm.ppf(np.linspace(0.05, 0.95, n))
grid_y = norm.ppf(np.linspace(0.05, 0.95, n))
for i, yi in enumerate(grid_x):
for j, xi in enumerate(grid_y):
z_sample = np.array([[xi, yi]])
z_sample = np.tile(z_sample, batch_size).reshape(batch_size, 2)
x_decoded = generator.predict(z_sample, batch_size=batch_size)
digit = x_decoded[0].reshape(digit_size, digit_size)
figure[i * digit_size: (i + 1) * digit_size,
j * digit_size: (j + 1) * digit_size] = digit
plt.figure(figsize=(10, 10))
plt.imshow(figure, cmap='Greys_r')
# plt.show()
plt.savefig('vae_digit.ps')
plt.close()
x_shuffled, y_shuffled = shuffleDataset(x, y)
# Split 60 / 40
split_index = int(len(x_shuffled) * (40 / 100.0))
x_train, x_val_test = x_shuffled[split_index:], x_shuffled[:split_index]
y_train, y_val_test = y_shuffled[split_index:], y_shuffled[:split_index]
# Split 40 into dev and test sets
split_index_val_test = int(len(x_val_test) * (50 / 100.0))
x_val, x_test = x_val_test[split_index_val_test:], x_val_test[:split_index_val_test]
y_val, y_test = y_val_test[split_index_val_test:], y_val_test[:split_index_val_test]
training_data = (Dataset.from_tensor_slices((x_train, y_train))
.shuffle(buffer_size=10)
.batch(args.batch_size)
.make_initializable_iterator())
validation_data = (Dataset.from_tensor_slices((x_val, y_val))
.shuffle(buffer_size=10)
.batch(args.batch_size)
.make_initializable_iterator())
next_element_training = training_data.get_next()
next_element_validation = validation_data.get_next()
del x, y
words_per_document = get_size(x_train)
num_classes = get_size(y_train)
vocabulary_size = len(vocab_dset)
filter_sizes = list(map(int, args.filter_sizes.split(",")))
training_measurements = f['measurements']
training_measurements = np.squeeze(training_measurements).transpose([1,0])
training_patches = f['patches_vec']
training_patches = np.squeeze(training_patches).transpose([1,0])
f = sio.loadmat(os.path.dirname(os.path.abspath(__file__)) + '/dataset/validation_dataset.mat')
validation_measurements = f['measurements']
validation_measurements = np.squeeze(validation_measurements).transpose([1,0])
validation_patches = f['patches_vec']
validation_patches = np.squeeze(validation_patches).transpose([1,0])
measurements_placeholder = tf.placeholder(training_measurements.dtype, [None, np.ceil(measurement_rate*(blockSize**2))], name='measurements')
patches_placeholder = tf.placeholder(training_patches.dtype, [None, blockSize**2], name='patches')
training_dataset = Dataset.from_tensor_slices((measurements_placeholder, patches_placeholder)).batch(50)
validation_dataset = Dataset.from_tensor_slices((measurements_placeholder, patches_placeholder)).batch(1000)
nEpochs = 20
iterator = Iterator.from_structure(training_dataset.output_types, training_dataset.output_shapes)
next_measurement, next_patch = iterator.get_next()
training_init_op = iterator.make_initializer(training_dataset)
validation_init_op = iterator.make_initializer(validation_dataset)
def build_phi(patch):
measurement = tf.layers.dense(patch, np.ceil(measurement_rate * blockSize ** 2), activation=tf.nn.relu, use_bias=False, kernel_initializer=tf.random_normal_initializer(mean=0.0, stddev=0.001, dtype=tf.float64), name='fc_phi')
return measurement
def model_fn(features, labels, mode, params, config):
feat_tensor = caption_tensor = cap_idx_tensor = cap_len_tensor = None
scaffold = None
bin_size = 8
if mode == ModeKeys.TRAIN or mode == ModeKeys.EVAL:
cap_lens = labels["index"].map(lambda t: tf.size(t))
# todo: cannot utilize GPU to accelerate input pipeline, so train 1 by 1
# def extract_feats(image):
# with tf.device("/gpu:0"):
# _, end_points = vgg.vgg_16(tf.expand_dims(image, 0),
# is_training=(mode == ModeKeys.TRAIN),
# spatial_squeeze=False)
# final_conv_layer = end_points['vgg_16/conv5/conv5_3']
# feats = spatial_pyramid_pooling(final_conv_layer, [bin_size], mode='avg')
# return tf.reshape(feats, shape=(bin_size * bin_size, tf.shape(final_conv_layer)[-1]))
# features = features.map(extract_feats)
datasets = (features, labels["raw"], labels["index"], cap_lens)
# todo: 512 is the feature depth, should not hard code here
# pad_size = ((bin_size * bin_size, 512), (), (None,), ())
pad_size = ((None, None, 3), (), (None, ), ())
# todo: cannot utilize GPU to accelerate input pipeline, so train 1 by 1
batches = Dataset.zip(datasets) \
.shuffle(buffer_size=200 * params.batch_size) \
.padded_batch(1, pad_size)
if mode == ModeKeys.TRAIN:
train_iterator = batches \
.repeat() \
.make_initializable_iterator()
feat_tensor, caption_tensor, cap_idx_tensor, cap_len_tensor = \
train_iterator.get_next()
tf.add_to_collection("train_initializer",
train_iterator.initializer)
if mode == ModeKeys.EVAL:
val_iterator = batches \
.make_initializable_iterator()
feat_tensor, caption_tensor, cap_idx_tensor, cap_len_tensor = \
val_iterator.get_next()
tf.add_to_collection("val_initializer", val_iterator.initializer)
scaffold = tf.train.Scaffold(init_op=val_iterator.initializer)
if mode == ModeKeys.INFER:
batches = features.batch(params.batch_size)
infer_iterator = batches.make_initializable_iterator()
feat_tensor = infer_iterator.get_next()
tf.add_to_collection("infer_initializer", infer_iterator.initializer)
feat_tensor = _extract_feats(bin_size, feat_tensor, mode)
if mode == ModeKeys.TRAIN:
variables_to_restore = slim.get_variables_to_restore(
exclude=['global_step'])
init_fn = assign_from_checkpoint_fn(params.vgg_model_path,
variables_to_restore)
# signature of sc
scaffold = tf.train.Scaffold(init_fn=lambda _, sess: init_fn(sess))
loss_op = None
train_op = None
predictions = None
model = AttendTell(vocab_size=params.vocab_size,
selector=params.selector,
dropout=params.dropout,
ctx2out=params.ctx2out,
prev2out=params.prev2out,
hard_attention=params.hard_attention,
mode=mode)
if mode != ModeKeys.INFER:
if params.use_sampler:
outputs = model.build_train(feat_tensor,
cap_idx_tensor,
use_generated_inputs=True)
else:
outputs = model.build_train(feat_tensor,
cap_idx_tensor,
use_generated_inputs=False)
loss_op = create_loss(outputs, cap_idx_tensor, cap_len_tensor)
train_op = _get_train_op(loss_op, params.learning_rate,
params.hard_attention)
else:
outputs = model.build_infer(feat_tensor)
predictions = tf.argmax(outputs, axis=-1)
return EstimatorSpec(mode=mode,
predictions=predictions,
loss=loss_op,
train_op=train_op,
scaffold=scaffold)
import tensorflow as tf
from tensorflow.contrib.data import Dataset, Iterator
# Toy data
train_imgs = tf.constant([
'train/img1.png', 'train/img2.png', 'train/img3.png', 'train/img4.png',
'train/img5.png', 'train/img6.png'
])
train_labels = tf.constant([0, 0, 0, 1, 1, 1])
val_imgs = tf.constant(
['val/img1.png', 'val/img2.png', 'val/img3.png', 'val/img4.png'])
val_labels = tf.constant([0, 0, 1, 1])
# create TensorFlow Dataset objects
tr_data = Dataset.from_tensor_slices((train_imgs, train_labels))
val_data = Dataset.from_tensor_slices((val_imgs, val_labels))
# create TensorFlow Iterator object
iterator = Iterator.from_structure(tr_data.output_types, tr_data.output_shapes)
next_element = iterator.get_next()
# create two initialization ops to switch between the datasets
training_init_op = iterator.make_initializer(tr_data)
validation_init_op = iterator.make_initializer(val_data)
with tf.Session() as sess:
# initialize the iterator on the training data
sess.run(training_init_op)
def model_fn_inner(features, labels, mode, params, config):
feat_tensor = cap_idx_tensor = cap_len_tensor = None
scaffold = None
if mode == ModeKeys.TRAIN or mode == ModeKeys.EVAL:
cap_lens = labels.map(lambda t: tf.size(t))
pad_size = ((params.bin_size * params.bin_size, 1536), (None, ), ())
batches = Dataset.zip((features, labels, cap_lens)) \
.shuffle(buffer_size=200 * params.batch_size) \
.padded_batch(params.batch_size, pad_size)
if mode == ModeKeys.TRAIN:
train_iterator = batches \
.repeat() \
.make_initializable_iterator()
feat_tensor, cap_idx_tensor, cap_len_tensor = \
train_iterator.get_next()
tf.add_to_collection("train_initializer",
train_iterator.initializer)
if mode == ModeKeys.EVAL:
val_iterator = batches \
.make_initializable_iterator()
feat_tensor, cap_idx_tensor, cap_len_tensor = \
val_iterator.get_next()
tf.add_to_collection("val_initializer", val_iterator.initializer)
scaffold = tf.train.Scaffold(init_op=val_iterator.initializer)
if mode == ModeKeys.INFER:
# for infer, we need to get image id.
batches = features.padded_batch(
params.batch_size, ((), (params.bin_size * params.bin_size, 1536)))
infer_iterator = batches.make_initializable_iterator()
image_id, feat_tensor = infer_iterator.get_next()
tf.add_to_collection("infer_initializer", infer_iterator.initializer)
loss_op = None
train_op = None
predictions = None
model = AttendTell(vocab_size=params.vocab_size,
dim_feature=(params.bin_size * params.bin_size, 1536),
selector=params.selector,
dropout=params.dropout,
ctx2out=params.ctx2out,
prev2out=params.prev2out,
hard_attention=params.hard_attention,
mode=mode)
if mode != ModeKeys.INFER:
if params.use_sampler:
outputs = model.build_train(feat_tensor,
cap_idx_tensor,
use_generated_inputs=True)
else:
outputs = model.build_train(feat_tensor,
cap_idx_tensor,
use_generated_inputs=False)
loss_op = create_loss(outputs, cap_idx_tensor, cap_len_tensor)
train_op = _get_train_op(loss_op, params.learning_rate,
params.hard_attention)
else:
outputs = model.build_infer(feat_tensor)
predictions = tf.argmax(outputs, axis=-1)
if mode != ModeKeys.INFER:
return EstimatorSpec(mode=mode,
predictions=predictions,
loss=loss_op,
train_op=train_op,
scaffold=scaffold)
else:
return EstimatorSpec(mode=mode,
predictions={
"image_id": image_id,
"predictions": predictions
},
loss=loss_op,
train_op=train_op,
scaffold=scaffold)
def task(task_name):
s = tf.constant(task_name, tf.string)
return Dataset.from_tensors(s).repeat()
X_train_ges = tf.constant(ges[train_idx])
X_train_obj = tf.constant(obj[train_idx])
X_train_head_path = tf.constant(X_head[train_idx])
X_valid_path = tf.constant(X[valid_idx])
X_valid_FA = tf.constant(FA[valid_idx])
X_valid_ges = tf.constant(ges[valid_idx])
X_valid_obj = tf.constant(obj[valid_idx])
X_valid_head_path = tf.constant(X_head[valid_idx])
X_test_path = tf.constant(X_test_path)
X_test_FA = tf.constant(X_test_FA)
X_test_ges = tf.constant(X_test_ges)
X_test_obj = tf.constant(X_test_obj)
X_test_head_path = tf.constant(X_test_head_path)
# create TensorFlow Dataset objects
dataset = Dataset.from_tensor_slices(
(X_train_path, X_train_FA, X_train_ges, X_train_obj, X_train_head_path))
test_dataset = Dataset.from_tensor_slices(
(X_test_path, X_test_FA, X_test_ges, X_test_obj, X_test_head_path))
valid_dataset = Dataset.from_tensor_slices(
(X_valid_path, X_valid_FA, X_valid_ges, X_valid_obj, X_valid_head_path))
def data_generator(X_train_path, X_train_FA, X_train_ges, X_train_obj,
X_train_head_path):
# read the img from file
img_file = tf.read_file(X_train_path)
img = tf.image.decode_image(img_file, channels=3)
img = tf.image.convert_image_dtype(img, tf.float64)
img.set_shape([1080, 1920, 3])
img = tf.image.resize_images(img, size=[height, width])
img = tf.image.random_flip_left_right(img)
def main(_):
FLAGS.eval_interval = 1000 # todo remove
if FLAGS.logdir is not None:
if FLAGS.taskid is not None:
FLAGS.logdir = FLAGS.logdir + '/t_' + str(FLAGS.taskid)
else:
FLAGS.logdir = FLAGS.logdir + '/t_' + str(random.randint(0,99999))
dataset_tools = import_module('tools.' + FLAGS.dataset)
NUM_LABELS = dataset_tools.NUM_LABELS
num_labels = NUM_LABELS
IMAGE_SHAPE = dataset_tools.IMAGE_SHAPE
image_shape = IMAGE_SHAPE
train_images, train_labels_svm = dataset_tools.get_data('train') # no train labels nowhere
test_images, test_labels = dataset_tools.get_data('test')
if FLAGS.zero_fact < 1:
# exclude a random set of zeros (not at the end, then there would be many batches without zeros)
keep = np.ones(len(train_labels_svm), dtype=bool)
zero_indices = np.where((train_labels_svm == 0))[0]
remove = np.random.uniform(0, 1, len(zero_indices))
zero_indices_to_remove = zero_indices[remove > FLAGS.zero_fact]
keep[zero_indices_to_remove] = False
train_images = train_images[keep]
train_labels_svm = train_labels_svm[keep]
print('using only a fraction of zeros, resulting in the following shape:', train_images.shape)
if FLAGS.num_unlabeled_images > 0:
unlabeled_train_images, _ = dataset_tools.get_data('unlabeled', max_num=np.min([FLAGS.num_unlabeled_images, 50000]))
train_images = np.vstack([train_images, unlabeled_train_images])
if FLAGS.normalize_input:
train_images = (train_images - 128.) / 128.
test_images = (test_images - 128.) / 128.
if FLAGS.use_test:
train_images = np.vstack([train_images, test_images])
train_labels_svm = np.hstack([train_labels_svm, test_labels])
#if FLAGS.dataset == 'svhn' and FLAGS.architecture == 'resnet_cifar_model':
# FLAGS.emb_size = 64
image_shape_crop = image_shape
c_test_imgs = test_images
c_train_imgs = train_images
# crop images to some random region. Intuitively, images should belong to the same cluster,
# even if a part of the image is missing
# (no padding, because the net could detect padding easily, and match it to other augmented samples that have
# padding)
if FLAGS.dataset == 'stl10':
image_shape_crop = [64, 64, 3]
c_test_imgs = test_images[:, 16:80, 16:80]
c_train_imgs = train_images[:, 16:80, 16:80]
def aug(image):
return apply_augmentation(image, target_shape=image_shape_crop, params=dataset_tools.augmentation_params)
def random_crop(image):
image_size = image_shape_crop[0]
image = tf.random_crop(image, [image_size, image_size, image_shape[2]])
return image
graph = tf.Graph()
with graph.as_default():
t_images = tf.placeholder("float", shape=[None] + image_shape)
dataset = Dataset.from_tensor_slices(t_images)
dataset = dataset.shuffle(buffer_size=10000, seed=47) # important, so that we have the same images in both sets
# parameters for buffering during augmentation. Only influence training speed.
nt = 8 if FLAGS.volta else 4 # that's not even enough, but there are no more CPUs
b = 10000
rf = FLAGS.num_augmented_samples
augmented_set = dataset
if FLAGS.shuffle_augmented_samples:
augmented_set = augmented_set.shuffle(buffer_size=10000, seed=47)
# get multiple augmented versions of the same image - they should later have similar embeddings
augmented_set = augmented_set.flat_map(lambda x: Dataset.from_tensors(x).repeat(rf))
augmented_set = augmented_set.map(aug, num_threads=nt, output_buffer_size=b)
dataset = dataset.map(random_crop, num_threads=1, output_buffer_size=b)
dataset = dataset.repeat().batch(FLAGS.unsup_batch_size)
augmented_set = augmented_set.repeat().batch(FLAGS.unsup_batch_size * rf)
iterator = dataset.make_initializable_iterator()
reg_iterator = augmented_set.make_initializable_iterator()
t_unsup_images = iterator.get_next()
t_reg_unsup_images = reg_iterator.get_next()
model_func = getattr(semisup.architectures, FLAGS.architecture)
model = semisup.SemisupModel(model_func, num_labels, image_shape_crop, optimizer='adam',
emb_size=FLAGS.emb_size,
dropout_keep_prob=FLAGS.dropout_keep_prob, num_blocks=FLAGS.num_blocks,
normalize_embeddings=FLAGS.normalize_embeddings, beta1=FLAGS.beta1,
beta2=FLAGS.beta2)
init_virt = []
for c in range(num_labels):
center = np.random.normal(0, 0.3, size=[1, FLAGS.emb_size])
noise = np.random.uniform(-0.01, 0.01, size=[FLAGS.virtual_embeddings_per_class, FLAGS.emb_size])
centroids = noise + center
init_virt.extend(centroids)
t_sup_emb = tf.Variable(tf.cast(np.array(init_virt), tf.float32), name="virtual_centroids")
t_sup_labels = tf.constant(
np.concatenate([[i] * FLAGS.virtual_embeddings_per_class for i in range(num_labels)]))
visit_weight = tf.placeholder("float", shape=[])
walker_weight = tf.placeholder("float", shape=[])
t_logit_weight = tf.placeholder("float", shape=[])
t_trafo_weight = tf.placeholder("float", shape=[])
t_l1_weight = tf.placeholder("float", shape=[])
t_norm_weight = tf.placeholder("float", shape=[])
t_learning_rate = tf.placeholder("float", shape=[])
t_sat_loss_weight = tf.placeholder("float", shape=[])
t_unsup_emb = model.image_to_embedding(t_unsup_images)
t_reg_unsup_emb = model.image_to_embedding(t_reg_unsup_images)
t_all_unsup_emb = tf.concat([t_unsup_emb, t_reg_unsup_emb], axis=0)
t_rsup_labels = tf.constant(np.concatenate([[i] * rf for i in range(FLAGS.unsup_batch_size)]))
rwalker_weight = tf.placeholder("float", shape=[])
rvisit_weight = tf.placeholder("float", shape=[])
if FLAGS.normalize_embeddings:
t_sup_logit = model.embedding_to_logit(tf.nn.l2_normalize(t_sup_emb, dim=1))
model.add_semisup_loss(
tf.nn.l2_normalize(t_sup_emb, dim=1), tf.nn.l2_normalize(t_unsup_emb, dim=1), t_sup_labels,
walker_weight=walker_weight, visit_weight=visit_weight,
match_scale=FLAGS.scale_match_ab)
model.reg_loss_aba = model.add_semisup_loss(
tf.nn.l2_normalize(t_reg_unsup_emb, dim=1), tf.nn.l2_normalize(t_unsup_emb, dim=1), t_rsup_labels,
walker_weight=rwalker_weight, visit_weight=rvisit_weight, match_scale=FLAGS.scale_match_ab,
est_err=False)
else:
t_sup_logit = model.embedding_to_logit(t_sup_emb)
model.add_semisup_loss(
t_sup_emb, t_unsup_emb, t_sup_labels,
walker_weight=walker_weight, visit_weight=visit_weight,
match_scale=FLAGS.scale_match_ab, est_err=True, name='c_association')
model.reg_loss_aba = model.add_semisup_loss(
t_reg_unsup_emb, t_unsup_emb, t_rsup_labels,
walker_weight=rwalker_weight, visit_weight=rvisit_weight, match_scale=FLAGS.scale_match_ab, est_err=False, name='aug_association')
model.add_logit_loss(t_sup_logit, t_sup_labels, weight=t_logit_weight)
t_reg_unsup_emb_singled = t_reg_unsup_emb[::FLAGS.num_augmented_samples]
t_unsup_logit = model.embedding_to_logit(t_unsup_emb)
t_reg_unsup_logit = model.embedding_to_logit(t_reg_unsup_emb_singled)
model.add_sat_loss(t_unsup_logit, t_reg_unsup_logit, weight=t_sat_loss_weight)
trafo_lc = semisup.NO_FC_COLLECTION if FLAGS.trafo_separate_loss_collection else semisup.LOSSES_COLLECTION
if FLAGS.trafo_weight > 0:
# only use a single augmented sample per sample
t_trafo_loss = model.add_transformation_loss(t_unsup_emb, t_reg_unsup_emb_singled, t_unsup_logit,
t_reg_unsup_logit, FLAGS.unsup_batch_size, weight=t_trafo_weight, label_smoothing=0, loss_collection=trafo_lc)
else:
t_trafo_loss = tf.constant(0)
model.add_emb_regularization(t_all_unsup_emb, weight=t_l1_weight)
model.add_emb_regularization(t_sup_emb, weight=t_l1_weight)
# make l2 norm = 3
model.add_emb_normalization(t_sup_emb, weight=t_norm_weight, target=FLAGS.norm_target)
model.add_emb_normalization(t_all_unsup_emb, weight=t_norm_weight, target=FLAGS.norm_target)
gradient_multipliers = {t_sup_emb: 1 }
[train_op, train_op_sat] = model.create_train_op(t_learning_rate, gradient_multipliers=gradient_multipliers)
summary_op = tf.summary.merge_all()
if FLAGS.logdir is not None:
summary_writer = tf.summary.FileWriter(FLAGS.logdir, graph)
saver = tf.train.Saver()
with tf.Session(graph=graph) as sess:
tf.global_variables_initializer().run()
sess.run(iterator.initializer, feed_dict={t_images: train_images})
sess.run(reg_iterator.initializer, feed_dict={t_images: train_images})
# optional: init from autoencoder
if FLAGS.restore_checkpoint is not None:
# logit fc layer cannot be restored
def is_main_net(x):
return 'logit_fc' not in x.name and 'Adam' not in x.name
variables = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='net')
variables = list(filter(is_main_net, variables))
restorer = tf.train.Saver(var_list=variables)
restorer.restore(sess, FLAGS.restore_checkpoint)
extra_feed_dict = {}
from numpy.linalg import norm
reg_warmup_steps = FLAGS.reg_warmup_steps
logit_weight_ = FLAGS.logit_weight
rwalker_weight_ = FLAGS.rwalker_weight
rvisit_weight_ = FLAGS.rvisit_weight
learning_rate_ = FLAGS.learning_rate
trafo_weight = FLAGS.trafo_weight
kmeans_initialized = False
for step in range(FLAGS.max_steps):
import time
start = time.time()
if FLAGS.init_with_kmeans:
if FLAGS.kmeans_sat_thresh is not None and not kmeans_initialized or \
FLAGS.kmeans_sat_thresh is None and step <= reg_warmup_steps:
walker_weight_ = 0
visit_weight_ = 0
logit_weight_ = 0
trafo_weight = 0
else:
walker_weight_ = FLAGS.walker_weight
visit_weight_ = FLAGS.visit_weight_base
logit_weight_ = FLAGS.logit_weight
trafo_weight = FLAGS.trafo_weight
else:
walker_weight_ = apply_envelope("log", step, FLAGS.walker_weight, reg_warmup_steps, 0)
visit_weight_ = apply_envelope("log", step, FLAGS.visit_weight_base, reg_warmup_steps, 0)
feed_dict = {rwalker_weight: rwalker_weight_ * FLAGS.reg_association_weight,
rvisit_weight: rvisit_weight_ * FLAGS.reg_association_weight,
walker_weight: walker_weight_ * FLAGS.cluster_association_weight,
visit_weight: visit_weight_ * FLAGS.cluster_association_weight,
t_l1_weight: FLAGS.l1_weight,
t_norm_weight: FLAGS.norm_weight,
t_logit_weight: logit_weight_,
t_trafo_weight: trafo_weight,
t_sat_loss_weight: 0,
t_learning_rate: 1e-6 + apply_envelope("log", step, learning_rate_, FLAGS.warmup_steps, 0)
}
_, sat_loss, train_loss, summaries, centroids, unsup_emb, reg_unsup_emb, estimated_error, p_ab, p_ba, p_aba, \
reg_loss, trafo_loss = sess.run(
[train_op, train_op_sat, model.train_loss, summary_op, t_sup_emb, t_unsup_emb, t_reg_unsup_emb,
model.estimate_error, model.p_ab,
model.p_ba, model.p_aba, model.reg_loss_aba, t_trafo_loss], {**extra_feed_dict, **feed_dict})
if FLAGS.kmeans_sat_thresh is not None and step % 200 == 0 and not kmeans_initialized:
sat_score = semisup.calc_sat_score(unsup_emb, reg_unsup_emb)
if sat_score > FLAGS.kmeans_sat_thresh:
print('initializing with kmeans', step, sat_score)
FLAGS.init_with_kmeans = True
kmeans_initialized = True
reg_warmup_steps = step # -> jump to next if clause
if FLAGS.init_with_kmeans and step == reg_warmup_steps:
# do kmeans, initialize with kmeans
embs = model.calc_embedding(c_train_imgs, model.test_emb, sess, extra_feed_dict)
kmeans = semisup.KMeans(n_clusters=num_labels, random_state=0).fit(embs)
init_virt = []
noise = 0.0001
for c in range(num_labels):
center = kmeans.cluster_centers_[c]
noise = np.random.uniform(-noise, noise, size=[FLAGS.virtual_embeddings_per_class, FLAGS.emb_size])
centroids = noise + center
init_virt.extend(centroids)
# init with K-Means
assign_op = t_sup_emb.assign(np.array(init_virt))
sess.run(assign_op)
model.reset_optimizer(sess)
rwalker_weight_ *= FLAGS.reg_decay_factor
rvisit_weight_ *= FLAGS.reg_decay_factor
if FLAGS.svm_test_interval is not None and step % FLAGS.svm_test_interval == 0 and step > 0:
svm_test_score, _ = model.train_and_eval_svm(c_train_imgs, train_labels_svm, c_test_imgs, test_labels,
sess, num_samples=5000)
print('svm score:', svm_test_score)
test_pred = model.classify(c_test_imgs, sess)
train_pred = model.classify(c_train_imgs, sess)
svm_test_score, _ = model.train_and_eval_svm_on_preds(train_pred, train_labels_svm, test_pred, test_labels,
sess, num_samples=5000)
print('svm score on logits:', svm_test_score)
if step % FLAGS.decay_steps == 0 and step > 0:
learning_rate_ = learning_rate_ * FLAGS.decay_factor
if step == 0 or (step + 1) % FLAGS.eval_interval == 0 or step == 99:
print('Step: %d' % step)
print('trafo loss', trafo_loss)
print('reg loss' , reg_loss)
print('Time for step', time.time() - start)
test_pred = model.classify(c_test_imgs, sess, extra_feed_dict).argmax(-1)
nmi = semisup.calc_nmi(test_pred, test_labels)
conf_mtx, score = semisup.calc_correct_logit_score(test_pred, test_labels, num_labels)
print(conf_mtx)
print('Test error: %.2f %%' % (100 - score * 100))
print('Test NMI: %.2f %%' % (nmi * 100))
print('Train loss: %.2f ' % train_loss)
print('Train loss no fc: %.2f ' % sat_loss)
print('Reg loss aba: %.2f ' % reg_loss)
print('Estimated Accuracy: %.2f ' % estimated_error)
sat_score = semisup.calc_sat_score(unsup_emb, reg_unsup_emb)
print('sat accuracy', sat_score)
embs = model.calc_embedding(c_test_imgs, model.test_emb, sess, extra_feed_dict)
c_n = norm(centroids, axis=1, ord=2)
e_n = norm(embs[0:100], axis=1, ord=2)
print('centroid norm', np.mean(c_n))
print('embedding norm', np.mean(e_n))
k_conf_mtx, k_score = semisup.do_kmeans(embs, test_labels, num_labels)
print(k_conf_mtx)
print('k means score:', k_score) # sometimes that kmeans is better than the logits
if FLAGS.logdir is not None:
sum_values = {
'test score': score,
'reg loss': reg_loss,
'centroid norm': np.mean(c_n),
'embedding norm': np.mean(c_n),
'k means score': k_score
}
summary_writer.add_summary(summaries, step)
for key, value in sum_values.items():
summary = tf.Summary(
value=[tf.Summary.Value(tag=key, simple_value=value)])
summary_writer.add_summary(summary, step)
# early stopping to save some time
if step == 34999 and score < 0.45:
break
if step == 14999 and score < 0.225:
break
if dataset == 'mnist' and step == 6999 and score < 0.25:
break
svm_test_score, _ = model.train_and_eval_svm(c_train_imgs, train_labels_svm, c_test_imgs, test_labels, sess,
num_samples=10000)
if FLAGS.logdir is not None:
path = saver.save(sess, FLAGS.logdir, model.step)
print('@@model_path:%s' % path)
print('FINAL RESULTS:')
print(conf_mtx)
print('Test error: %.2f %%' % (100 - score * 100))
print('final_score', score)
print('@@test_error:%.4f' % score)
print('@@train_loss:%.4f' % train_loss)
print('@@reg_loss:%.4f' % reg_loss)
print('@@estimated_error:%.4f' % estimated_error)
print('@@centroid_norm:%.4f' % np.mean(c_n))
print('@@emb_norm:%.4f' % np.mean(e_n))
print('@@k_score:%.4f' % k_score)
print('@@svm_score:%.4f' % svm_test_score)
eos = '-1'
padding = '0'
string1 = 'a b a d e f a g e a d g g h r e f g j'
string2 = 'a g r c g h r h r h r f j k a e'
all_words = set(string1.split() + string2.split())
all_words = list(all_words) + [eos, padding]
test_batch = ['a b c c c', 'd e c', 'd e c c', 'd e e', 'd e e e e e', 'd']
test_batch = sorted(test_batch, key=lambda x: len(x))
src_strings = tf.placeholder(shape=[None], dtype=tf.string, name='source')
src_dataset = Dataset.from_tensor_slices(src_strings)
src_dataset = src_dataset.map(lambda src: tf.string_split([src]).values)
src_dataset = src_dataset.map(lambda src: (src, tf.size(src)))
batch_size = 2
batched_dataset = src_dataset.padded_batch(batch_size,
padded_shapes=(tf.TensorShape(
[None]), tf.TensorShape([])),
padding_values=(eos, 0))
batch_iterator = batched_dataset.make_initializable_iterator()
next_element = batch_iterator.get_next()
with tf.Session() as sess:
def test_model(self,
x_test,
y_test,
return_accuracy=False,
print_out=True,
voting=True):
with self.graph.as_default():
test_dataset = Dataset.from_tensor_slices(
(self.x_test_placeholder, self.y_test_placeholder)).map(
view_merging, num_threads=8).batch(self.batch_sz)
my_test_iterator = test_dataset.make_initializable_iterator()
test_next = my_test_iterator.get_next()
test_n = x_test.shape[0]
self.sess.run(my_test_iterator.initializer,
feed_dict={
self.x_test_placeholder: x_test,
self.y_test_placeholder: y_test
})
test_correct = 0
test_predictions = np.zeros((test_n))
m = 0
while True:
try:
x_test_now, y_test_now = self.sess.run(test_next)
test_correct_m, test_prediction_m = self.sess.run(
[self.correct, self.predictions],
feed_dict={
self.x: x_test_now,
self.y: y_test_now,
self.is_training: False
})
test_correct += test_correct_m
test_predictions[m * self.batch_sz:(m + 1) *
self.batch_sz] = test_prediction_m
m += 1
except tf.errors.OutOfRangeError:
break
test_accuracy = test_correct / test_n
if print_out:
print("Testing the model")
print('Test accuracy of {0:.3g}'.format(test_accuracy))
sensitivity, specificity, PPV, NPV, _, _false_positives_idx, _false_negatives_idx, total_test_cases = model_metrics(
test_predictions, y_test)
print(
'For the test set: sensitivity/specificity are {0:.3g} / {1:.3g}. The PPV/NPV are {2:.3g} / {3:.3g}.'
.format(sensitivity, specificity, PPV, NPV))
print('total test cases: ', total_test_cases)
if voting:
print('Voting now')
self.predictions = test_predictions
if return_accuracy:
print('Test accuracy of {0:.3g}'.format(test_accuracy))
return test_accuracy
def read(self, batch_size, num_epochs=1, shuffle=False, task_spec=None):
"""
Reads the data and return a tuple of (inputs,outputs)
:param batch_size: the batch size of the returned inputs/outputs
:param num_epochs: the number of epochs to read the dataset
:param shuffle: whether to shuffle the data or not
:param task_spec: the task spec of the training. I will help to know whether it is
distributed training or not
:return: The result of calling dataset.make_one_shot_iterator().get_next()
"""
# create the dataset of files with the data
# TODO in TF 1.3 use: dataset = Dataset.list_files(self.data_files_pattern)
from tensorflow.python.ops import gen_io_ops
dataset = Dataset.from_tensor_slices(
gen_io_ops.matching_files(self.data_files_pattern))
if shuffle:
# read one sample per file
# TODO in TF 1.3 use:
# dataset = dataset.interleave(self.dataset_class,
# # number of readers the same as number of CPUs
# cycle_length=multiprocessing.cpu_count() + 1,
# # block size is 1 to get directly a flat map
# block_length=1)
files = self._read_files_once(dataset)
import random
random.shuffle(files)
dataset = self.dataset_class(files)
else:
# reads files sequentially
files = self._read_files_once(dataset)
dataset = self.dataset_class(files)
# set the number of epochs
dataset = dataset.repeat(num_epochs)
if task_spec and task_spec.num_workers > 1:
# split the dataset in shards
# TODO in TF 1.4 use: dataset = dataset.shard(task_spec.num_workers, task_spec.index)
from tensorflow.python.ops import math_ops
def filter_fn(elem_index, _):
mod_result = math_ops.mod(elem_index, task_spec.num_workers)
return math_ops.equal(mod_result, task_spec.index)
dataset = dataset.enumerate().filter(filter_fn).map(
lambda _, elem: elem)
if shuffle:
# shuffle the samples
if self.shuffle_size is None:
raise ValueError('shuffle_size has not been set')
dataset = dataset.shuffle(buffer_size=self.shuffle_size)
# process each example. We check the method is defined in the child class:
if self._flat_map.__func__ not in TFDataSet.__dict__.values():
dataset = dataset.flat_map(self._flat_map)
if self._map.__func__ not in TFDataSet.__dict__.values():
dataset = dataset.map(
self._map,
# use as many threads as CPUs + 1
# TODO in TF 1.4 use: num_parallel_calls=multiprocessing.cpu_count() + 1,
num_threads=multiprocessing.cpu_count() + 1,
# buffer the data as CPUs * batch_size + minimum_size
output_buffer_size=batch_size * multiprocessing.cpu_count() +
self.min_queue_examples)
if self.padded_shapes is not None:
dataset = dataset.padded_batch(batch_size, self.padded_shapes,
self.padded_values)
else:
dataset = dataset.batch(batch_size)
return dataset.make_one_shot_iterator().get_next()
return predictions
batch_size = 128
buffer_size = 10000
steps_per_epoch = int(np.ceil(60000 / float(batch_size))) # = 469
epochs = 5
num_classes = 10
(x_train, y_train), (x_test, y_test) = mnist.load_data()
x_train = x_train.astype(np.float32) / 255
x_train = np.expand_dims(x_train, -1)
y_train = tf.one_hot(y_train, num_classes)
# Create the dataset and its associated one-shot iterator.
dataset = Dataset.from_tensor_slices((x_train, y_train))
dataset = dataset.repeat()
dataset = dataset.shuffle(buffer_size)
dataset = dataset.batch(batch_size)
iterator = dataset.make_one_shot_iterator()
# Model creation using tensors from the get_next() graph node.
inputs, targets = iterator.get_next()
model_input = layers.Input(tensor=inputs)
model_output = cnn_layers(model_input)
train_model = keras.models.Model(inputs=model_input, outputs=model_output)
train_model.compile(optimizer=keras.optimizers.RMSprop(lr=2e-3, decay=1e-5),
loss='categorical_crossentropy',
metrics=['accuracy'],
target_tensors=[targets])
def input_fn(self, num_threads=1):
"""
Receives sequences from defined data path and processes it to feed it into the seq2seq system.
:param num_threads: Number of parallel operations
:return: training iterator for iterating over training data and validation iterator for iterating over
validation data
"""
# Get subject sequences from file
sequences_subject = TextLineDataset(self.data_path + "sequences_subject.txt")
sequences_subject = sequences_subject.map(self.split_string, num_threads=num_threads)
# Get content sequences from file
sequences_content = TextLineDataset(self.data_path + "sequences_content.txt")
sequences_content = sequences_content.map(self.split_string, num_threads=num_threads)
# Get n best answer sequences from file
sequences_n_best_answers = TextLineDataset(self.data_path + "sequences_n_best_answers.txt")
sequences_n_best_answers = sequences_n_best_answers.flat_map(self.split_multi_string)
# Merge sequences into dataset
all_data = Dataset.zip((sequences_subject, sequences_content, sequences_n_best_answers))
# Get length for all sequences
all_data = all_data.flat_map(self.get_seq_len_and_join_ba)
# Filter sequence by maximum sequence length
all_data = all_data.filter(self.filter_by_sequence_length)
# Process target sequences by setting GO and EOS symbols
all_data = all_data.flat_map(self.process_target)
# Pad all sequences to a fixed length
all_data = all_data.map(self.process_pad)
# Count of validation data. In the thesis, the value 10000 was used. To work with little data count, the value
# is set to 100 for now
n_val_data = 100
# Make validation data by defining count, repeat data after count
validation_data = all_data.take(n_val_data).repeat()
# Process a padding for validation data
validation_data = validation_data.padded_batch(self.batch_size, padded_shapes=(
[None], [], [None], [], [None, self.max_seq_len], [None, self.max_seq_len], [None]))
# Make training data by defining count, repeat data after count
all_data = all_data.skip(n_val_data)
all_data = all_data.repeat()
# Shuffle training data after 10000 iterations
all_data = all_data.shuffle(10000)
# Process a padding for training data
all_data = all_data.padded_batch(self.batch_size, padded_shapes=(
[None], [], [None], [], [None, self.max_seq_len], [None, self.max_seq_len], [None]))
# Make iterators for iterating over training and validation data
training_iterator = all_data.make_one_shot_iterator()
validation_iterator = validation_data.make_one_shot_iterator()
return training_iterator, validation_iterator
import tensorflow as tf
import numpy as np
from tensorflow.contrib.data import Dataset
# load your data or create your data in here
npx = np.random.uniform(-1, 1, (1000, 1)) # x data
npy = np.power(npx, 2) + np.random.normal(0, 0.1, size=npx.shape) # y data
npx_train, npx_test = np.split(npx, [800]) # training and test data
npy_train, npy_test = np.split(npy, [800])
# use placeholder, later you may need different data, pass the different data into placeholder
tfx = tf.placeholder(npx_train.dtype, npx_train.shape)
tfy = tf.placeholder(npy_train.dtype, npy_train.shape)
# create dataloader
dataset = Dataset.from_tensor_slices((tfx, tfy))
dataset = dataset.shuffle(
buffer_size=1000) # choose data randomly from this buffer
dataset = dataset.batch(32) # batch size you will use
dataset = dataset.repeat(3) # repeat for 3 epochs
iterator = dataset.make_initializable_iterator(
) # later we have to initialize this one
# your network
bx, by = iterator.get_next() # use batch to update
l1 = tf.layers.dense(bx, 10, tf.nn.relu)
out = tf.layers.dense(l1, npy.shape[1])
loss = tf.losses.mean_squared_error(by, out)
train = tf.train.GradientDescentOptimizer(0.1).minimize(loss)
sess = tf.Session()
def pad_front(frames, dataset):
#Create `frames-1` empty inputs and prepend them to the dataset.
#This simulates starting from the first frame in a real game.
pad = Dataset.from_tensor_slices(np.zeros(frames - 1, dtype=np.int32))
return pad.concatenate(dataset)
def main(_):
num_epochs = 10
batch_size = 64
with tf.Graph().as_default():
# Import data
train_images, train_labels, validation_images, validation_labels, test_images, test_labels = load_mnist(
FLAGS.data_dir)
num_steps = int(np.floor(train_images.shape[0] / batch_size))
#make placeholders for dataset
features_placeholder_train = tf.placeholder(dtype=tf.float32,
shape=train_images.shape)
labels_placeholder_train = tf.placeholder(dtype=train_labels.dtype,
shape=train_labels.shape)
#make data set from placeholders
dataset = Dataset.from_tensor_slices(
(features_placeholder_train, labels_placeholder_train))
#add batching parameter
dataset = dataset.shuffle(buffer_size=10000)
dataset = dataset.batch(batch_size)
dataset = dataset.repeat(num_epochs)
# # confirm that dataset is producing expected batched data for training
# print(train_dataset.output_types)
# print(train_dataset.output_shapes)
#add noise to training images
#make iterator
iterator = dataset.make_initializable_iterator()
(next_x_test, next_y_test) = iterator.get_next()
# Build the graph for the deep net
y_conv, keep_prob = deepnn(next_x_test)
print(y_conv)
print(next_y_test)
with tf.name_scope('loss'):
cross_entropy = tf.nn.softmax_cross_entropy_with_logits(
labels=next_y_test, logits=y_conv)
cross_entropy = tf.reduce_mean(cross_entropy)
with tf.name_scope('adam_optimizer'):
train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy)
with tf.name_scope('accuracy'):
correct_prediction = tf.equal(tf.argmax(y_conv, 1),
tf.argmax(next_y_test, 1))
correct_prediction = tf.cast(correct_prediction, tf.float32)
accuracy = tf.reduce_mean(correct_prediction)
graph_location = tempfile.mkdtemp()
print('Saving graph to: %s' % graph_location)
train_writer = tf.summary.FileWriter(graph_location)
train_writer.add_graph(tf.get_default_graph())
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
#feed images to create datasets
sess.run(iterator.initializer,
feed_dict={
features_placeholder_train: train_images,
labels_placeholder_train: train_labels
})
for epoch_iter in range(num_epochs):
print("epoch %d" % (epoch_iter))
start_time = time.time()
for step in range(num_steps):
try:
if (step % 50 == 0):
train_accuracy = accuracy.eval(
feed_dict={keep_prob: 1.0})
time_so_far = time.time() - start_time
print(
" accuracy at step %d: %f . Time elapsed: %f" %
(step, train_accuracy, time_so_far))
else:
_ = sess.run([train_step],
feed_dict={keep_prob: .8})
except tf.errors.OutOfRangeError:
break
print('test accuracy %g' % accuracy.eval())
def random_image():
return None, Dataset.from_tensors(
np.random.random((784,)).astype(np.float32))
def __init__(self, txt_file, mode, batch_size, num_classes, shuffle=True,
buffer_size=1000):
"""Create a new ImageDataGenerator.
Recieves a path string to a text file, which consists of many lines,
where each line has first a path string to an image and seperated by
a space an integer, referring to the class number. Using this data,
this class will create TensrFlow datasets, that can be used to train
e.g. a convolutional neural network.
Args:
txt_file: Path to the text file.
mode: Either 'training' or 'validation'. Depending on this value,
different parsing functions will be used.
batch_size: Number of images per batch.
num_classes: Number of classes in the dataset.
shuffle: Wether or not to shuffle the data in the dataset and the
initial file list.
buffer_size: Number of images used as buffer for TensorFlows
shuffling of the dataset.
Raises:
ValueError: If an invalid mode is passed.
"""
self.txt_file = txt_file
self.num_classes = num_classes
# retrieve the data from the text file
self._read_txt_file()
# number of samples in the dataset
self.data_size = len(self.labels)
# initial shuffling of the file and label lists (together!)
if shuffle:
self._shuffle_lists()
# convert lists to TF tensor
self.img_paths = convert_to_tensor(self.img_paths, dtype=dtypes.string)
self.labels = convert_to_tensor(self.labels, dtype=dtypes.int32)
# create dataset
data = Dataset.from_tensor_slices((self.img_paths, self.labels))
# distinguish between train/infer. when calling the parsing functions
if mode == 'training':
data = data.map(self._parse_function_train, num_threads=8,
output_buffer_size=100*batch_size)
elif mode == 'inference':
data = data.map(self._parse_function_inference, num_threads=8,
output_buffer_size=100*batch_size)
else:
raise ValueError("Invalid mode '%s'." % (mode))
# shuffle the first `buffer_size` elements of the dataset
if shuffle:
data = data.shuffle(buffer_size=buffer_size)
# create a new dataset with batches of images
data = data.batch(batch_size)
self.data = data
import numpy as np
from tensorflow.contrib.data import Dataset
# load your data or create your data in here
npx = np.random.uniform(-1, 1, (1000, 1)) # x data
npy = np.power(npx, 2) + np.random.normal(0, 0.1, size=npx.shape) # y data
npx_train, npx_test = np.split(npx, [800]) # training and test data
npy_train, npy_test = np.split(npy, [800])
# use placeholder, later you may need different data, pass the different data into placeholder
tfx = tf.placeholder(npx_train.dtype, npx_train.shape)
tfy = tf.placeholder(npy_train.dtype, npy_train.shape)
# create dataloader
dataset = Dataset.from_tensor_slices((tfx, tfy))
dataset = dataset.shuffle(buffer_size=1000) # choose data randomly from this buffer
dataset = dataset.batch(32) # batch size you will use
dataset = dataset.repeat(3) # repeat for 3 epochs
iterator = dataset.make_initializable_iterator() # later we have to initialize this one
# your network
bx, by = iterator.get_next() # use batch to update
l1 = tf.layers.dense(bx, 10, tf.nn.relu)
out = tf.layers.dense(l1, npy.shape[1])
loss = tf.losses.mean_squared_error(by, out)
train = tf.train.GradientDescentOptimizer(0.1).minimize(loss)
sess = tf.Session()
# need to initialize the iterator in this case
sess.run([iterator.initializer, tf.global_variables_initializer()], feed_dict={tfx: npx_train, tfy: npy_train})
