def model_function(features, targets):
targets = tf.one_hot(targets, 2, 1, 0) # two perceptrons in output
outputs = layers.fully_connected(inputs=features,
num_outputs=2,
activation_fn=tf.sigmoid)
outputs_dict = {"labels": outputs}
# Calculate loss using mean squared error
loss = losses.mean_squared_error(outputs, targets)
# Create training operation
optimizer = layers.optimize_loss(
loss=loss,
# step is not an integer but a wrapper around it, just as Java has 'Integer' on top of 'int'
global_step=tf.contrib.framework.get_global_step(),
learning_rate=0.001,
optimizer="SGD")
# Why return 'loss' separately when it is already a part of optimizer?
# evaluate() needs only - outputs_dict,loss [does not need optimizer since it is not learning]
# fit() needs only - outputs_dict,loss,optimizer [does not need outputs_dict since it is not predicting]
# predict needs only - outputs_dict
# So, 'loss' sent separately for use by evaluate()
return outputs_dict, loss, optimizer
def model_fn_example(dataset_tensor, evaluation, batch_size):
input, output = dataset_tensor
net_output = layers.fully_connected(input, 1, activation_fn=None)
batch_error = losses.mean_squared_error(output, net_output)
graph_data = {}
global_step = training_util.get_or_create_global_step()
# use different metrics depending of evaluation
if evaluation:
# accumulate the error for the result
error_sum = tf.Variable(0.0, dtype=tf.float32, name='accumulated_error', trainable=False)
error_sum = tf.assign_add(error_sum, batch_error)
count = tf.Variable(0.0, dtype=tf.float32, name='data_samples', trainable=False)
count = tf.assign_add(count, 1)
error = error_sum / count
graph_data['error'] = error
else:
# use moving averages for the error
ema = tf.train.ExponentialMovingAverage(decay=0.9)
update_op = ema.apply([batch_error])
error = ema.average(batch_error)
# add train operator
sgd = tf.train.GradientDescentOptimizer(0.00001)
train_op = sgd.minimize(batch_error, global_step)
graph_data['error'] = error
graph_data['update_op'] = update_op
graph_data['train_op'] = train_op
# add error to summary
tf.summary.scalar('mse_error', error)
return graph_data
def build_loss(self):
with tf.variable_scope('loss'):
# Loss Eqn (5)
# [batch_size, 1, maze_size]
self.targets_tiled = tf.expand_dims(self.targets, 1)
# [batch_size, K + 1, maze_size]
self.targets_tiled = tf.tile(self.targets_tiled, [1, self.max_depth + 1, 1])
self.loss_preturns = losses.mean_squared_error(self.g_preturns, self.targets_tiled, scope='preturns')
losses.add_loss(self.loss_preturns)
tf.summary.scalar('loss_preturns', self.loss_preturns)
# Loss Eqn (7)
self.loss_lambda_preturns = losses.mean_squared_error(
self.g_lambda_preturns, self.targets, scope='lambda_preturns')
losses.add_loss(self.loss_lambda_preturns)
tf.summary.scalar('loss_lambda_preturns', self.loss_lambda_preturns)
self.total_loss = losses.get_total_loss(name='total_loss')
def model_function(features, targets, mode):
# don't need one-hot encoding since target is already in one-hot format
# sigmoid also will work although the interpretability is difficult;
# The output with the max. value corresponds to the 'class' - whether sigmoid or softmax
outputs = layers.fully_connected(
inputs=features,
num_outputs=10, # 10 perceptrons for 10 numbers (0 to 9)
activation_fn=tf.sigmoid)
# Calculate loss using mean squared error
loss = losses.mean_squared_error(outputs, targets)
optimizer = layers.optimize_loss(
loss=loss,
# step is not an integer but a wrapper around it, just as Java has 'Integer' on top of 'int'
global_step=tf.contrib.framework.get_global_step(),
learning_rate=0.001,
optimizer="SGD")
# Return fractional values corresponding to the sigmoid perceptron outputs
# Class of output (i.e., predicted number) corresponds to the perceptron returning the highest fractional value
# Returning both fractional values and corresponding labels
# 2nd parameter of '1' for argmax() means row-wise - this is what we want. For column-wise, use '0'.
return {'probs': outputs, 'labels': tf.argmax(outputs, 1)}, loss, optimizer
def build_loss(self):
with tf.variable_scope('loss'):
# Loss Eqn (5)
# [batch_size, 1, maze_size]
self.targets_tiled = tf.expand_dims(self.targets, 1)
# [batch_size, K + 1, maze_size]
self.targets_tiled = tf.tile(self.targets_tiled,
[1, self.max_depth + 1, 1])
self.loss_preturns = losses.mean_squared_error(self.g_preturns,
self.targets_tiled,
scope='preturns')
losses.add_loss(self.loss_preturns)
tf.summary.scalar('loss_preturns', self.loss_preturns)
# Loss Eqn (7)
self.loss_lambda_preturns = losses.mean_squared_error(
self.g_lambda_preturns, self.targets, scope='lambda_preturns')
losses.add_loss(self.loss_lambda_preturns)
tf.summary.scalar('loss_lambda_preturns',
self.loss_lambda_preturns)
self.total_loss = losses.get_total_loss(name='total_loss')
def __init__(self, N_FEATURES, LEARNING_RATE):
'''
Constructor
'''
self.x = tf.placeholder(tf.float32, shape=(None, N_FEATURES))
y = tf.placeholder(tf.float32, shape=(None,1))
self.p = tf.placeholder(tf.float32)
self.logits = layers.fully_connected(self.x, 56, activation_fn=tf.nn.relu)
self.logits = layers.dropout(self.logits, keep_prob=self.p)
self.logits = layers.fully_connected(self.x, 56, activation_fn=tf.nn.relu)
self.logits = layers.dropout(self.logits, keep_prob=self.p)
y_ = layers.fully_connected(self.logits, 1)
self.loss = losses.mean_squared_error(y, y_)
# Objective
# loss = tf.reduce_mean(tf.reduce_sum(tf.square(y-y_)) / tf.square(y_ - tf.reduce_mean(y_))) # Equivalent to minimize R2
self.train_op = tf.train.AdamOptimizer(LEARNING_RATE).minimize(self.loss)
def model_function(features, targets, mode):
#convert targets to one-hot vector representation
targets = tf.one_hot(targets, 2, 1, 0)
# Configure the single layer perceptron model
outputs = layers.fully_connected(inputs=features,
num_outputs=2,
activation_fn=tf.sigmoid)
# Calculate loss using mean squared error
loss = losses.mean_squared_error(outputs, targets)
# Create an optimizer for minimizing the loss function
optimizer = layers.optimize_loss(
loss=loss,
global_step=tf.contrib.framework.get_global_step(),
learning_rate=0.001,
optimizer="SGD")
return {'labels': outputs}, loss, optimizer
def train():
resolution = [128, 128]
batch_size = 2
# Create data inputs
training_dataset = random_images_from_spectograms('data/spectograms',
resolution=resolution,
batch_size=batch_size)
iterator = tf.data.Iterator.from_structure(training_dataset.output_types,
training_dataset.output_shapes)
sampled_images = iterator.get_next()
training_init_op = iterator.make_initializer(training_dataset)
# validation_init_op = iterator.make_initializer(validation_dataset)
# Create tensorflow graph
autoencoded_images = conv_ae_2d(sampled_images, encoder_setup,
decoder_setup)
loss = mean_squared_error(labels=sampled_images,
predictions=autoencoded_images)
optimizer = GradientDescentOptimizer(learning_rate=1e-3)
train_op = optimizer.minimize(loss)
# Run 20 epochs in which the training dataset is traversed, followed by the
# validation dataset.
with tf.Session() as sess:
epochs = 100
for epoch in range(epochs):
# Training loop
sess.run(training_init_op)
sess.run(tf.global_variables_initializer())
with tqdm.trange(10) as t:
for i in t:
# Description will be displayed on the left
t.set_description('GEN %i' % i)
_, loss = sess.run([train_op, loss])
t.set_postfix(loss=loss)
print("epoch {}".format(epoch))
def main(_):
os.environ["CUDA_VISIBLE_DEVICES"] = FLAGS.gpu
num_gpus = len(FLAGS.gpu.split(','))
config = tf.ConfigProto()
config.gpu_options.per_process_gpu_memory_fraction = 0.9
config.gpu_options.allow_growth = True
config.allow_soft_placement = True
if not tf.gfile.Exists(FLAGS.data_dir):
raise RuntimeError('data direction is not exist!')
if tf.gfile.Exists(FLAGS.log_dir):
tf.gfile.DeleteRecursively(FLAGS.log_dir)
tf.gfile.MakeDirs(FLAGS.log_dir)
if not tf.gfile.Exists(FLAGS.ckpt_dir):
tf.gfile.MakeDirs(FLAGS.ckpt_dir)
train_example_batch, train_label_batch = input_pipeline(
tf.train.match_filenames_once(os.path.join(FLAGS.data_dir, 'train', '*.tfrecords')), FLAGS.batch_size, FLAGS.patch_size)
valid_example_batch, valid_label_batch = input_pipeline(
tf.train.match_filenames_once(os.path.join(FLAGS.data_dir, 'valid', '*.tfrecords')), FLAGS.batch_size, FLAGS.patch_size)
f = open(FLAGS.out_file, 'w')
if not f:
raise RuntimeError('OUTPUT FILE OPEN ERROR!!!!!!')
with tf.name_scope('input'):
x1 = tf.placeholder(tf.float32, [None, FLAGS.patch_size, FLAGS.patch_size, 3], 'x1')
x2 = tf.placeholder(tf.float32, [None, FLAGS.patch_size, FLAGS.patch_size, 3], 'x2')
tf.summary.image('show', x1, 1)
tf.summary.image('show', x2, 1)
with tf.name_scope('label'):
y_1 = tf.placeholder(tf.int64, [None, 1], 'y1')
y_2 = tf.placeholder(tf.int64, [None, 1], 'y2')
y_ = tf.cast(tf.equal(y_1, y_2), tf.int64)
is_training = tf.placeholder(tf.bool)
y = twins.build_net(x1, x2, FLAGS.blocks, is_training)
y = tf.reshape(tf.nn.sigmoid(y), [-1])
weights = tf.reshape(tf.cast(y_, tf.float32) * 10, [-1])
y_ = tf.reshape(y_, [-1])
with tf.name_scope('scores'):
loss.mean_squared_error(y, y_, weights)
total_loss = tf.contrib.losses.get_total_loss(add_regularization_losses=True, name='total_loss')
with tf.name_scope('accuracy'):
correct_prediction = tf.equal(tf.cast(tf.round(y), tf.int64), y_)
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
tf.summary.scalar('loss', total_loss)
tf.summary.scalar('accuracy', accuracy)
with tf.name_scope('train'):
global_step = tf.Variable(FLAGS.start_step, name="global_step")
# learning_rate = tf.train.piecewise_constant(global_step, [32000, 64000, 108000, ], [0.01, 0.001, 0.0001, 0.00001])
learning_rate = tf.train.exponential_decay(0.1, global_step, 32000, 0.1)
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
train_step = tf.train.MomentumOptimizer(learning_rate, momentum=FLAGS.momentum).minimize(total_loss,
global_step=global_step)
tf.summary.scalar('lr', learning_rate)
merged = tf.summary.merge_all()
with tf.name_scope("saver"):
saver = tf.train.Saver(name="saver")
with tf.Session(config=config) as sess:
sess.run(tf.local_variables_initializer())
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)
if tf.gfile.Exists(os.path.join(FLAGS.ckpt_dir, 'checkpoint')):
saver.restore(sess, os.path.join(FLAGS.ckpt_dir, FLAGS.model_name))
else:
sess.run(tf.global_variables_initializer())
train_writer = tf.summary.FileWriter(FLAGS.log_dir + '/train', sess.graph)
test_writer = tf.summary.FileWriter(FLAGS.log_dir + '/test', sess.graph)
train_writer.flush()
test_writer.flush()
def feed_dict(train, on_training):
def get_batch(data, labels):
d, l = sess.run([data, labels])
d = d.astype(np.float32)
l = l.astype(np.int64)
return d, l
if train:
x1s, y1s = get_batch(train_example_batch, train_label_batch)
x2s, y2s = get_batch(train_example_batch, train_label_batch)
else:
x1s, y1s = get_batch(valid_example_batch, valid_label_batch)
x2s, y2s = get_batch(valid_example_batch, valid_label_batch)
return {x1: x1s, x2: x2s, y_1: y1s, y_2: y2s, is_training: on_training}
for i in range(FLAGS.start_step, FLAGS.max_steps + 1):
feed = feed_dict(True, True)
sess.run(train_step, feed_dict=feed)
if i % 100 == 0 and i != 0: # Record summaries and test-set accuracy
loss0, acc0, summary = sess.run([total_loss, accuracy, merged], feed_dict=feed_dict(False, False))
test_writer.add_summary(summary, i)
loss1, acc1, summary = sess.run([total_loss, accuracy, merged], feed_dict=feed_dict(True, False))
train_writer.add_summary(summary, i)
print(time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), file=f)
print('step %d: train_acc=%f, train_loss=%f; test_acc=%f, test_loss=%f' % (i, acc1, loss1, acc0, loss0),
file=f)
saver.save(sess, os.path.join(FLAGS.ckpt_dir, FLAGS.model_name))
f.flush()
coord.request_stop()
coord.join(threads)
train_writer.close()
test_writer.close()
f.close()
# softmax output. Gives global meaning to both the perceptrons combined
# output is [0.26,0.73], [0.26,0.73].
# [e^6/(e^6+e^7),e^7/(e^6 + e^7) ] = [1/1+e, 1/1+e^-1] = [0.26,0.73]
# [e^12/(e^12+e^13),e^13/(e^12 + e^13) ] = [1/1+e, 1/1+e^-1] = [0.26,0.73]
# Note how the result is very different from that obtained with sigmoid activation function.
nnout6 = layers.fully_connected(
inputs=features,
weights_initializer=tf.constant_initializer([[1.0, 1.0], [2.0, 2.0]]),
biases_initializer=tf.constant_initializer([1.0, 2.0]),
num_outputs=2,
activation_fn=tf.nn.softmax)
session = tf.Session()
session.run(tf.initialize_all_variables())
session.run(nnout6)
outputs = tf.constant([0, 1, 0, 1])
targets = tf.constant([1, 1, 1, 0])
sq_loss1 = losses.mean_squared_error(outputs, targets)
log_loss1 = losses.log_loss(outputs, targets)
outputs2 = tf.constant([[100.0, -100.0, -100.0], [-100.0, 100.0, -100.0],
[-100.0, -100.0, 100.0]])
targets2 = tf.constant([[0, 0, 1], [1, 0, 0], [0, 1, 0]])
sq_loss2 = losses.mean_squared_error(outputs2, targets2)
session = tf.Session()
session.run(tf.initialize_all_variables())
session.run(sq_loss1) # 0.75 = [(0-1)^2 + (1-1)^2 + (0-1)^2 + (1-0)^2] / 4
session.run(sq_loss2) # 10067 = (6*100^2 + 3*101^2)/9
session.run(log_loss1) # sigma(-y_i*log(y_i))
# Get first observation
observation = env.reset()
N_FEATURES = 108
LEARNING_RATE = 0.001
x = tf.placeholder(tf.float32, shape=(None, N_FEATURES))
y = tf.placeholder(tf.float32, shape=(None, 1))
p = tf.placeholder(tf.float32)
logits = layers.fully_connected(x, 56, activation_fn=tf.nn.relu)
logits = layers.dropout(logits, keep_prob=p)
logits = layers.fully_connected(x, 56, activation_fn=tf.nn.relu)
logits = layers.dropout(logits, keep_prob=p)
y_ = layers.fully_connected(logits, 1)
loss = losses.mean_squared_error(y, y_)
# loss = tf.reduce_mean(tf.reduce_sum(tf.square(y-y_)) / tf.square(y_ - tf.reduce_mean(y_))) # Equivalent to minimize R2
train_op = tf.train.AdamOptimizer(LEARNING_RATE).minimize(loss)
from sklearn.cross_validation import train_test_split
traindf, testdf = train_test_split(observation.train.drop(
axis=1, labels=["id", "timestamp"]).dropna(),
train_size=0.8,
test_size=0.2)
Y_train = traindf["y"]
X_train = traindf.drop(axis=1, labels=["y"])
Y_test = testdf["y"]
import tensorflow as tf
import numpy as np
import tensorflow.contrib.losses as losses
import tensorflow.contrib.metrics as metrics
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets("MNIST_data/", one_hot=True)
x = tf.placeholder(tf.float32, [None, 784])
W = tf.Variable(tf.zeros([784, 10]))
b = tf.Variable(tf.zeros([10]))
y = tf.nn.softmax(tf.matmul(x, W) + b)
y_ = tf.placeholder(tf.float32, [None, 10])
cross_entropy = losses.mean_squared_error(y, y_)
train_step = tf.train.GradientDescentOptimizer(0.5).minimize(cross_entropy)
sess = tf.Session()
sess.run(tf.initialize_all_variables())
saver = tf.train.Saver()
saver.restore(
sess,
"/home/hardik/Desktop/MTech_Project/Scripts/Python/Brain_Research_Python/MNIST_data/myModel.ckpt"
)
correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
print(
sess.run(accuracy, feed_dict={
x: mnist.test.images,