def digit_classification():
sess = tf.Session()
K.set_session(sess)
img = tf.placeholder(tf.float32, shape=(None, 784))
labels = tf.placeholder(tf.float32, shape=(None, 10))
x = Dense(128, activation='relu')(img)
x = Dropout(0.5)(x)
x = Dense(128, activation='relu')(x)
x = Dropout(0.5)(x)
preds = Dense(10, activation='softmax')(x)
loss = tf.reduce_mean(categorical_crossentropy(labels, preds))
mnist_data = input_data.read_data_sets('MNIST_data', one_hot=True)
train_step = tf.train.GradientDescentOptimizer(0.5).minimize(loss)
with sess.as_default():
for i in range(100):
batch = mnist_data.train.next_batch(50)
train_step.run(feed_dict={
img: batch[0],
labels: batch[1],
K.learning_phase(): 1
})
acc_value = accuracy(labels, preds)
with sess.as_default():
print acc_value.eval(
feed_dict={
img: mnist_data.test.images,
labels: mnist_data.test.labels,
K.learning_phase(): 0
})
def simple_network(b_type,
graph,
binarize_input=False,
backprop_type="Identity",
initialization=he_normal()):
with graph.as_default():
assert b_type in ["full", "BinaryNet", "BinaryConnect"]
# Define the context manager for quantification and set up its conditions.
qm = QuantificationManager(b_type)
qm.enable_clipping(10.0)
qm.define_quantification_condition("W",
deterministic_binary(1.0),
initialization=initialization)
qm.define_quantification_condition("b",
deterministic_binary(1.0),
initialization=binary_choice(1.0))
# Define variables for use in training
x = tf.placeholder(tf.float32, shape=(None, 784), name="x")
labels = tf.placeholder(tf.float32, shape=(None, 10), name="y")
global_step = tf.Variable(0, trainable=False, name="global_step")
learning_rate = tf.placeholder(tf.float32, name="learning_rate")
actual_rate = tf.train.exponential_decay(learning_rate, global_step,
250, 0.95)
# Open a tf.Variable_scope which our QuantificationManager will be monitoring
with qm.monitor_tf_variable_scope("feedforward"):
# Binarize input if you want
if binarize_input and b_type != "full":
with graph.gradient_override_map({"Round": "Identity"}):
x = tf.round(x)
h1 = DiscreteDense(256,
graph,
b_type=b_type,
name="first_hidden",
backprop_type=backprop_type)(x)
o = Dense(10, name="output")(h1)
# Gather ops useful for forward propagations
output = tf.identity(o, name="output")
loss = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(logits=output,
labels=labels))
acc_value = accuracy(labels, output)
# Gather ops useful for backward propagations
optimizer = tf.train.AdamOptimizer(actual_rate)
grads_vars = qm.handle_gradients(optimizer.compute_gradients(loss))
# Receive the necessary operations from the QuantificationManager
clip_ops = qm.clip_full_precision_ops() + [
tf.assign(global_step, global_step + tf.constant(1))
]
disc_ops = qm.rediscretization_ops()
initialization_ops = qm.initialization_ops()
# Receive the training op
backprop = optimizer.apply_gradients(grads_vars)
return [backprop, loss, acc_value, clip_ops, disc_ops, initialization_ops]
def eval_accuracy(labels, preds):
"""
https://blog.keras.io/keras-as-a-simplified-interface-to-tensorflow-tutorial.html
"""
acc_value = accuracy(labels, preds)
with sess.as_default():
eval_result = acc_value.eval()
return eval_result.mean()
def _buildModel(self, qsize, vsize, vector_size):
t_shape, t_dtype = (), tf.float32
if not vector_size:
x_shape = (None, )
x_dtype = tf.int32
else:
x_shape = (None, vector_size)
x_dtype = tf.float32
x_in = tf.placeholder(x_dtype, shape=x_shape)
t_in = tf.placeholder(t_dtype, shape=t_shape)
batch_size = tf.placeholder_with_default(tf.constant(64), shape=())
# Input queue
q = tf.PaddingFIFOQueue(qsize, (x_dtype, t_dtype),
shapes=(x_shape, t_shape))
enqueue_op = q.enqueue((x_in, t_in))
# Fetched variables
x, t = q.dequeue_many(batch_size)
# Model definition
model = Sequential()
if not vector_size:
model.add(
Embedding(output_dim=512, input_dim=vsize + 2, mask_zero=True))
model.add(GRU(32))
else:
model.add(GRU(32, input_dim=vector_size))
model.add(Dense(64, activation='relu'))
model.add(Dense(64, activation='relu'))
model.add(Dense(64, activation='relu'))
model.add(Dense(1, activation='sigmoid'))
y = K.squeeze(model(x), 1) # shape = (batch_size,) - same as t
# Metrics
loss = K.mean(K.binary_crossentropy(y, t))
acc = accuracy(t, y)
# Trainer
train_op = tf.train.AdamOptimizer().minimize(loss)
return {
'x': x_in,
't': t_in,
'y': y,
'q': q,
'acc': acc,
'loss': loss,
'train_op': train_op,
'enqueue_op': enqueue_op,
'batch_size': batch_size
}
def predict(model, test_set, test_labels):
print('\n\nPredicting on test set...')
preds = model.predict(test_set, verbose=1)
print('\nPredicted Class Indices: \n')
pred_class_indices = np.argmax(preds, axis=1)
print(pred_class_indices)
num_test = len(pred_class_indices)
print('Num predictions: ', num_test)
test_acc = accuracy(np.argmax(test_labels, axis=1), pred_class_indices)
print('\nTest set accuracy: {0:.2f}%'.format(
np.sum(test_acc) / num_test * 100))
def patch_based_cnn_model(sess, dropout_prob=0.5, l_rate=0.5, n_classes=2):
# Placeholders
img = tf.placeholder(tf.float32, shape=(None, 101, 101, 3))
labels = tf.placeholder(tf.float32, shape=(None, 2))
# Layers
convnet = Conv2D(80, 6, strides=1, padding='valid', activation=None, kernel_initializer='he_normal')(img)
convnet = tf.nn.local_response_normalization(convnet)
convnet = Activation('relu')(convnet)
convnet = MaxPooling2D(pool_size=(2, 2), strides=2)(convnet)
convnet = Conv2D(120, 5, strides=1, padding='valid', activation=None, kernel_initializer='he_normal')(convnet)
convnet = tf.nn.local_response_normalization(convnet)
convnet = Activation('relu')(convnet)
convnet = MaxPooling2D(pool_size=(2, 2), strides=2)(convnet)
convnet = Conv2D(160, 3, strides=1, padding='valid', activation=None, kernel_initializer='he_normal')(convnet)
convnet = Conv2D(200, 3, strides=1, padding='valid', activation=None, kernel_initializer='he_normal')(convnet)
convnet = MaxPooling2D(pool_size=(2, 2), strides=2)(convnet)
convnet = tf.reshape(convnet, [-1, 9*9*200])
convnet = Dense(320, activation='relu')(convnet)
convnet = Dropout(dropout_prob)(convnet)
convnet = Dense(320, activation='relu')(convnet)
convnet = Dropout(dropout_prob)(convnet)
preds = Dense(n_classes, activation='linear')(convnet)
preds = Activation('softmax')(preds)
sess.run(tf.global_variables_initializer())
# loss funtion
loss = tf.reduce_mean(categorical_crossentropy(labels, preds))
# Training operation
train_step = tf.train.GradientDescentOptimizer(l_rate).minimize(loss)
# Accurace metric
accuracy_metric = tf.reduce_mean(accuracy(labels, preds))
return [preds, accuracy_metric, img, labels, train_step]
def patch_based_cnn_model(dropout_prob=0.5, l_rate=0.5, n_classes=2):
# Placeholders
img = tf.placeholder(tf.float32, shape=(None, 101, 101, 3))
labels = tf.placeholder(tf.float32, shape=(None, 2))
# Layers
conv1 = Conv2D(80, 6, strides=1, padding='same', activation=None, kernel_initializer='he_normal')(img)
conv1 = tf.nn.local_response_normalisation(conv1) # FIXME
conv1 = Activation('relu')(conv1)
conv1 = MaxPooling2D(pool_size=(2, 2), strides=2)(conv1)
conv2 = Conv2D(120, 5, strides=1, padding='same', activation=None, kernel_initializer='he_normal')(conv1)
conv2 = tf.nn.local_response_normalisation(conv2) # FIXME
conv2 = Activation('relu')(conv2)
conv2 = MaxPooling2D(pool_size=(2, 2), strides=2)(conv2)
conv3 = Conv2D(160, 3, strides=1, padding='same', activation=None, kernel_initializer='he_normal')(conv2)
conv4 = Conv2D(200, 3, strides=1, padding='same', activation=None, kernel_initializer='he_normal')(conv3)
conv4 = MaxPooling2D(pool_size=(3, 3), strides=2)(conv4)
conv4_flatten = tf.reshape(conv4, [-1, 9*9*200])
dense1 = Dense(320, activation='relu')(conv4_flatten)
dense1 = Dropout(dropout_prob)(dense1)
dense2 = Dense(320, activation='relu')(dense1)
dense2 = Dropout(dropout_prob)(dense2)
preds = Dense(n_classes, activation='softmax')(dense2)
pred_clas = tf.argmax(preds, axis=1)
# loss funtion
loss = tf.reduce_mean(categorical_crossentropy(labels, preds))
# Training operation
train_step = tf.train.GradientDescentOptimizer(l_rate).minimize(loss)
# Accurace metric
acc_value = tf.reduce_mean(accuracy(labels, preds))
return [preds, pred_class, loss, train_step]
def tf_model():
sess = tf.Session()
K.set_session(sess)
print(K.learning_phase())
# This placeholder will contain our input digits, as flat vectors
img = tf.placeholder(tf.float32, shape=(None,784))
# Keras layers can be called on TensorFlow tensors:
x = Dense(128, activation='relu')(img) # fully-connected layer with 128 units and ReLU activation
x = Dropout(0.5)(x)
x = Dense(128, activation='relu')(x)
x = Dropout(0.5)(x)
preds = Dense(10, activation='softmax')(x) # output layer with 10 units and a softmax activation
labels = tf.placeholder(tf.float32, shape=(None, 10))
loss = tf.reduce_mean(categorical_crossentropy(labels, preds))
mnist_data = input_data.read_data_sets('MNIST_data', one_hot=True)
train_step = tf.train.GradientDescentOptimizer(0.5).minimize(loss)
# Initialize all variables
init_op = tf.global_variables_initializer()
sess.run(init_op)
# Run training loop
with sess.as_default():
for i in range(100):
print(K.learning_phase())
batch = mnist_data.train.next_batch(50)
train_step.run(feed_dict={img: batch[0],
labels: batch[1],
K.learning_phase(): 1})
acc_value = accuracy(labels, preds)
with sess.as_default():
print(acc_value.eval(feed_dict={img:mnist_data.test.images,
labels: mnist_data.test.labels}))
print('-------------------')
'''Load lata'''
from tensorflow.examples.tutorials.mnist import input_data
mnist_data = input_data.read_data_sets('MNIST_data', one_hot=True)
'''Train network - TensorFlow'''
# Training step
train_step = tf.train.AdamOptimizer(0.5).minimize(loss)
#TODO: Implement network training alternative in Keras
# Initialize all variables
init_op = tf.global_variables_initializer()
sess.run(init_op)
# Run training loop
with sess.as_default():
for i in range(10000):
batch = mnist_data.train.next_batch(50)
#train_step.run(feed_dict={img: batch[0], labels: batch[1]})
sess.run(train_step, feed_dict={img: batch[0], labels: batch[1]})
'''Evaluate model - Keras'''
from keras.metrics import categorical_accuracy as accuracy
acc_value = tf.reduce_mean(accuracy(labels, output_preds))
with sess.as_default():
print(
"Accuracy: ",
acc_value.eval(feed_dict={
img: mnist_data.test.images,
labels: mnist_data.test.labels
}))
for i in range(100):
batch = mnist_data.train.next_batch(50)
train_step.run(feed_dict={
img: batch[0],
labels: batch[1],
K.learning_phase(): 1
})
print("Epoch : " + str(ep + 1) + " loss is : " + str(
loss.eval(feed_dict={
img: batch[0],
labels: batch[1],
K.learning_phase(): 0
})))
#------------------------------------------ Accuracy metric
acc_value = tf.reduce_mean(accuracy(labels, preds))
with sess.as_default():
print(
acc_value.eval(
feed_dict={
img: mnist_data.test.images,
labels: mnist_data.test.labels,
K.learning_phase(): 0
}))
#------------------------------------------ Saving the model
saver = tf.train.Saver()
saver.save(
sess, './tmp/models/mnist_ff_keras/model_ff_keras'
) # Let say global_step are the number of epochs the model has gone through
def accuracy(logits, labels):
ypred = tf.cast(tf.round(logits), tf.int32)
ytrue = tf.cast(labels, tf.int32)
from keras.metrics import binary_accuracy as accuracy
return tf.reduce_mean(accuracy(ytrue, ypred))
# final_tensor = tf.nn.sigmoid(outputs, name="outputs")
# cross_entropy = tf.nn.sigmoid_cross_entropy_with_logits(logits=outputs,labels=targets)
loss = tf.reduce_mean(categorical_crossentropy(targets, outputs))
#loss = tf.reduce_mean(binary_crossentropy(targets, outputs))
#mean_loss = tf.reduce_mean(cross_entropy) ###### CHECK IF CROSS ENTROPY is the LOSS
#optimizer=tf.train.AdamOptimizer(learning_rate=0.001).minimize(loss);
optimizer = tf.train.GradientDescentOptimizer(0.001).minimize(loss)
##### PLA WITH THIS FOR ACCURACY
#out_equals_target = tf.equal(tf.arg_max(outputs,1), tf.arg_max(targets,1))
acc_value = accuracy(targets, outputs)
sess = tf.InteractiveSession()
initializer = tf.global_variables_initializer()
sess.run(initializer)
K.set_session(sess)
batch_size = 100
max_epochs = 50000
prev_validation_loss = 9999999.
for epoch_counter in range(max_epochs):
cur_epoch_loss = 0
with np.load(data_folder + 'test_75k.npz') as data:
def run(LR, val, RNN_TYPE, TIMESTEPS = 64, GPU_FLAG=True, NUM_BATCH = None, SAVE_WEIGHTS = False, VERBOSE = True):
quantify = identity#deterministic_ternary(val)
num_timesteps = TIMESTEPS
train_g, test_g, train_s, test_s = small_generators(batch_size)
num_batch_in_epoch = train_s[0]
num_batch = 2 * num_batch_in_epoch if not NUM_BATCH else NUM_BATCH
_init = ternary_choice(val) if val is not np.inf else "uniform"
i_init = ternary_choice(val) if val is not np.inf else "orthogonal"
###########################
#### Model Definition ####
i = tf.placeholder(tf.float32, shape=(batch_size, num_timesteps, num_inputs), name="X")
labels = tf.placeholder(tf.float32, shape=(batch_size, num_timesteps, num_classes), name="y")
with tf.device('/gpu:2') if RNN_TYPE != Clockwork and GPU_FLAG else tf.device('/cpu:0'):
if RNN_TYPE == Clockwork:
layer1 = RNN_TYPE(HIDDEN_SIZE, init = _init, inner_init= i_init, periods=[1, 2, 4, 8, 16], stateful=True, return_sequences=True)
h1 = layer1(i)
else:
layer1 = RNN_TYPE(HIDDEN_SIZE, init = _init, inner_init = i_init ,stateful=True, return_sequences=True)
h1 = layer1(i)
with tf.device('/gpu:3') if RNN_TYPE != Clockwork and GPU_FLAG else tf.device('/cpu:0'):
if RNN_TYPE == Clockwork:
layer2 = RNN_TYPE(num_classes, init = _init, inner_init= i_init, periods=[1, 2, 4, 8, 16, 32, 64], stateful=True, return_sequences=True)
o = layer2(h1)
else:
layer2 = RNN_TYPE(num_classes, init= _init, inner_init= i_init, stateful=True, return_sequences=True)
o = layer2(h1)
loss = tf.reduce_mean(categorical_crossentropy(labels, o))
acc_value = accuracy(labels, o)
real_valued = []
var_to_real = {}
for w in tf.trainable_variables():
v = tf.Variable(w.initialized_value(), dtype=tf.float32, trainable=False)
var_to_real[w] = v
real_valued.append(v)
update_ops = []
for old_value, new_value in layer1.updates + layer2.updates:
update_ops.append(tf.assign(old_value, new_value).op)
##########################
#### Train loop ####
learning_rate = tf.placeholder(tf.float32, shape=(), name="LR")
optimizer = tf.train.AdamOptimizer(learning_rate)
grads_vars = optimizer.compute_gradients(loss)
#grads = []
#for g, v in grads_vars:
# grads.append(g)
#clipped_grads, global_norm = tf.clip_by_global_norm(grads, 1)
for k, (grad, var) in enumerate(grads_vars):
grads_vars[k] = (tf.clip_by_value(grad, -1, 1), var_to_real[var])
app = optimizer.apply_gradients(grads_vars)
assignments = [tf.assign(w, quantify(var_to_real[w])).op for w in tf.trainable_variables()]
clips = [tf.assign(w, tf.clip_by_value(w, -val, val)).op for w in real_valued]
saver = tf.train.Saver()
losses = []
accuracies = []
#T.silence()
lr = LR
init_op = tf.initialize_all_variables()
with sess.as_default():
init_op.run()
for w in tf.trainable_variables():
tf.assign(var_to_real[w], w).op.run()
for batch in np.arange(num_batch):
for assign in assignments:
assign.run()
X, y = train_g.next()
app.run(feed_dict={i: X, labels: y, learning_rate: lr})
for op in update_ops:
op.run(feed_dict={i: X})
for w in clips:
w.run()
if batch % how_often == 0:
validation_loss = 0.0
validation_acc = 0.0
count = 0
while count < test_s[0]:
(test_X, test_Y) = test_g.next()
curr_loss = loss.eval(feed_dict={i: test_X, labels: test_y})
acc = acc_value.eval(feed_dict={i: test_X, labels: test_y})
validation_loss += curr_loss
validation_acc += acc
count += 1
if done: break
losses.append(validation_loss / count)
accuracies.append(validation_acc / count)
if SAVE_WEIGHTS: saver.save(sess, SAVE_PATH)
if VERBOSE:
printProgress(batch, num_batch_in_epoch, how_often, losses[-1])
print("Accuracy at last batch: {0}".format(validation_acc / count))
with open(LOSS_PATH + "{0}_{1}_{2}_{3}.w".format(LR, val, RNN_TYPE.__name__, TIMESTEPS), "wb") as f:
pickle.dump([losses, accuracies], f)
lr *= .9
x = Dense(128, activation='relu')(img)
x = Dropout(0.5)(x)
x = Dense(128, activation='relu')(x)
x = Dropout(0.5)(x)
preds = Dense(10, activation='softmax')(x)
loss = tf.reduce_mean(categorical_crossentropy(labels, preds))
train_step = tf.train.GradientDescentOptimizer(0.5).minimize(
loss) # TensorFlow model
print('Initializing variables')
sess.run(tf.global_variables_initializer())
with sess.as_default():
for i in range(100):
batch = mnist_data.train.next_batch(50)
train_step.run(feed_dict={
img: batch[0],
labels: batch[1],
K.learning_phase(): 1
}) # K.learning_phase()为1，表示为训练模式， K.learning_phase()为0，表示为测试模式
acc_value = accuracy(labels, preds) # keras model
with sess.as_default():
print(
acc_value.eval(
feed_dict={
img: mnist_data.test.images,
labels: mnist_data.test.labels,
K.learning_phase(): 0
}))
for i in range(epochs):
distibutedTensorFlow.trainbatches(X, Y, dask_spec)
else:
results = []
device_name = "/cpu:0"
with tf.device(device_name):
x, y = retmodel()
opt = tf.train.AdamOptimizer
labels = tf.placeholder(tf.float32, shape=(None, outputdim))
encode_y = tf.argmax(y, axis=1)
encode_labels = tf.argmax(labels, axis=1)
confusion = tf.confusion_matrix(encode_labels, encode_y)
acc_value = accuracy(labels, y)
loss = tf.reduce_mean(categorical_crossentropy(labels, y))
#train_step = opt.minimize(loss)
train_step = tf.train.AdamOptimizer().minimize(loss)
sess = tf.Session()
saver = tf.train.Saver()
init_op = tf.global_variables_initializer()
sess.run(init_op)
X_train, X_test, y_train, y_test = train_test_split(X,
dummy_y,
test_size=0.15,
random_state=42)
layer3.trainable_weights,
grad_ys=p_gy_l3)
x_l4 = y_l3
y_l4 = layer4(x_l4)
loss = K.mean(categorical_crossentropy(labels, y_l4))
gy_l3 = tf.gradients(loss, y_l3)[0]
loss_dni3 = K.mean(K.sum((p_gy_l3 - gy_l3)**2, 1))
grad_trainable_weights_dni3 = tf.gradients(loss_dni3, cDNI3.trainable_weights)
grad_trainable_weights_l4 = tf.gradients(loss, layer4.trainable_weights)
gparams = grad_trainable_weights_l1 + grad_trainable_weights_dni1 + grad_trainable_weights_l2 + grad_trainable_weights_dni2 + grad_trainable_weights_l3 + grad_trainable_weights_dni3 + grad_trainable_weights_l4
with tf.control_dependencies(gparams):
updates = optimizer.get_updates(params, gparams)
acc = accuracy(labels, y_l4)
mnist_data = input_data.read_data_sets('MNIST_data', one_hot=True)
with sess.as_default():
sess.run(tf.initialize_all_variables())
for i in range(500000):
batch = mnist_data.train.next_batch(256)
sess.run(updates,
feed_dict={
x: batch[0],
labels: batch[1],
K.learning_phase(): 1
})
if i % 1000 == 0:
print "epoch: {}".format(256 * i // len(mnist_data.train.images))
model.compile(optimizer=optimizer,
metrics=['acc'],
loss="categorical_crossentropy")
return model
def create_hyper():
batches = [50]
optimizer = ['adadelta']
droptout = [0.3]
return {"batch_size": batches, "optimizer": optimizer, "drop": droptout}
model = KerasClassifier(build_fn=build_model, verbose=1)
hyperparameters = create_hyper()
search = RandomizedSearchCV(estimator=model,
param_distributions=hyperparameters,
n_iter=5)
search.fit(x_train, y_train)
print(search.best_params_)
pred = search.predict(x_test)
pred = np_utils.to_categorical(pred)
# print(pred)
# print(y_test)
score = accuracy(y_test, pred)
# score = search.score(x_test)
print(score)
def expr_score(y_true, y_pred):
f1 = f1_score(y_true, y_pred)
acc = accuracy(y_true, y_pred)
return 0.67 * f1 + 0.33 * acc
p_gy_p = cDNI2(K.concatenate((y_p, labels), axis=1))
grad_trainable_weights_p = tf.gradients(y_p, layer_p.trainable_weights, grad_ys=p_gy_p)
x_n = y_p
y_n = layer_n(x_n)
loss = K.mean(categorical_crossentropy(labels, y_n))
grad_trainable_weights_n = tf.gradients(loss, layer_n.trainable_weights)
gy_p = tf.gradients(loss, y_p)
loss_dni2 = K.mean(K.sum((p_gy_p-gy_p)**2, 1))
grad_trainable_weights_dni2 = tf.gradients(loss_dni2, cDNI2.trainable_weights)
with tf.control_dependencies(grad_trainable_weights_dni2+grad_trainable_weights_p+grad_trainable_weights_n):
gparams = grad_trainable_weights_n+grad_trainable_weights_dni2+grad_trainable_weights_p
updates = optimizer.get_updates(params, gparams)
acc = accuracy(labels, y_n)
mnist_data = input_data.read_data_sets('MNIST_data', one_hot=True)
with sess.as_default():
sess.run(tf.initialize_all_variables())
for i in range(500000):
batch = mnist_data.train.next_batch(256)
sess.run(updates, feed_dict={x: batch[0], labels: batch[1], K.learning_phase(): 1})
if i%100 == 0:
print "epoch: {}".format(256*i//len(mnist_data.train.images))
print "acc: {}".format(acc.eval(feed_dict={x: mnist_data.test.images, labels: mnist_data.test.labels, K.learning_phase(): 0}))
print "loss: {}".format(loss.eval({x: mnist_data.test.images, labels: mnist_data.test.labels, K.learning_phase(): 0}))
def run(LR,
val,
RNN_TYPE,
TIMESTEPS=128,
quant=None,
GPU_FLAG=True,
NUM_EPOCH=1,
NUM_BATCH=None,
SAVE_WEIGHTS=False,
VERBOSE=True,
WHICH=None):
if quant is None: assert val is np.inf
quantify = identity if quant is None else quant(val)
num_timesteps = TIMESTEPS
num_classes, c_to_l, l_to_c = p_char_mapping(TEXT)
t_g = text_generator(TEXT, num_timesteps, batch_size, percent=.95)
test_g = text_generator(TEXT,
num_timesteps,
batch_size,
percent=.05,
from_back=True)
x_y_generator = data_target_generator(t_g, c_to_l, num_classes)
test_generator = data_target_generator(test_g, c_to_l, num_classes)
num_batch_in_epoch = data_len(TEXT, batch_size, num_timesteps, percent=.95)
num_test = data_len(TEXT, batch_size, num_timesteps, percent=.05)
num_batch = NUM_EPOCH * num_batch_in_epoch if not NUM_BATCH else NUM_BATCH
how_often = num_batch_in_epoch // 4
_init = "he_normal" if val == np.inf else ternary_choice(val)
i_init = "identity"
###########################
#### Model Definition ####
i = tf.placeholder(tf.float32,
shape=(batch_size, num_timesteps, num_classes + 1),
name="X")
labels = tf.placeholder(tf.float32,
shape=(batch_size, num_timesteps, num_classes + 1),
name="y")
with tf.device('/gpu:0'):
if RNN_TYPE == Clockwork:
layer1 = RNN_TYPE(HIDDEN_SIZE,
init=_init,
inner_init=i_init,
periods=[1, 2, 4, 8, 16, 32],
stateful=True,
return_sequences=True)
h1 = layer1(i)
else:
layer1 = RNN_TYPE(HIDDEN_SIZE,
init=_init,
inner_init=i_init,
stateful=True,
return_sequences=True)
h1 = layer1(i)
with tf.device('/gpu:1'):
if RNN_TYPE == Clockwork:
layer2 = RNN_TYPE(HIDDEN_SIZE,
init=_init,
inner_init=i_init,
periods=[1, 2, 4, 8, 16, 32],
stateful=True,
return_sequences=True)
h2 = layer2(h1)
else:
layer2 = RNN_TYPE(HIDDEN_SIZE,
init=_init,
inner_init=i_init,
stateful=True,
return_sequences=True)
h2 = layer2(h1)
with tf.device('/gpu:2'):
layer3 = TimeDistributed(Dense(num_classes + 1, init=_init))
o = layer3(h2)
loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(o, labels))
acc_value = accuracy(labels, o)
global_step = tf.Variable(0, trainable=False)
to_quantize = []
if WHICH == "all":
to_quantize = tf.trainable_variables()
elif WHICH == "hidden":
for v in tf.trainable_variables():
if "U" in v.name.split("_") or "U:0" in v.name.split("_"):
to_quantize.append(v)
elif WHICH == "input":
for v in tf.trainable_variables():
if "W" in v.name.split("_") or "W:0" in v.name.split("_"):
to_quantize.append(v)
if "b" in v.name.split("_") or "b:0" in v.name.split("_"):
to_quantize.append(v)
real_valued = []
var_to_real = {}
for w in to_quantize:
v = tf.Variable(w.initialized_value(),
dtype=tf.float32,
trainable=False)
var_to_real[w] = v
real_valued.append(v)
update_ops = []
for old_value, new_value in layer1.updates + layer2.updates:
update_ops.append(tf.assign(old_value, new_value).op)
##########################
#### Train loop ####
learning_rate = tf.placeholder(tf.float32, shape=(), name="LR")
optimizer = tf.train.AdamOptimizer(learning_rate)
grads_vars = optimizer.compute_gradients(
loss,
colocate_gradients_with_ops=True
if RNN_TYPE is not Clockwork else False)
grads = []
vars_ = []
for k, (g, v) in enumerate(grads_vars):
vars_.append(v)
grads.append(g)
clipped_grads, global_norm = tf.clip_by_global_norm(grads, 1)
for k, (grad, var) in enumerate(grads_vars):
grads_vars[k] = (clipped_grads[k],
var_to_real[var] if var in to_quantize else var)
app = optimizer.apply_gradients(grads_vars, global_step=global_step)
assignments = [tf.assign(w, quantify(var_to_real[w])) for w in to_quantize]
clips = [
tf.assign(w, tf.clip_by_value(w, -val, val)).op for w in real_valued
]
saver = tf.train.Saver()
losses = []
accuracies = []
#T.silence()
lr = LR
init_op = tf.initialize_all_variables()
with sess.as_default():
init_op.run()
for batch in np.arange(num_batch):
sess.run(assignments)
X, y = x_y_generator.next()
app.run(feed_dict={i: X, labels: y, learning_rate: lr})
sess.run(update_ops, feed_dict={i: X})
sess.run(clips)
if batch % how_often == 0:
validation_loss = 0.0
validation_acc = 0.0
count = 0
while count < num_test:
test_X, test_y = test_generator.next()
curr_loss, acc = sess.run([loss, acc_value], {
i: test_X,
labels: test_y
})
validation_loss += curr_loss
validation_acc += acc
count += 1
losses.append(validation_loss / count)
accuracies.append(validation_acc / count)
if SAVE_WEIGHTS: saver.save(sess, SAVE_PATH)
if VERBOSE:
printProgress(batch, num_batch_in_epoch, how_often,
losses[-1])
print("Accuracy at last batch: {0}".format(validation_acc /
count))
with open(
LOSS_PATH + "{0}_{1}_{2}_{3}_{4}.w".format(
val, RNN_TYPE.__name__, TIMESTEPS, WHICH,
quant.__name__), "wb") as f:
pickle.dump([losses, accuracies], f)
if (validation_acc /
count) < 0.005 and batch > num_batch / 2 or (
validation_acc == 0 and batch > num_batch / 5):
print("Returning early due to failure.")
return
def compute_accuracy(y_pred=[], y_true=[]):
y_pred, y_true = np.array([y if y else -1 for y in y_pred], dtype=int), \
np.array(y_true, dtype=int)
acc = metrics.accuracy(y_pred, y_true)
return K.eval(K.sum(acc)) / y_true.shape[0]
plt.xlim(lims)
plt.ylim(lims)
_ = plt.plot(lims, lims)
error = pred_test - target_test
plt.hist(error, bins = 25)
plt.xlabel("Prediction Error [OIL_RATE]")
_ = plt.ylabel("Count")
# new Data predict
#[['CHOKE','WHP','FLP','WHT']]
new_test_data =[[39,1719,339,189]]
new_pred_test = model.predict(pd.DataFrame(new_test_data, columns=['CHOKE','WHP','FLP','WHT']))
print("Shape: {}".format(pred_test.shape))
print(pred_test)
mean_square_error = mean_squared_error(target_test,pred_test)
# The mean squared error
print('Mean squared error: %.2f' % mean_square_error)
mean_square_logarithmic_error = mean_squared_logarithmic_error(target_test, pred_test)
# The mean squared error
print('Mean squared logarithmic error: %.2f' % mean_square_logarithmic_error)
acc = metrics.accuracy(target_test, pred_test)
print('Accuracy: {}'.format(acc))
#print_summary(model, line_length=None, positions=None, print_fn=None)
def run(LR,
val,
RNN_TYPE,
TIMESTEPS=None,
quant=None,
GPU_FLAG=True,
NUM_EPOCH=1,
NUM_BATCH=None,
SAVE_WEIGHTS=False,
VERBOSE=True,
WHICH=None):
tf.reset_default_graph()
sess = tf.Session()
K.set_session(sess)
K.manual_variable_initialization(True)
if quant is None:
assert val is np.inf
quant = identity
w_val, u_val = np.inf, np.inf
else:
w_val, u_val = val
num_timesteps = TIMESTEPS
g = music_generator(MOLDAU,
batch_size,
num_timesteps,
percent=.0001,
offset=0.1)
test_g = music_generator(MOLDAU,
batch_size,
num_timesteps,
percent=.0001,
offset=0.1)
quantify_w = quant(w_val)
quantify_u = quant(u_val)
x_y_generator = music_pair_generator(g)
test_generator = music_pair_generator(test_g)
num_batch_in_epoch, num_classes, train_timesteps = music_len(MOLDAU,
batch_size,
num_timesteps,
percent=.0001,
offset=0.1)
num_test, _, test_timesteps = music_len(MOLDAU,
batch_size,
num_timesteps,
percent=.0001,
offset=0.1)
num_batch = NUM_EPOCH * num_batch_in_epoch if not NUM_BATCH else NUM_BATCH
how_often = 1
_init = nary_uniform if val == np.inf else ternary_choice(w_val)
b_init = zero if val == np.inf else ternary_choice(u_val)
i_init = scale_identity(1.0) if val == np.inf else scale_identity(u_val)
if num_timesteps is None:
num_timesteps = train_timesteps - 1
###########################
#### Model Definition ####
i = tf.placeholder(tf.float32,
shape=(batch_size, num_timesteps, num_classes),
name="X")
labels = tf.placeholder(tf.float32,
shape=(batch_size, num_timesteps, num_classes),
name="y")
with tf.device('/gpu:0') if GPU_FLAG else tf.device('/cpu:0'):
if RNN_TYPE == Clockwork:
layer1 = RNN_TYPE(HIDDEN_SIZE,
init=_init,
inner_init=i_init,
bias_init=b_init,
periods=[1, 2, 4, 8, 16, 32, 64, 128, 256],
return_sequences=True)
h1 = layer1(i)
elif RNN_TYPE == GRU:
layer1 = RNN_TYPE(HIDDEN_SIZE,
init=_init,
inner_init=i_init,
return_sequences=True)
h1 = layer1(i)
else:
layer1 = RNN_TYPE(HIDDEN_SIZE,
init=_init,
inner_init=i_init,
bias_init=b_init,
return_sequences=True)
h1 = layer1(i)
with tf.device('/gpu:1') if GPU_FLAG else tf.device('/cpu:0'):
if RNN_TYPE == Clockwork:
layer2 = RNN_TYPE(HIDDEN_SIZE,
init=_init,
inner_init=i_init,
bias_init=b_init,
periods=[1, 2, 4, 8, 16, 32, 64, 128, 256],
return_sequences=True)
h2 = layer2(h1)
elif RNN_TYPE == GRU:
layer2 = RNN_TYPE(HIDDEN_SIZE,
init=_init,
inner_init=i_init,
return_sequences=True)
h2 = layer2(h1)
else:
layer2 = RNN_TYPE(HIDDEN_SIZE,
init=_init,
inner_init=i_init,
bias_init=b_init,
return_sequences=True)
h2 = layer2(h1)
with tf.device('/gpu:2') if GPU_FLAG else tf.device('/cpu:0'):
layer3 = TimeDistributed(Dense(num_classes, init=_init))
o = layer3(h2)
loss = tf.reduce_mean(mean_squared_error(labels, o))
acc_value = accuracy(labels, o)
global_step = tf.Variable(0, trainable=False)
to_quantize_w = []
to_quantize_u = []
for v in tf.trainable_variables():
if "U" in v.name.split("_") or "U:0" in v.name.split(
"_") or "b" in v.name.split("_") or "b:0" in v.name.split("_"):
to_quantize_u.append(v)
else:
to_quantize_w.append(v)
real_valued_u = []
var_to_real_u = {}
for w in to_quantize_u:
v = tf.Variable(w.initialized_value(),
dtype=tf.float32,
trainable=False)
var_to_real_u[w] = v
real_valued_u.append(v)
real_valued_w = []
var_to_real_w = {}
for w in to_quantize_w:
v = tf.Variable(w.initialized_value(),
dtype=tf.float32,
trainable=False)
var_to_real_w[w] = v
real_valued_w.append(v)
update_ops = []
# for old_value, new_value in layer1.updates + layer2.updates:
# update_ops.append(tf.assign(old_value, new_value).op)
##########################
#### Train loop ####
learning_rate = tf.placeholder(tf.float32, shape=(), name="LR")
optimizer = tf.train.AdamOptimizer(learning_rate)
grads_vars = optimizer.compute_gradients(
loss,
colocate_gradients_with_ops=True
if RNN_TYPE is not Clockwork else False)
grads = []
vars_ = []
for k, (g, v) in enumerate(grads_vars):
vars_.append(v)
grads.append(g)
# print(v)
# print(g)
#clipped_grads, global_norm = tf.clip_by_global_norm(grads, 1)
for k, (grad, var) in enumerate(grads_vars):
if var in to_quantize_u:
new_var = var_to_real_u[var]
elif var in to_quantize_w:
new_var = var_to_real_w[var]
else:
new_var = var
grads_vars[k] = (tf.clip_by_value(grad, -10, 10), new_var)
# grads_vars[k] = (clipped_grads[k], var_to_real[var] if var in to_quantize else var)
app = optimizer.apply_gradients(grads_vars, global_step=global_step)
assignments_u = [
tf.assign(w, quantify_u(var_to_real_u[w])) for w in to_quantize_u
]
clips_u = [
tf.assign(w, tf.clip_by_value(w, -u_val, u_val)).op
for w in real_valued_u
]
assignments_w = [
tf.assign(w, quantify_w(var_to_real_w[w])) for w in to_quantize_w
]
clips_w = [
tf.assign(w, tf.clip_by_value(w, -w_val, w_val)).op
for w in real_valued_w
]
saver = tf.train.Saver()
losses = []
accuracies = []
#T.silence()
lr = LR
init_op = tf.global_variables_initializer()
with sess.as_default():
init_op.run()
for batch in np.arange(num_batch):
sess.run([assignments_u, assignments_w])
X, y = x_y_generator.next()
fd = {i: np.zeros_like(X), labels: y, learning_rate: lr}
app.run(feed_dict=fd)
# print(var_to_real_u[vars_[1]].eval())
# print(grads[1].eval(feed_dict=fd))
sess.run(update_ops, feed_dict=fd)
sess.run([clips_w, clips_u])
if batch % how_often == 0:
validation_loss = 0.0
validation_acc = 0.0
count = 0
while count < num_test:
test_X, test_y = test_generator.next()
curr_loss, acc = sess.run([loss, acc_value], {
i: np.zeros_like(test_X),
labels: test_y
})
validation_loss += curr_loss
validation_acc += acc
count += 1
losses.append(validation_loss / count)
accuracies.append(validation_acc / count)
if SAVE_WEIGHTS: saver.save(sess, SAVE_PATH)
if VERBOSE:
printProgress(batch, num_batch_in_epoch, how_often,
losses[-1])
print("Accuracy at last batch: {0}".format(validation_acc /
count))
with open(
LOSS_PATH + "{0}_{1}_{2}_{3}_{4}.w".format(
val, RNN_TYPE.__name__, TIMESTEPS, WHICH,
quant.__name__ if quant is not None else ""),
"wb") as f:
pickle.dump([losses, accuracies], f)
if (validation_acc /
count) < 0.005 and batch > num_batch / 2 or (
validation_acc == 0 and batch > num_batch / 5):
print("Returning early due to failure.")
return
return_sequences=False,
W_regularizer=l2(0.01),
inner_activation='sigmoid')(lstm_input4)
var2 = Dense(100, bias=False, activation='tanh')(var0)
var = Dropout(0.5)(var2)
predictions = Dense(2, bias=False, activation='softmax')(var)
labels = tf.placeholder(tf.float32, shape=(None, 2))
loss = tf.reduce_mean(binary_crossentropy(labels, predictions))
train_step = tf.train.AdagradOptimizer(learning_rate=0.1).minimize(loss)
acc_value = accuracy(labels, predictions)
print '-------Model building complete--------'
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
# load dataset embedding
dataset = pickle.load(open('..data/Chinese/character/allembedding.p', 'r'))
dataset_size = dataset.shape[0]
# load dataset label
label0 = codecs.open('..data/Chinese/character/alllabel.txt', 'r',
'utf8').readlines()
label = []
for e in label0:
def run(LR,
val,
RNN_TYPE,
TIMESTEPS=1,
quant=None,
GPU_FLAG=True,
NUM_EPOCH=1,
NUM_BATCH=None,
SAVE_WEIGHTS=False,
VERBOSE=True,
WHICH=None):
if quant is None: assert val is np.inf
quantify = identity if quant is None else quant(val)
num_timesteps = TIMESTEPS
num_classes, c_to_l, l_to_c = p_char_mapping(TEXT)
t_g = text_generator(TEXT, num_timesteps, batch_size, percent=.95)
test_g = text_generator(TEXT,
num_timesteps,
batch_size,
percent=.05,
from_back=True)
x_y_generator = data_target_generator(t_g, c_to_l, num_classes)
test_generator = data_target_generator(test_g, c_to_l, num_classes)
num_batch_in_epoch = data_len(TEXT, batch_size, num_timesteps, percent=.95)
num_test = data_len(TEXT, batch_size, num_timesteps, percent=.05)
num_batch = NUM_EPOCH * num_batch_in_epoch if not NUM_BATCH else NUM_BATCH
how_often = num_batch_in_epoch // 4
_init = "he_normal" if val == np.inf else ternary_choice(val)
i_init = "identity"
loaded_weights = pickle.load(open("../results/weights/weights.weights"))[0]
###########################
#### Model Definition ####
i = tf.placeholder(tf.float32,
shape=(batch_size, num_timesteps, num_classes + 1),
name="X")
labels = tf.placeholder(tf.float32,
shape=(batch_size, num_timesteps, num_classes + 1),
name="y")
with tf.device('/gpu:0') if GPU_FLAG else tf.device('/cpu:0'):
if RNN_TYPE == Clockwork:
layer1 = RNN_TYPE(HIDDEN_SIZE,
init=_init,
inner_init=i_init,
periods=[1, 2, 4, 8, 16, 32],
stateful=True,
return_sequences=True)
h1 = layer1(i)
else:
layer1 = RNN_TYPE(HIDDEN_SIZE,
init=_init,
inner_init=i_init,
stateful=True,
return_sequences=True)
h1 = layer1(i)
with tf.device('/gpu:1') if GPU_FLAG else tf.device('/cpu:0'):
if RNN_TYPE == Clockwork:
layer2 = RNN_TYPE(HIDDEN_SIZE,
init=_init,
inner_init=i_init,
periods=[1, 2, 4, 8, 16, 32],
stateful=True,
return_sequences=True)
h2 = layer2(h1)
else:
layer2 = RNN_TYPE(HIDDEN_SIZE,
init=_init,
inner_init=i_init,
stateful=True,
return_sequences=True)
h2 = layer2(h1)
with tf.device('/gpu:2') if GPU_FLAG else tf.device('/cpu:0'):
layer3 = TimeDistributed(Dense(num_classes + 1, init=_init))
o = layer3(h2)
loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(o, labels))
acc_value = accuracy(labels, o)
update_ops = []
for old_value, new_value in layer1.updates + layer2.updates:
update_ops.append(tf.assign(old_value, new_value).op)
name_v = {v.name: v for v in tf.trainable_variables()}
intitial_assignments = []
for (name, val) in loaded_weights:
intitial_assignments.append(name_v[name].assign_add(tf.constant(val)))
layer_2_activations = []
init_op = tf.initialize_all_variables()
with sess.as_default():
init_op.run()
sess.run(intitial_assignments)
for batch in np.arange(num_batch):
X, y = x_y_generator.next()
acts = h2.eval(feed_dict={i: X})
layer_2_activations.append(acts)
sess.run(update_ops, feed_dict={i: X})
with open(LOSS_PATH + "layer_2.a", "wb"):
pickle.dump([layer_2_activations], f)
from keras.layers import Input,Dense,Dropout
img=Input(shape=(784,))
x=Dense(128,activation='relu')(img)
x=Dropout(0.5)(x)
x=Dense(128,activation='relu')(x)
x=Dropout(0.5)(x)
preds=Dense(10,activation='softmax')(x)
labels=Input(shape=(10,))
loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=preds,labels=labels))
from tensorflow.examples.tutorials.mnist import input_data
mnist_data = input_data.read_data_sets('MNIST_data', one_hot=True)
train_step=tf.train.GradientDescentOptimizer(0.5).minimize(loss)
for _ in range(10):
batch=mnist_data.train.next_batch(50)
sess.run(train_step,feed_dict={img:batch[0],labels:batch[1],K.learning_phase():1})
from keras.metrics import categorical_accuracy as accuracy
acc_value=accuracy(labels,preds)
print (sess.run(acc_value,feed_dict={img:mnist_data.test.images,labels:mnist_data.test.labels,K.learning_phase():0}))
def step(self):
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
K.set_session(sess)
def zero_pad_channels(x, pad=0):
"""
Function for Lambda layer
"""
pattern = [[0, 0], [0, 0], [0, 0], [pad - pad // 2, pad // 2]]
return tf.pad(x, pattern)
def residual_block(x, nb_filters=16, subsample_factor=1):
prev_nb_channels = K.int_shape(x)[3]
if subsample_factor > 1:
subsample = (subsample_factor, subsample_factor)
# shortcut: subsample + zero-pad channel dim
shortcut = AveragePooling2D(pool_size=subsample)(x)
else:
subsample = (1, 1)
# shortcut: identity
shortcut = x
if nb_filters > prev_nb_channels:
shortcut = Lambda(
zero_pad_channels,
arguments={'pad': nb_filters - prev_nb_channels})(shortcut)
y = BatchNormalization(axis=3)(x)
y = Activation('relu')(y)
y = Convolution2D(nb_filters,
3,
3,
subsample=subsample,
init='he_normal',
border_mode='same')(y)
y = BatchNormalization(axis=3)(y)
y = Activation('relu')(y)
y = Convolution2D(nb_filters,
3,
3,
subsample=(1, 1),
init='he_normal',
border_mode='same')(y)
out = merge([y, shortcut], mode='sum')
return out
# this placeholder will contain our input digits
img = tf.placeholder(tf.float32,
shape=(None, self.img_col, self.img_row,
self.img_channels))
labels = tf.placeholder(tf.float32, shape=(None, self.nb_classes))
# img = K.placeholder(ndim=4)
# labels = K.placeholder(ndim=1)
# Keras layers can be called on TensorFlow tensors:
x = Convolution2D(16, 3, 3, init='he_normal', border_mode='same')(img)
for i in range(0, self.blocks_per_group):
nb_filters = 16 * self.widening_factor
x = residual_block(x, nb_filters=nb_filters, subsample_factor=1)
for i in range(0, self.blocks_per_group):
nb_filters = 32 * self.widening_factor
if i == 0:
subsample_factor = 2
else:
subsample_factor = 1
x = residual_block(x,
nb_filters=nb_filters,
subsample_factor=subsample_factor)
for i in range(0, self.blocks_per_group):
nb_filters = 64 * self.widening_factor
if i == 0:
subsample_factor = 2
else:
subsample_factor = 1
x = residual_block(x,
nb_filters=nb_filters,
subsample_factor=subsample_factor)
x = BatchNormalization(axis=3)(x)
x = Activation('relu')(x)
x = AveragePooling2D(pool_size=(8, 8),
strides=None,
border_mode='valid')(x)
x = tf.reshape(x, [-1, np.prod(x.get_shape()[1:].as_list())])
# Readout layer
preds = Dense(self.nb_classes, activation='softmax')(x)
loss = tf.reduce_mean(categorical_crossentropy(labels, preds))
optimizer = tf.train.GradientDescentOptimizer(0.5).minimize(loss)
'''
with sess.as_default():
for i in range(10):
batch = self.next_batch(self.batch_num)
_, l = sess.run([optimizer, loss],
feed_dict={img: batch[0], labels: batch[1]})
print(l)
'''
with sess.as_default():
batch = self.next_batch(self.batch_num)
_, l = sess.run([optimizer, loss],
feed_dict={
img: batch[0],
labels: batch[1]
})
print('Loss', l)
acc_value = accuracy(labels, preds)
'''
img = tf.placeholder(tf.float32, shape=(None, 784))
labels = tf.placeholder(tf.float32, shape=(None, 10))
model = Sequential()
model.add(Dense(128, activation='relu', input_dim=784))
model.add(Dropout(0.5))
model.add(Dense(128, activation='relu'))
model.add(Dropout(0.5))
model.add(Dense(10, activation='softmax'))
preds = model(img)
loss = tf.reduce_mean(categorical_crossentropy(labels, preds))
train_step = tf.train.GradientDescentOptimizer(0.5).minimize(loss)
with sess.as_default():
for i in range(100):
batch = mnist_data.train.next_batch(50)
train_step.run(feed_dict={
img: batch[0],
labels: batch[1],
K.learning_phase(): 1
})
acc_value = accuracy(labels, preds)
with sess.as_default():
print acc_value.eval(
feed_dict={
img: mnist_data.test.images,
labels: mnist_data.test.labels,
K.learning_phase(): 0
})
def main(_):
# Needed to make sure the logging output is visible.
# See https://github.com/tensorflow/tensorflow/issues/3047
tf.logging.set_verbosity(tf.logging.INFO)
# Prepare necessary directories that can be used during training
# prepare_file_system()
# Look at the folder structure, and create lists of all the images.
image_lists = create_image_lists(GENERAL_SETTING['image_dir'],
GENERAL_SETTING['testing_percentage'],
GENERAL_SETTING['validation_percentage'])
class_count = len(image_lists.keys())
if class_count == 0:
tf.logging.error('No valid folders of images found at ' +
FLAGS.image_dir)
return -1
if class_count == 1:
tf.logging.error('Only one valid folder of images found at ' +
FLAGS.image_dir +
' - multiple classes are needed for classification.')
return -1
sess = tf.Session()
K.set_session(sess)
base_model = InceptionV3(weights='imagenet',
include_top=False,
input_shape=(299, 299, 3))
for layer in base_model.layers:
layer.trainable = False
x = base_model.output
x = GlobalAveragePooling2D()(x)
x = Dense(64, input_shape=(2048, ), activation='relu')(x)
# x = Dropout(0.2)(x)
predictions = Dense(10, activation='softmax')(x)
#
# model = Model(input=base_model.input, outputs=predictions)
input = base_model.input
labels = tf.placeholder(tf.float32, shape=(None, 10))
from keras.objectives import categorical_crossentropy
loss = tf.reduce_mean(categorical_crossentropy(labels, predictions))
train_step = tf.train.GradientDescentOptimizer(0.5).minimize(loss)
acc_ops = accuracy(labels, predictions)
loss_ops = mean_loss(labels, predictions)
init_op = tf.global_variables_initializer()
sess.run(init_op)
# jpeg_data_tensor, decoded_image_tensor = add_jpeg_decoding()
with sess.as_default():
# Set up the image decoding sub-graph.
jpeg_data_tensor, decoded_image_tensor = add_jpeg_decoding(
model_info['input_width'], model_info['input_height'],
model_info['input_depth'], model_info['input_mean'],
model_info['input_std'])
# with slim.arg_scope(inception_resnet_v2_arg_scope()):
# logits, end_points = inception_resnet_v2(images, num_classes=dataset.num_classes, is_training=True)
evaluation_step, cross_entropy_value, prediction = add_evaluation_step(
predictions, input)
for i in range(100):
(train_data, train_ground_truth,
_) = get_random_decoded_images(sess, image_lists, 16, 'training',
GENERAL_SETTING['image_dir'],
jpeg_data_tensor,
decoded_image_tensor)
# print (train_data.size, train_ground_truth.size)
# train_data = np.array(train_data)
# print(train_data.shape)
train_step.run(feed_dict={
input: train_data,
labels: train_ground_truth
})
# acc_value = acc_ops.eval(feed_dict={input: train_data, labels: train_ground_truth})
# loss_value = loss_ops.eval(feed_dict={input: train_data, labels: train_ground_truth})
# print (acc_value, loss_value)
train_accuracy, cross_entropy_value = sess.run(
[evaluation_step, cross_entropy_value])
print(train_accuracy, cross_entropy_value)
