tokenize.texts_to_matrix([text])
x_train = tokenize.texts_to_matrix(train_posts)
x_test = tokenize.texts_to_matrix(test_posts)
print(tokenize.word_index)
encoder = LabelEncoder()
encoder.fit(train_tags)
y_train = encoder.transform(train_tags)
y_test = encoder.transform(test_tags)
np.max(y_train) + 1
num_classes = np.max(y_train) + 1
y_train = utils.to_categorical(y_train, num_classes)
y_test = utils.to_categorical(y_test, num_classes)
print('x_train shape :', x_train.shape)
print('x_test shape :', x_test.shape)
print('y_train shape :', y_train.shape)
print('y_test shape :', y_test.shape)
batch_size = 32
epochs = 5
model = Sequential()
model.add(Dense(512, input_shape=(max_words, )))
model.add(Activation('relu'))
model.add(Dense(num_classes))
model.add(Activation('softmax'))
"""
Load keras dataset
"""
from tensorflow.contrib.keras.python.keras.datasets import mnist
from tensorflow.contrib.keras.python.keras.utils import to_categorical
mnist_path = mnist.get_file(
fname='mnist.npz',
origin=None) # the dataset is already downloaded, no origin is needed
"""
(['def get_file(fname,\n',
' origin,\n',
' untar=False,\n',
' md5_hash=None,\n',
' file_hash=None,\n',
" cache_subdir='datasets',\n",
" hash_algorithm='auto',\n",
' extract=False,\n',
" archive_format='auto',\n",
' cache_dir=None):\n',
"""
(train_img, train_lab), (test_img, test_lab) = mnist.load_data(path=mnist_path)
# Convert labels to categorical one-hot encoding
train_lab_hot = to_categorical(train_lab, num_classes=10)
def main():
training_images, training_labels, test_images, test_labels = load_dataset()
# plt.imshow(training_images[:,:,0], cmap='gray')
# plt.show()
N = training_labels.size
Nt = test_labels.size
perm_train = np.random.permutation(N)
training_labels = training_labels[perm_train]
training_images = training_images[perm_train, :, :] / 255.0
training_images = np.expand_dims(training_images, -1)
print(training_images.shape)
test_images = test_images / 255.0
test_images = np.expand_dims(test_images, -1)
# pdb.set_trace()
training_labels = to_categorical(training_labels, NUM_CLASSES)
test_labels = to_categorical(test_labels, NUM_CLASSES)
BATCH_SIZE = 32*8
WIDTH, HEIGHT = 28, 28
epochs = 5
# Defiining the placeholders
input_data = tf.placeholder(dtype=tf.float32, shape=[None, HEIGHT, WIDTH, 1], name='data')
input_labels = tf.placeholder(dtype=tf.float32, shape=[None, NUM_CLASSES], name='labels')
do_rate = tf.placeholder(dtype=tf.float32, name='dropout_rate')
# pdb.set_trace()
'''
with tf.name_scope('conv1'):
with tf.variable_scope('conv1'):
W_conv1 = tf.get_variable('w', [3,3,1,32])
b_conv1 = tf.get_variable('b', [32])
conv1 = tf.nn.conv2d(input=input_data, filter=W_conv1, strides=[1,1,1,1], padding='SAME')
relu1 = tf.nn.relu(conv1 + b_conv1)
with tf.name_scope('pool1'):
pool1 = tf.nn.max_pool(value=relu1, ksize=[1,2,2,1], strides=[1,2,2,1], padding='SAME')
with tf.name_scope('conv2'):
with tf.variable_scope('conv2'):
W_conv2 = tf.get_variable('w', [3,3,32,32])
b_conv2 = tf.get_variable('b', [32])
conv2 = tf.nn.conv2d(input=pool1, filter=W_conv2, strides=[1,1,1,1], padding='VALID')
relu2 = tf.nn.relu(conv2 + b_conv2)
with tf.name_scope('pool2'):
pool2 = tf.nn.max_pool(value=relu2, ksize=[1,2,2,1], strides=[1,2,2,1], padding='SAME')
with tf.name_scope('dense1'):
with tf.variable_scope('dense1'):
W_dense1 = tf.get_variable('w', [6*6*32,128])
b_dense1 = tf.get_variable('b', 128)
flat = tf.reshape(pool2, [-1,6*6*32], name='reshape')
dense1= tf.matmul(flat, W_dense1)
relu3 = tf.nn.relu(dense1 + b_dense1)
with tf.name_scope('dropout'):
dropout = tf.nn.dropout(relu3, do_rate)
with tf.name_scope('output'):
with tf.variable_scope('output'):
W_out = tf.get_variable('w', [128,NUM_CLASSES])
b_out = tf.get_variable('b', [NUM_CLASSES])
output = tf.matmul(dropout, W_out) + b_out
'''
print('-------------------------------------------------------')
""" Using Keras layers instead """
#input_layer = Input(shape=(HEIGHT, WIDTH, 1), name='input_layer')
Kcnn1 = Conv2D(filters=32, kernel_size=3, strides=(1,1), padding='same', activation='relu')(input_data)
Kmaxpool = MaxPooling2D(pool_size=2)(Kcnn1)
"""
with tf.name_scope('conv2'):
with tf.variable_scope('conv2'):
W_conv2 = tf.get_variable('w', [3,3,32,32])
b_conv2 = tf.get_variable('b', [32])
conv2 = tf.nn.conv2d(input=Kmaxpool, filter=W_conv2, strides=[1,1,1,1], padding='VALID')
Kcnn2 = tf.nn.relu(conv2 + b_conv2)
"""
Kcnn2 = Conv2D(filters=32, kernel_size=3, strides=(1,1), padding='valid', activation='relu')(Kmaxpool)
Kmaxpool = MaxPooling2D(pool_size=2)(Kcnn2)
Kflat = Flatten()(Kmaxpool)
Kdense1 = Dense(units=128, activation='relu')(Kflat)
Kdropout = Dropout(.5)(Kdense1)
output = Dense(units=NUM_CLASSES, activation='linear')(Kdropout)
""" The rest of the code is almost the same as in pure_tf_mnist.py,
except for the feed_dict, where instead of do_rate in tensorflow,
we need to provide keras specific dropout tensor 'learning_phase'
in the backend of Keras. """
print('-------------------------------------------------------')
print('\n\n')
print('-------------------------------------------------------')
print('--------------- Trainable parameters ------------------')
print('-------------------------------------------------------')
total_parameters = 0
for v in tf.trainable_variables():
shape = v.get_shape()
print(shape)
#pdb.set_trace()
params = 1
for dim in shape:
params *= dim.value
total_parameters += params
print('total_parameters = {}'.format(total_parameters))
print('-------------------------------------------------------\n\n')
loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=input_labels, logits=output, name='loss'))
train_op = tf.train.AdamOptimizer(1e-4).minimize(loss)
accuracy = tf.cast(tf.equal(tf.argmax(input_labels, 1), tf.argmax(output, 1)), tf.float32)
# Training:
sess = tf.Session()
sess.run(tf.global_variables_initializer())
writer = tf.summary.FileWriter('graph', sess.graph)
for i in range(epochs):
steps = (int)(np.ceil(float(N)/float(BATCH_SIZE)))
total_l = 0
total_acc = 0
for step in range(steps):
x_in, y_in = get_batch(step, BATCH_SIZE, training_images, training_labels)
l, acc, _ = sess.run([loss, accuracy, train_op], {input_data:x_in, input_labels:y_in, learning_phase():1})#do_rate:0.5})
total_l += l
total_acc += np.sum(acc)
#pdb.set_trace()
total_acc /= np.float32(N)
print("Epoch {}: Training loss = {}, Training accuracy = {}".format(i,total_l,total_acc))
# Test:
total_acc = 0
steps = (int)(np.ceil(float(Nt)/float(BATCH_SIZE)))
for step in range(steps):
x_in, y_in = get_batch(step, BATCH_SIZE, test_images, test_labels)
acc = sess.run([accuracy], {input_data:x_in, input_labels:y_in, learning_phase():0})#do_rate:1})
total_acc += np.sum(acc)
total_acc /= np.float32(Nt)
print('\n--------------------------\n')
print("Test accuracy = {}".format(total_acc))
sess.close()
writer.close()
def validate(self):
self.model.eval()
n_class = len(self.val_loader.dataset.class_names)
val_loss = 0
visualizations = []
label_trues, label_preds = [], []
for batch_idx, (data_1, target_1, name_1, target_1_s) in tqdm.tqdm(
enumerate(self.val_loader),
total=len(self.val_loader),
desc='Valid iteration=%d' % self.iteration,
ncols=80,
leave=False):
if self.cuda:
data_1, target_1, target_1_s = data_1.cuda(), target_1.cuda(
), target_1_s.cuda()
data_1, target_1, target_1_s = Variable(data_1), Variable(
target_1), Variable(target_1_s)
img1 = data_1.cpu().data.numpy()[0].transpose(1, 2, 0)
lbl1 = target_1_s.cpu().data.numpy()[0]
feats_1_sim = self.features[name_1[0]]
feats_1 = self.model(data_1)
n, h, w = target_1_s.size()
pseudo_label = np.zeros((h * w, n_class))
for batch_idx_2, (data_2, target_2, name_2,
target_2_s) in enumerate(self.train_loader):
feats_2_sim = self.features[name_2[0]]
dissimilarity = correlation(feats_1_sim, feats_2_sim)
if (np.squeeze(dissimilarity) > 0.315):
continue
if self.cuda:
data_2, target_2, target_2_s = data_2.cuda(
), target_2.cuda(), target_2_s.cuda()
data_2, target_2, target_2_s = Variable(data_2), Variable(
target_2), Variable(target_2_s)
feats_2 = self.model(data_2)
score = self.merge(feats_1, feats_2)
score = nn.Sigmoid()(score)
score = score.squeeze(0).squeeze(0)
score[score >= 0.5] = 1
score[score < 0.5] = 0
target2s_rep = target_2_s.view(1, h * w).repeat(h * w, 1)
score[target2s_rep < 0] = 0
#img2 = data_2.cpu().data.numpy()[0].transpose(1,2,0)
lbl2 = target_2_s.cpu().data.numpy()[0]
lbl2oh = to_categorical(lbl2, n_class)
transfered = np.dot(score.cpu().data.numpy(), lbl2oh)
pseudo_label += transfered
pseudo_label_max = np.max(pseudo_label, axis=-1)
pseudo_label = np.argmax(pseudo_label, axis=-1)
pseudo_label[pseudo_label_max == 0] = -1
pseudo_label = np.reshape(pseudo_label, (h, w))
plt.subplot(121)
plt.imshow(lbl1)
plt.title('Ground Truth')
plt.subplot(122)
plt.imshow(pseudo_label)
plt.title('Label Transfer Result')
plt.show()
pdb.set_trace()
val_loss /= len(self.val_loader)
pdb.set_trace()
print('\nval_loss = {}\n'.format(val_loss))
def one_hot_encode_object_array(arr):
'''One hot encode a numpy array of objects (e.g. strings)'''
uniques, ids = np.unique(
arr, return_inverse=True) # convert 3 words into 0, 1, 2
return to_categorical(ids, len(uniques)) # convert 0, 1, 2 to one-hot
from tensorflow.contrib.keras.python.keras.preprocessing.text import Tokenizer
from tensorflow.contrib.keras.python.keras.utils import to_categorical
samples = ['The cat sat on the mat.', 'The dog ate my homework.']
# We create a tokenizer, configured to only take
# into account the top-1000 most common on words
tokenizer = Tokenizer(num_words=10) # 1000 或者 10
# The builds the word index
tokenizer.fit_on_texts(samples)
# This turns strings into lists of integer indices.
sequences = tokenizer.texts_to_sequences(samples)
# 每个样本的单词数量没有超过10个
sequence_matrix_1 = to_categorical(sequences[0], 10)
sequence_matrix_2 = to_categorical(sequences[1], 10)
# You could also directly get the one-hot binary representations.
# Note that other vectorization modes than one-hot encoding are supported!
one_hot_results = tokenizer.texts_to_matrix(samples, mode='binary')
# This is how you can recover the word index that was computed
word_index = tokenizer.word_index
print('Found %s unique tokens.' % len(word_index))
# 下面两个tensor表达的是同一个样本数据
sequence_matrix_1
one_hot_results[0]
def main():
training_images, training_labels, test_images, test_labels = load_dataset()
# plt.imshow(training_images[:,:,0], cmap='gray')
# plt.show()
N = training_labels.size
Nt = test_labels.size
perm_train = np.random.permutation(N)
training_labels = training_labels[perm_train]
training_images = training_images[perm_train, :, :] / 255.0
training_images = np.expand_dims(training_images, -1)
print(training_images.shape)
test_images = test_images / 255.0
test_images = np.expand_dims(test_images, -1)
# pdb.set_trace()
training_labels = to_categorical(training_labels, NUM_CLASSES)
test_labels = to_categorical(test_labels, NUM_CLASSES)
BATCH_SIZE = 32 * 8
WIDTH, HEIGHT = 28, 28
epochs = 9
# Defiining the placeholders
input_data = tf.placeholder(dtype=tf.float32,
shape=[None, HEIGHT, WIDTH, 1],
name='data')
input_labels = tf.placeholder(dtype=tf.float32,
shape=[None, NUM_CLASSES],
name='labels')
do_rate = tf.placeholder(dtype=tf.float32, name='dropout_rate')
# pdb.set_trace()
with tf.name_scope('conv1'):
with tf.variable_scope('conv1'):
W_conv1 = tf.get_variable('w', [3, 3, 1, 32])
b_conv1 = tf.get_variable('b', [32])
conv1 = tf.nn.conv2d(input=input_data,
filter=W_conv1,
strides=[1, 1, 1, 1],
padding='SAME')
relu1 = tf.nn.relu(conv1 + b_conv1)
with tf.name_scope('pool1'):
pool1 = tf.nn.max_pool(value=relu1,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding='SAME')
with tf.name_scope('conv2'):
with tf.variable_scope('conv2'):
W_conv2 = tf.get_variable('w', [3, 3, 32, 32])
b_conv2 = tf.get_variable('b', [32])
conv2 = tf.nn.conv2d(input=pool1,
filter=W_conv2,
strides=[1, 1, 1, 1],
padding='VALID')
relu2 = tf.nn.relu(conv2 + b_conv2)
with tf.name_scope('pool2'):
pool2 = tf.nn.max_pool(value=relu2,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding='SAME')
with tf.name_scope('dense1'):
with tf.variable_scope('dense1'):
W_dense1 = tf.get_variable('w', [6 * 6 * 32, 128])
b_dense1 = tf.get_variable('b', 128)
flat = tf.reshape(pool2, [-1, 6 * 6 * 32], name='reshape')
dense1 = tf.matmul(flat, W_dense1)
relu3 = tf.nn.relu(dense1 + b_dense1)
with tf.name_scope('dropout'):
dropout = tf.nn.dropout(relu3, do_rate)
with tf.name_scope('output'):
with tf.variable_scope('output'):
W_out = tf.get_variable('w', [128, NUM_CLASSES])
b_out = tf.get_variable('b', [NUM_CLASSES])
output = tf.matmul(dropout, W_out) + b_out
print('\n\n-------------------------------------------------------')
print('--------------- Trainable parameters ------------------')
print('-------------------------------------------------------')
total_parameters = 0
for v in tf.trainable_variables():
shape = v.get_shape()
print(shape)
params = 1
for dim in shape:
params *= dim.value
total_parameters += params
print('total_parameters = {}'.format(total_parameters))
print('-------------------------------------------------------\n\n')
loss = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(labels=input_labels,
logits=output,
name='loss'))
train_op = tf.train.AdamOptimizer(1e-4).minimize(loss)
accuracy = tf.cast(
tf.equal(tf.argmax(input_labels, 1), tf.argmax(output, 1)), tf.float32)
# Training:
sess = tf.Session()
sess.run(tf.global_variables_initializer())
writer = tf.summary.FileWriter('graph', sess.graph)
for i in range(epochs):
steps = (int)(np.ceil(float(N) / float(BATCH_SIZE)))
total_l = 0
total_acc = 0
for step in range(steps):
x_in, y_in = get_batch(step, BATCH_SIZE, training_images,
training_labels)
l, acc, _ = sess.run([loss, accuracy, train_op], {
input_data: x_in,
input_labels: y_in,
do_rate: 0.5
})
total_l += l
total_acc += np.sum(acc)
#pdb.set_trace()
total_acc /= np.float32(N)
print("Epoch {}: Training loss = {}, Training accuracy = {}".format(
i, total_l, total_acc))
# Test:
steps = (int)(np.ceil(float(Nt) / float(BATCH_SIZE)))
for step in range(steps):
x_in, y_in = get_batch(step, BATCH_SIZE, test_images, test_labels)
acc = sess.run([accuracy], {
input_data: x_in,
input_labels: y_in,
do_rate: 1
})
total_acc += np.sum(acc)
total_acc /= np.float32(Nt)
print('\n--------------------------\n')
print("Test accuracy = {}".format(total_acc))
sess.close()
writer.close()
...and more. """ """ ### Multilayer Perceptron (MLP) for multi-class softmax classification: """ from tensorflow.contrib.keras.python.keras.models import Sequential from tensorflow.contrib.keras.python.keras.layers import Dense, Dropout, Activation from tensorflow.contrib.keras.python.keras.optimizers import SGD from tensorflow.contrib.keras.python.keras.utils import to_categorical # Generate dummy data import numpy as np x_train = np.random.random((1000, 20)) y_train = to_categorical(np.random.randint(10, size=(1000, 1)), num_classes=10) x_test = np.random.random((100, 20)) y_test = to_categorical(np.random.randint(10, size=(100, 1)), num_classes=10) model = Sequential() # Dense(64) is a fully-connected layer with 64 hidden units. # in the first layer, you must specify the expected input data shape: # here, 20-dimensional vectors. model.add(Dense(64, activation='relu', input_dim=20)) model.add(Dropout(0.5)) model.add(Dense(64, activation='relu')) model.add(Dropout(0.5)) model.add(Dense(10, activation='softmax')) # specify optimizer sgd = SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True) # param adjust
tokenizer.fit_on_texts(
texts
) # texts: a list of strings, or generator of strings; Updates internal vocabulary based on a list of texts; # tokenizer.document_count: num of samples processed so far; # tokenizer.text_to_word_sequence: convert a long string to a list of words; # tokenizer.word_counts: dictionary, {word: counts} added up in each and every sample; # tokenizer.word_docs: dictionary {unique_word: counts}, each sample count unique word only once, add up if appear in a different sample; # self.word_counts.__len__(): total unique words in all samples; # self.word_docs.get("the"): how many documents or samples have "the"; # self.word_counts.get("the"): how many times "the" has occured in all samples; # wcounts = list(self.word_counts.items()): wcounts is a list of tuples (word, counts); # wcounts.sort(key=lambda x: x[1], reverse=True): sort the list from highest to smallest counts; # sorted_voc = [wc[0] for wc in wcounts]: get a list of words sorted by counts from highest to lowest
sequences = tokenizer.texts_to_sequences(
texts
) # Transforms each text in texts in a sequence of integers (each integer refers to a word); # Only top "num_words" most frequent words (top 20000 most frequent words out of 174047 unique words will be taken into account. Only words known by the tokenizer will be taken into account. # sequences: a list of 19997 sublist, each list has less 20000 unique but most frequent words `for sq in sequences: np.array(sq).max()`
word_index = tokenizer.word_index # total num of unique words in all samples; also total vocabularies based on all samples here
print('Found %s unique tokens.' % len(word_index))
data = pad_sequences(
sequences, maxlen=MAX_SEQUENCE_LENGTH
) # Pads each sequence to the same length (length of the longest sequence), If maxlen is provided, any sequence longer than maxlen is truncated to maxlen. Truncation happens off either the beginning (default) or the end of the sequence. Supports post-padding and pre-padding (default). # previously, maximum length of each sequence is 20000, now maxlen is set to 1000, then we can check data's sublist length won't be longer than 1000; # data.shape == (19997, 1000), for each sample text, there are 1000 most frequent words to summarize it
labels = to_categorical(
np.asarray(labels)
) # each sample text has its category, from 0 to 19; # to_categorical convert 0-9 to one-hot encoding
print('Shape of data tensor:', data.shape)
print('Shape of label tensor:', labels.shape)
# split the data into a training set and a validation set
indices = np.arange(data.shape[0]) # indices of all samples
np.random.shuffle(indices) # shuffle the indices
data = data[indices] # shuffle data samples
labels = labels[indices] # shuffle labels (one hot encoded)
num_test_samples = int(TEST_SPLIT * data.shape[0])
x_train = data[:-num_test_samples] # split train and test sets on features
y_train = labels[:-num_test_samples] # split train, test on labels
x_test = data[-num_test_samples:]
y_test = labels[-num_test_samples:]
print(max([len(t) for t in texts]))
print(min([len(t) for t in texts]))
tokenizer = Tokenizer(num_words=MAX_NB_WORDS)
tokenizer.fit_on_texts(texts)
sequences = tokenizer.texts_to_sequences(texts)
print(max([len(t) for t in sequences]))
print(min([len(t) for t in sequences]))
word_index = tokenizer.word_index
print('Found %s unique tokens.' % len(word_index))
data = pad_sequences(sequences)
labels = to_categorical(np.asarray(labels))
print('Shape of data tensor:', data.shape)
MAX_SEQUENCE_LENGTH = data.shape[1]
print('Shape of label tensor:', labels.shape)
# split the data into a training set and a validation set
indices = np.arange(data.shape[0])
np.random.shuffle(indices)
data = data[indices]
labels = labels[indices]
nb_validation_samples = int(VALIDATION_SPLIT * data.shape[0])
embeddings_index = {}
f = open(os.path.join(GLOVE_DIR, 'glove.6B.300d.txt'), encoding="utf-8")
for line in f:
values = line.split()
for it in range(100000):
if it % 1000 == 0:
n_sample = 16
Z_sample = sample_Z(n_sample, Z_dim)
y_sample = np.zeros(shape=[n_sample, y_dim])
y_sample[range(n_sample), np.random.randint(0, 10, n_sample)] = 1
samples = sess.run(G_sample, feed_dict={Z: Z_sample, y: y_sample})
fig = plot(samples)
plt.savefig('out/{}.png'.format(str(i).zfill(3)), bbox_inches='tight')
i += 1
plt.close(fig)
X_mb, y_mb = next(gen)
y_mb = to_categorical(y_mb, num_classes=y_dim) * 0.9
Z_sample = sample_Z(X_mb.shape[0], Z_dim)
_, D_loss_curr = sess.run([D_solver, D_loss],
feed_dict={
X: X_mb,
Z: Z_sample,
y: y_mb
})
_, G_loss_curr = sess.run([G_solver, G_loss],
feed_dict={
Z: Z_sample,
y: y_mb
})
if it % 1000 == 0:
def main():
training_images, training_labels, test_images, test_labels = load_dataset()
# plt.imshow(training_images[:,:,0], cmap='gray')
# plt.show()
N = training_labels.size
Nt = test_labels.size
perm_train = np.random.permutation(N)
training_labels = training_labels[perm_train]
training_images = training_images[perm_train, :, :] / 255.0
training_images = np.expand_dims(training_images, -1)
print(training_images.shape)
test_images = test_images / 255.0
test_images = np.expand_dims(test_images, -1)
# pdb.set_trace()
training_labels = to_categorical(training_labels, NUM_CLASSES)
test_labels = to_categorical(test_labels, NUM_CLASSES)
BATCH_SIZE = 32 * 8
WIDTH, HEIGHT = 28, 28
epochs = 30
# Defiining the placeholders
input_data = tf.placeholder(dtype=tf.float32,
shape=[None, HEIGHT, WIDTH, 1],
name='data')
input_labels = tf.placeholder(dtype=tf.float32,
shape=[None, NUM_CLASSES],
name='labels')
do_rate = tf.placeholder(dtype=tf.float32, name='dropout_rate')
# pdb.set_trace()
with tf.name_scope('conv1'):
with tf.variable_scope('conv1'):
W_conv1 = tf.get_variable('w', [3, 3, 1, 32])
b_conv1 = tf.get_variable('b', [32])
conv1 = tf.nn.conv2d(input=input_data,
filter=W_conv1,
strides=[1, 1, 1, 1],
padding='SAME')
relu1 = tf.nn.relu(conv1 + b_conv1)
with tf.name_scope('pool1'):
pool1 = tf.nn.max_pool(value=relu1,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding='SAME')
with tf.name_scope('conv2'):
with tf.variable_scope('conv2'):
W_conv2 = tf.get_variable('w', [3, 3, 32, 32])
b_conv2 = tf.get_variable('b', [32])
conv2 = tf.nn.conv2d(input=pool1,
filter=W_conv2,
strides=[1, 1, 1, 1],
padding='VALID')
relu2 = tf.nn.relu(conv2 + b_conv2)
with tf.name_scope('pool2'):
pool2 = tf.nn.max_pool(value=relu2,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding='SAME')
with tf.name_scope('dense1'):
with tf.variable_scope('dense1'):
W_dense1 = tf.get_variable('w', [6 * 6 * 32, 128])
b_dense1 = tf.get_variable('b', 128)
flat = tf.reshape(pool2, [-1, 6 * 6 * 32], name='reshape')
dense1 = tf.matmul(flat, W_dense1)
relu3 = tf.nn.relu(dense1 + b_dense1)
with tf.name_scope('dropout'):
dropout = tf.nn.dropout(relu3, do_rate)
with tf.name_scope('output'):
with tf.variable_scope('output'):
W_out = tf.get_variable('w', [128, NUM_CLASSES])
b_out = tf.get_variable('b', [NUM_CLASSES])
output = tf.matmul(dropout, W_out) + b_out
'''
################################################################
""" Using Keras layers instead """
#input_layer = Input(shape=(HEIGHT, WIDTH, 1), name='input_layer')
Kcnn1 = Conv2D(filters=32, kernel_size=3, strides=(1,1), padding='same', activation='relu')(input_data)
Kmaxpool1 = MaxPooling2D(pool_size=2)(Kcnn1)
Kcnn2 = Conv2D(filters=32, kernel_size=3, strides=(1,1), padding='valid', activation='relu')(Kmaxpool1)
Kmaxpool2 = MaxPooling2D(pool_size=2)(Kcnn2)
Kflat = Flatten()(Kmaxpool2)
Kdense1 = Dense(units=128, activation='relu')(Kflat)
Kdropout = Dropout(.5)(Kdense1)
output = Dense(units=NUM_CLASSES, activation='softmax')(Kdropout)
""" The rest of the code is almost the same as in pure_tf_mnist.py,
except for the feed_dict, where instead of do_rate in tensorflow,
we need to provide keras specific dropout tensor 'learning_phase'
in the backend of Keras. """
################################################################
'''
print('\n\n')
print('-------------------------------------------------------')
print('--------------- Trainable parameters ------------------')
print('-------------------------------------------------------')
total_parameters = 0
for v in tf.trainable_variables():
shape = v.get_shape()
print(shape)
#pdb.set_trace()
params = 1
for dim in shape:
params *= dim.value
total_parameters += params
print('total_parameters = {}'.format(total_parameters))
print('-------------------------------------------------------\n\n')
loss = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(labels=input_labels,
logits=output,
name='loss'))
train_op = tf.train.AdamOptimizer(1e-4).minimize(loss)
accuracy = tf.cast(
tf.equal(tf.argmax(input_labels, 1), tf.argmax(output, 1)), tf.float32)
print('')
print('-------------------------------------------------------')
print('---------- Starting a TF session ----------------------')
print('-------------------------------------------------------')
print('')
tf_weights = []
tf.set_random_seed(1234)
# Training:
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
writer = tf.summary.FileWriter('graph', sess.graph)
print('-------------------------------------------------------')
print('--------------- Training phase ------------------------')
print('-------------------------------------------------------')
for i in range(epochs):
steps = (int)(np.ceil(float(N) / float(BATCH_SIZE)))
total_l = 0
total_acc = 0
for step in range(steps):
x_in, y_in = get_batch(step, BATCH_SIZE, training_images,
training_labels)
l, acc, _ = sess.run([loss, accuracy, train_op], {
input_data: x_in,
input_labels: y_in,
do_rate: 0.5
})
total_l += l
total_acc += np.sum(acc)
#pdb.set_trace()
total_acc /= np.float32(N)
print(
"Epoch {}: Training loss = {}, Training accuracy = {}".format(
i, total_l, total_acc))
# Test:
total_acc = 0
steps = (int)(np.ceil(float(Nt) / float(BATCH_SIZE)))
for step in range(steps):
x_in, y_in = get_batch(step, BATCH_SIZE, test_images, test_labels)
acc = sess.run([accuracy], {
input_data: x_in,
input_labels: y_in,
do_rate: 1
})
total_acc += np.sum(acc)
total_acc /= np.float32(Nt)
print('\n-----------------------')
print("Test accuracy = {}".format(total_acc))
print('-------------------------------------------------------')
#################################################################
### Exporting the trained weights into a list of numpy vectors
for v in tf.trainable_variables():
tf_weights.append(sess.run(v))
writer.close()
print('')
print('-------------------------------------------------------')
print('---------- Starting a Keras session -------------------')
print('-------------------------------------------------------')
print('')
#################################################################
""" Building a Keras Model """
input_layer = Input(shape=(HEIGHT, WIDTH, 1), name='input_layer')
Kkcnn1 = Conv2D(filters=32,
kernel_size=3,
strides=(1, 1),
padding='same',
activation='relu')(input_layer)
Kkmaxpool1 = MaxPooling2D(pool_size=2)(Kkcnn1)
Kkcnn2 = Conv2D(filters=32,
kernel_size=3,
strides=(1, 1),
padding='valid',
activation='relu')(Kkmaxpool1)
Kkmaxpool2 = MaxPooling2D(pool_size=2)(Kkcnn2)
Kkflat = Flatten()(Kkmaxpool2)
Kkdense1 = Dense(units=128, activation='relu')(Kkflat)
Kkdropout = Dropout(.5)(Kkdense1)
output_layer = Dense(units=NUM_CLASSES, activation='softmax')(Kkdropout)
model = Model(inputs=input_layer, outputs=output_layer)
model.compile(optimizer=tf.train.AdamOptimizer(),
loss='categorical_crossentropy',
metrics=['accuracy'])
#################################################################
#################################################################
### Loarding the already trained weights, onto the keras layers
c = 0 # counter for iterating over tensorflow trainable variables
#pdb.set_trace()
for l in model.layers:
trainable_weights = l.trainable_weights
if not trainable_weights:
# empty trainable weight list in this keras layer; so move on to the next layer.
continue
len_w = len(
trainable_weights
) # e.g. for a normal conv layer, it is two: weight and bias.
l.set_weights(tf_weights[c:c + len_w])
c += len_w
accuracy = model.evaluate(x=test_images,
y=test_labels,
batch_size=BATCH_SIZE)
print('\n')
print('Keras test score = {}'.format(accuracy))
print('\n')
letters = []
pixel_vals = []
print("Starting to read...")
with open('DATA/letter.data', 'r') as f:
reader = csv.reader(f, delimiter='\t')
for row in reader:
letters.append(row[1])
pixel_vals.append(row[6:-1])
print("Applying label encoding...")
labels = np.asarray(letters)
le = LabelEncoder()
labels_encoded = le.fit_transform(labels)
labels_onehot = to_categorical(labels_encoded, num_classes=26)
with open(r"lencoder.pkl", "wb") as output_file:
cPickle.dump(le, output_file)
print("Creating collection...")
collection = np.zeros((1, 20, 20, 1))
for i in tqdm(range(0, len(pixel_vals))):
mat_img = np.asmatrix(pixel_vals[i]).reshape(16, 8).astype(float)
mat_img_tot = np.zeros((20, 20))
mat_img_tot[2:18, 6:14] = mat_img
collection = np.vstack((collection, mat_img_tot.reshape(1, 20, 20, 1)))
collection = collection[1:]
print("Saving collection...")
model = Sequential()
model.add(LSTM(32, return_sequences=True,
input_shape=(timesteps,
data_dim))) # input (?, 8, 16), output (?, ?, 32)
model.add(LSTM(32, return_sequences=True)) # output (?, ?, 32)
model.add(LSTM(32)) # output (?, 32)
model.add(Dense(10, activation='softmax')) # (?, 10)
model.compile(loss='categorical_crossentropy',
optimizer='rmsprop',
metrics=['accuracy'])
# Generate dummy training data
x_train = np.random.random((1000, timesteps, data_dim))
y_train = np.random.randint(10, size=(1000, 1))
y_train = to_categorical(y_train, num_classes=10)
# Generate dummy validation data
x_val = np.random.random((100, timesteps, data_dim))
y_val = np.random.randint(10, size=(100, 1))
y_val = to_categorical(y_val, num_classes=10)
model.fit(x_train,
y_train,
batch_size=64,
epochs=1,
validation_data=(x_val, y_val))
"""
LSTM
' def __init__(self,\n',
x = Dense(64, activation='relu')(x)
predictions = Dense(10, activation='softmax')(x)
# Create a model needs an input tensor and an output tensor
model = Model(inputs=inputs, outputs=predictions)
# before training, a model has to have optimizer, loss and metrics
model.compile(optimizer='rmsprop',
loss='categorical_crossentropy',
metrics=['accuracy'])
# Generate dummy data
import numpy as np
data = np.random.random((1000, 784))
labels = np.random.randint(10, size=(1000, 1))
labels = to_categorical(labels, num_classes=10)
model.fit(data, labels, validation_split=0.2) # starts training
"""
Use_2: model as layer
## All models are callable, just like layers
1. you can treat any model as if it were a layer, by calling it on a tensor.
2. calling a model is use both its architecture and latest weights
"""
x = Input(shape=(784, ))
# This works, and returns the 10-way softmax we defined above.
# like doing forward pass
model.add(Dense(32, activation='relu', input_dim=100))
model.add(Dense(1, activation='sigmoid'))
model.compile(optimizer='rmsprop',
loss='binary_crossentropy',
metrics=['accuracy'])
# Train the model, iterating on the data in batches of 32 samples
print("\nNo validation_set split")
model.fit(data, labels1, epochs=1, batch_size=32)
#######################################
# For a single-input model with 10 classes (categorical classification):
# Generate dummy data
import numpy as np
data = np.random.random((1000, 100))
labels2 = np.random.randint(10, size=(1000, 1))
from tensorflow.contrib.keras.python.keras.utils import to_categorical
# Convert labels to categorical one-hot encoding
one_hot_labels = to_categorical(labels2, num_classes=10)
model = Sequential()
model.add(Dense(32, activation='relu', input_dim=100))
model.add(Dense(10, activation='softmax'))
model.compile(optimizer='rmsprop',
loss='categorical_crossentropy',
metrics=['accuracy'])
# Train the model, iterating on the data in batches of 32 samples
print("\nintro validation_split")
model.fit(data, one_hot_labels, validation_split=0.2, epochs=1, batch_size=32)
