def main():
training_images, training_labels, test_images, test_labels = load_dataset()
# plt.imshow(training_images[:,:,0], cmap='gray')
# plt.show()
perm_train = np.random.permutation(training_labels.size)
training_labels = training_labels[perm_train]
training_images = training_images[perm_train, :, :] / 255.0
training_images = np.expand_dims(training_images, -1)
print(training_images.shape)
test_images = test_images / 255.0
test_images = np.expand_dims(test_images, -1)
# pdb.set_trace()
# training_labels = to_categorical(training_labels, NUM_CLASSES)
# test_labels = to_categorical(test_labels, NUM_CLASSES)
BATCH_SIZE = 32 * 8
WIDTH, HEIGHT = 28, 28
# Defiining the network
input_layer = Input(shape=(HEIGHT, WIDTH, 1), name='input_layer')
cnn1 = Conv2D(filters=32,
kernel_size=3,
strides=(1, 1),
padding='same',
activation='relu')(input_layer)
maxpool = MaxPooling2D(pool_size=2)(cnn1)
cnn2 = Conv2D(filters=32,
kernel_size=3,
strides=(1, 1),
padding='valid',
activation='relu')(maxpool)
maxpool = MaxPooling2D(pool_size=2)(cnn2)
flat = Flatten()(maxpool)
dense1 = Dense(units=128, activation='relu')(flat)
dropout = Dropout(.5)(dense1)
output_layer = Dense(units=NUM_CLASSES, activation='softmax')(dropout)
model = Model(inputs=input_layer, outputs=output_layer)
model.compile(optimizer=tf.train.AdamOptimizer(),
loss='sparse_categorical_crossentropy',
metrics=['accuracy'])
# pdb.set_trace()
print(model.summary())
model.fit(x=training_images,
y=training_labels,
batch_size=BATCH_SIZE,
epochs=30,
verbose=1,
validation_data=(test_images, test_labels))
accuracy = model.evaluate(x=test_images,
y=test_labels,
batch_size=BATCH_SIZE)
print('test score = {}'.format(accuracy))
x = Dropout(0.3)(x)
x = Conv2D(64, (8, 1), padding='same', use_bias=False)(inputs)
x = Conv2D(64, (8, 1), padding='same', use_bias=False, activation='relu')(x)
x = BatchNormalization(axis=1)(x)
x = Dropout(0.3)(x)
x = Conv2D(64, (1, 5), padding='valid', use_bias=False, activation='relu')(x)
x = Reshape((128, 64))(x)
x = Bidirectional(LSTM(20, dropout=0.2, return_sequences=False))(x)
#x = Flatten()(x)
x = Dense(256, use_bias=False, activation='relu')(x)
x = BatchNormalization(axis=-1)(x)
x = Dropout(0.5)(x)
logits = Dense(2, use_bias=False)(x)
model = Model(inputs=inputs, outputs=logits)
model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
model.summary()
history = model.fit(train,
train_label,
validation_split=2 / 7,
batch_size=batch_size,
epochs=epochs,
shuffle=True)
Important Tips:
Contemplating the following two tips deeply and patiently !!!!!!!!!!
Compare model.summary() and pp model.trainable_weights, we can see that how Conv1D weights or filters are used to screen embedding_1 tensor
In fact, for all layers, either weights of Conv1D, Dense, or that of Conv2D, consider them to be filters, to screen previous layer's tensor
How embedding weights transform input_1 (?, 1000) to embedding_1 (?, 1000, 100)? deserve research later
"""
model_path = "/Users/Natsume/Downloads/data_for_all/word_embeddings/pretrainedWordEmbedding_2.h5"
if os.path.isfile(model_path):
model = load_model(
"/Users/Natsume/Downloads/data_for_all/word_embeddings/pretrainedWordEmbedding_2.h5"
)
model.fit(x_train, y_train, batch_size=128, epochs=1, validation_split=0.2)
# validation_data=(x_val, y_val))
model.save(
"/Users/Natsume/Downloads/data_for_all/word_embeddings/pretrainedWordEmbedding_3.h5"
)
loss, accuracy = model.evaluate(x_test,
y_test,
batch_size=len(x_test),
verbose=1)
preds = model.predict(x_test)
preds_integer = np.argmax(preds, axis=1)
from tensorflow.contrib.keras.python.keras.layers import Dropout, BatchNormalization, Input, Dense from tensorflow.contrib.keras.python.keras.models import Model import numpy as np from tensorflow.contrib.keras.python.keras import backend as K from tensorflow.contrib.keras.python.keras import losses x = np.random.random((100,10))*2 y = np.random.randint(2, size=(100,1)) input_tensor = Input(shape=(10,)) bn_tensor = BatchNormalization()(input_tensor) dp_tensor = Dropout(0.7)(bn_tensor) final_tensor = Dense(1)(dp_tensor) model = Model(input_tensor, final_tensor) model.compile(optimizer='SGD', loss='mse', metrics=['accuracy']) from tensorflow.contrib.keras.python.keras import callbacks from tensorflow.contrib.keras.python.keras.callbacks import EarlyStopping early_stopping = EarlyStopping(monitor='val_loss', patience=5) # patience=2 hist1=model.fit(x, y, validation_split=0.2, callbacks=[early_stopping], epochs=100) print(hist1.history) hist2=model.fit(x, y, validation_split=0.3, epochs=10) # checkout all the losses and metrics print(hist2.history)
' class_weight: optional dictionary mapping\n' ' class indices (integers) to\n' " a weight (float) to apply to the model's loss for the samples\n" ' from this class during training.\n' ' This can be useful to tell the model to "pay more attention" to\n' ' samples from an under-represented class.\n' '\n' 'Returns:\n' ' Scalar training loss\n' ' (if the model has a single output and no metrics)\n' ' or list of scalars (if the model has multiple outputs\n' ' and/or metrics). The attribute `model.metrics_names` will give you\n' ' the display labels for the scalar outputs.') """ # fit_loop() will help split x, y into batches for training in batches hist = model.fit(x, y, validation_split=0.1, batch_size=2, epochs=5) hist.history # a dict of val_loss and loss hist.history['val_loss'] hist.history['loss'] """ ([' def fit(self,\n', ' x=None,\n', ' y=None,\n', ' batch_size=32,\n', ' epochs=1,\n', ' verbose=1,\n', ' callbacks=None,\n', ' validation_split=0.,\n', ' validation_data=None,\n', ' shuffle=True,\n', ' class_weight=None,\n',
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import confusion_matrix
# In[22]:
# use logistic regression as the model
model = LogisticRegression()
# In[23]:
model.fit(trainData, trainLabels)
# In[24]:
# use rank-1 and rank-5 predictions
f = open('output/flowers_17/xception/results.txt', "w")
rank_1 = 0
rank_5 = 0
# In[25]:
# loop over test data
flatten_layer_output = K.function([model.layers[0].input], [model.layers[2].output]) flatten_out = flatten_layer_output([x])[0] flatten_weights = K.batch_get_value(model.layers[2].weights) ## get dense layer output with real values dense_layer_output = K.function([model.layers[0].input], [model.layers[3].output]) dense_out = dense_layer_output([x])[0] dense_weights = K.batch_get_value(model.layers[3].weights) ### Start to train num_epochs = 2 for epoch in range(num_epochs): # train 50 samples for once model.fit(val_img_array, val_lab_array, batch_size=10, epochs=1, verbose=1) """ def fit(self,\n', ' x=None,\n', ' y=None,\n', ' batch_size=32,\n', ' epochs=1,\n', ' verbose=1,\n', ' callbacks=None,\n', ' validation_split=0.,\n', ' validation_data=None,\n', ' shuffle=True,\n', ' class_weight=None,\n', ' sample_weight=None,\n',
# Training
sequence_input = Input(shape=(MAX_SEQUENCE_LENGTH, ), dtype='int32')
embedded_sequences = embedding_layer(sequence_input)
x = Conv1D(128, 5, activation='relu')(embedded_sequences)
print('sequence_input: ', sequence_input)
print('embedded_sequences: ', embedded_sequences)
print('x: ', x)
x = MaxPooling1D(5)(x)
x = Conv1D(128, 5, activation='relu')(x)
x = MaxPooling1D(5)(x)
x = Conv1D(128, 5, activation='relu')(x)
x = MaxPooling1D(35)(x) # global max pooling
x = Flatten()(x)
x = Dense(128, activation='relu')(x)
preds = Dense(len(labels_index), activation='softmax')(x)
model = Model(sequence_input, preds)
model.compile(loss='categorical_crossentropy',
optimizer='rmsprop',
metrics=['acc'])
# happy learning!
model.fit(x_train,
y_train,
validation_data=(x_val, y_val),
epochs=2,
batch_size=128)
ntrain = 10000
trainidx = random.sample(range(0, X_train.shape[0]), ntrain)
XT = X_train[trainidx, :, :, :]
# Pre-train the discriminator network ...
noise_gen = np.random.uniform(0, 1, size=[XT.shape[0], 100])
generated_images = generator.predict(noise_gen)
X = np.concatenate((XT, generated_images))
n = XT.shape[0]
y = np.zeros([2 * n, 2])
y[:n, 1] = 1
y[n:, 0] = 1
make_trainable(discriminator, True)
discriminator.fit(X, y, epochs=1, batch_size=128)
y_hat = discriminator.predict(X)
# Measure accuracy of pre-trained discriminator network
y_hat_idx = np.argmax(y_hat, axis=1)
y_idx = np.argmax(y, axis=1)
diff = y_idx - y_hat_idx
n_tot = y.shape[0]
n_rig = (diff == 0).sum()
acc = n_rig * 100.0 / n_tot
print("Accuracy: %0.02f pct (%d of %d) right" % (acc, n_rig, n_tot))
# set up loss storage vector
losses = {"d": [], "g": []}
import numpy as np
input_tensor = Input(shape=(32, ))
layer = Dense(5)
layer.trainable = False
tensor_no_train = layer(input_tensor)
frozen_model = Model(input_tensor, tensor_no_train)
# in the model below, the weights of `layer` will not be updated during training
frozen_model.compile(optimizer='rmsprop', loss='mse')
layer.trainable = True
tensor_do_train = layer(input_tensor)
trainable_model = Model(input_tensor, tensor_do_train)
# with this model the weights of the layer will be updated during training
# (which will also affect the above model since it uses the same layer instance)
trainable_model.compile(optimizer='rmsprop', loss='mse')
# create dataset
data = np.random.random((100, 32)) # check on source
labels = np.random.randint(5, size=(100, 5)) # check source
frozen_model.fit(data, labels, epochs=5) # no validation_set, then no val_loss
frozen_model.fit(
data, labels, validation_split=0.3, epochs=5
) # this does NOT update the weights of `layer`, therefore, loss stays the same
trainable_model.fit(
data, labels, validation_split=0.3, epochs=5
) # this updates the weights of `layer`, therefore, loss keeps changing
# Create a model needs an input tensor and an output tensor
model = Model(inputs=inputs, outputs=predictions)
# before training, a model has to have optimizer, loss and metrics
model.compile(optimizer='rmsprop',
loss='categorical_crossentropy',
metrics=['accuracy'])
# Generate dummy data
import numpy as np
data = np.random.random((1000, 784))
labels = np.random.randint(10, size=(1000, 1))
labels = to_categorical(labels, num_classes=10)
model.fit(data, labels, validation_split=0.2) # starts training
"""
Use_2: model as layer
## All models are callable, just like layers
1. you can treat any model as if it were a layer, by calling it on a tensor.
2. calling a model is use both its architecture and latest weights
"""
x = Input(shape=(784, ))
# This works, and returns the 10-way softmax we defined above.
# like doing forward pass
y = model(x)
"""
# This defines a model with two inputs and two outputs:
model = Model(inputs=[main_input, auxiliary_input],
outputs=[main_output, auxiliary_output])
# We compile the model and assign a weight of 0.2 to the auxiliary loss.
# To specify different `loss_weights` or `loss` for each different output, you can use a list or a dictionary.
# Here we pass a single loss as the `loss` argument, so the same loss will be used on all outputs.
model.compile(optimizer='rmsprop',
loss='binary_crossentropy',
loss_weights=[1., 0.2])
# We can train the model by passing it lists of input arrays and target arrays:
model.fit([main_input_array, auxiliary_input_array],
[main_output_array, auxiliary_output_array],
validation_split=0.2,
epochs=2,
batch_size=32)
# Since our inputs and outputs are named (we passed them a "name" argument),
# We could also have compiled the model via:
model.compile(optimizer='rmsprop',
loss={
'main_output': 'binary_crossentropy',
'aux_output': 'binary_crossentropy'
},
loss_weights={
'main_output': 1.,
'aux_output': 0.2
})
