def test_scalar_weighted(self):
cce_obj = losses.CategoricalCrossentropy()
y_true = K.constant([[1, 0, 0], [0, 1, 0], [0, 0, 1]])
y_pred = K.constant([[.9, .05, .05], [.5, .89, .6], [.05, .01, .94]])
loss = cce_obj(y_true, y_pred, sample_weight=2.3)
assert np.isclose(K.eval(loss), .7449, atol=1e-3)
# Test with logits.
logits = K.constant([[8., 1., 1.], [0., 9., 1.], [2., 3., 5.]])
cce_obj = losses.CategoricalCrossentropy(from_logits=True)
loss = cce_obj(y_true, logits, sample_weight=2.3)
assert np.isclose(K.eval(loss), .132, atol=1e-3)
def test_all_correct_unweighted(self):
y_true = K.constant([[1, 0, 0], [0, 1, 0], [0, 0, 1]])
y_pred = K.constant([[1., 0., 0.], [0., 1., 0.], [0., 0., 1.]])
cce_obj = losses.CategoricalCrossentropy()
loss = cce_obj(y_true, y_pred)
assert np.isclose(K.eval(loss), 0.0, atol=1e-3)
# Test with logits.
logits = K.constant([[10., 0., 0.], [0., 10., 0.], [0., 0., 10.]])
cce_obj = losses.CategoricalCrossentropy(from_logits=True)
loss = cce_obj(y_true, logits)
assert np.isclose(K.eval(loss), 0.0, atol=1e-3)
def test_sample_weighted(self):
cce_obj = losses.CategoricalCrossentropy()
y_true = K.constant([[1, 0, 0], [0, 1, 0], [0, 0, 1]])
y_pred = K.constant([[.9, .05, .05], [.5, .89, .6], [.05, .01, .94]])
sample_weight = K.constant([[1.2], [3.4], [5.6]], shape=(3, 1))
loss = cce_obj(y_true, y_pred, sample_weight=sample_weight)
assert np.isclose(K.eval(loss), 1.0696, atol=1e-3)
# Test with logits.
logits = K.constant([[8., 1., 1.], [0., 9., 1.], [2., 3., 5.]])
cce_obj = losses.CategoricalCrossentropy(from_logits=True)
loss = cce_obj(y_true, logits, sample_weight=sample_weight)
assert np.isclose(K.eval(loss), 0.31829, atol=1e-3)
def train_mnist_model_batch_sharded(model, optimizer, mesh, num_epochs,
steps_per_epoch, global_batch_size):
dataset, _ = get_mnist_datasets(NUM_CLASS, global_batch_size)
input_image_layout = dtensor.Layout.batch_sharded(mesh, "batch", rank=4)
input_label_layout = dtensor.Layout.batch_sharded(mesh, "batch", rank=2)
loss_obj = losses.CategoricalCrossentropy()
num_local_devices = mesh.num_local_devices()
iterator = iter(dataset)
train_losses = []
for epoch in range(num_epochs):
total_loss = 0.00
for _ in range(steps_per_epoch):
images, labels = next(iterator)
images = tf.split(images, num_local_devices)
labels = tf.split(labels, num_local_devices)
d_images = dtensor.pack(images, input_image_layout)
d_labels = dtensor.pack(labels, input_label_layout)
total_loss += train_step(model, d_images, d_labels, loss_obj,
optimizer)
train_loss = tf.reduce_mean(total_loss / steps_per_epoch)
logging.info("Epoch %d, Loss: %f", epoch, train_loss)
train_losses.append(train_loss)
return train_losses
def cnn_lstm_optimization(x_train, y_train, x_test, y_test, params):
"""Randomized search to optimize parameters of Neural Network."""
optimization_model = models.Sequential()
optimization_model.add(
layers.Conv1D(params['convolutional'],
params['kernel'],
activation=params['conv_activation']))
optimization_model.add(layers.MaxPooling1D(pool_size=1))
optimization_model.add(layers.LSTM(params['units'], return_sequences=True))
optimization_model.add(layers.LSTM(params['units'],
return_sequences=False))
optimization_model.add(layers.Dropout(0.5))
optimization_model.add(layers.Dense(params['dense'], activation=relu))
optimization_model.add(layers.Dense(params['dense'], activation=relu))
optimization_model.add(layers.Dense(int(num_classes),
activation='softmax'))
optimization_model.compile(
optimizer=params['optimizer'],
loss=losses.CategoricalCrossentropy(),
metrics=['accuracy', talos.utils.metrics.f1score])
history = optimization_model.fit(x_train,
y_train,
batch_size=None,
epochs=params['epoch'],
validation_data=(x_test, y_test))
return history, optimization_model
def test_no_reduction(self):
y_true = K.constant([[1, 0, 0], [0, 1, 0], [0, 0, 1]])
logits = K.constant([[8., 1., 1.], [0., 9., 1.], [2., 3., 5.]])
cce_obj = losses.CategoricalCrossentropy(
from_logits=True, reduction=Reduction.NONE)
loss = cce_obj(y_true, logits)
assert np.allclose(K.eval(loss), (0.001822, 0.000459, 0.169846), atol=1e-3)
def test_label_smoothing(self):
logits = K.constant([[100.0, -100.0, -100.0]])
y_true = K.constant([[1, 0, 0]])
label_smoothing = 0.1
# Softmax Cross Entropy Loss: -\sum_i p_i \log q_i
# where for a softmax activation
# \log q_i = x_i - \log \sum_j \exp x_j
# = x_i - x_max - \log \sum_j \exp (x_j - x_max)
# For our activations, [100, -100, -100]
# \log ( exp(0) + exp(-200) + exp(-200) ) = 0
# so our log softmaxes become: [0, -200, -200]
# Label smoothing: z' = z * (1 - L) + L/n
# 1 = 1 - L + L/n
# 0 = L/n
# Applying the above two fns to the given input:
# -0 * (1 - L + L/n) + 200 * L/n + 200 * L/n = 400 L/n
cce_obj = losses.CategoricalCrossentropy(
from_logits=True, label_smoothing=label_smoothing)
loss = cce_obj(y_true, logits)
expected_value = 400.0 * label_smoothing / 3.0
assert np.isclose(K.eval(loss), expected_value, atol=1e-3)
kf = StratifiedKFold(n_splits=5, shuffle=True, random_state=2020)
fold_num = 1
for train_ind, val_ind in kf.split(np.zeros(len(X_train)), Y_train):
x_train = X_train[train_ind]
y_train = keras.utils.to_categorical(Y_train[train_ind], len(classes))
x_val = X_train[val_ind]
y_val = keras.utils.to_categorical(Y_train[val_ind], len(classes))
ntrain = len(x_train)
nval = len(x_val)
train_gen = idg_aug.flow(x_train, y_train, BATCH_SIZE)
val_gen = idg.flow(x_val, y_val, BATCH_SIZE)
model = build_model()
model.compile(loss=losses.CategoricalCrossentropy(from_logits=True),
optimizer=optimizers.Adam(LEARNING_RATE),
metrics=['accuracy'])
#save best model
output_dir = save_dir+str(fold_num)+'.h5'
checkpoint = ModelCheckpoint(output_dir,
monitor='val_accuracy',
save_best_only=True, mode='max')
callbacks = [checkpoint]
history = model.fit(train_gen,
steps_per_epoch=ntrain // BATCH_SIZE,
epochs=EPOCHS,
callbacks=callbacks,
validation_data = val_gen,
padding='valid')(seq_embedded)
pooled_feat = layers.GlobalMaxPooling1D()(conv_feat) #MaxPooling
conv_models.append(pooled_feat)
conv_merged = layers.concatenate(
conv_models, axis=1) #filter size가 2,3,4,5인 결과들 Concatenation
model_dropout = layers.Dropout(0.5)(conv_merged)
logits = layers.Dense(8, activation='softmax')(model_dropout)
model = Model(seq_input, logits) #(입력,출력)
model.compile(optimizer='adam',
loss=losses.CategoricalCrossentropy(),
metrics=['accuracy'])
model.summary()
#학습 시작
history = model.fit(X_train,
TRAIN_LABELS,
epochs=50,
verbose=True,
validation_data=(X_test, TEST_LABELS),
batch_size=512)
print(history.history['loss'])
print(history.history['accuracy'])
from keras.models import load_model
model.save('KU_NLP') #모델 저장하기
def test_config(self):
cce_obj = losses.CategoricalCrossentropy(
reduction=losses_utils.Reduction.SUM, name='bce_1')
assert cce_obj.name == 'bce_1'
assert cce_obj.reduction == losses_utils.Reduction.SUM
# 首先通过 compile 函数指定网络使用的优化器对象，损失函数，评价指标等：
# 导入优化器，损失函数模块
# 采用 Adam 优化器，学习率为 0.01;采用交叉熵损失函数，包含 Softmax
# 创建 5 层的全连接层网络
network = Sequential([
layers.Dense(256, activation='relu'),
layers.Dense(128, activation='relu'),
layers.Dense(64, activation='relu'),
layers.Dense(32, activation='relu'),
layers.Dense(10)
])
network.build(input_shape=(None, 28 * 28))
network.summary()
network.compile(
optimizer=optimizers.Adam(lr=0.01),
loss=losses.CategoricalCrossentropy(from_logits=True),
metrics=['accuracy'] # 设置测量指标为准确率
)
# 8.2.2模型训练
# 模型装配完成后，即可通过 fit()函数送入待训练的数据和验证用的数据集
# 指定训练集为 train_db，验证集为 val_db,训练 5 个 epochs，每 2 个 epoch 验证一次
# 返回训练信息保存在 history 中
history = network.fit(train_db,
epochs=5,
validation_data=val_db,
validation_freq=2)
# 其中 train_db 为 tf.data.Dataset 对象，也可以传入 Numpy Array 类型的数据；epochs 指定训 练迭代的 epochs 数；validation_data 指定用于验证(测试)的数据集和验证的频率 validation_freq。 运行上述代码即可实现网络的训练与验证的功能，fit 函数会返回训练过程的数据记录 history，其中 history.history 为字典对象，包含了训练过程中的 loss，测量指标等记录项：
history.history # 打印训练记录
# 加载一个 batch 的测试数据
x, y = next(iter(db_test))
model.summary()
#read input
data = pd.read_csv(filename,header=None,index_col=0)
weights = model.layers[0].get_weights()
#ftiaxnw to neo neurwniko, sto opoio tha valw ta varh tou dwsmenou
new_model = keras.Sequential()
new_model.add(layers.Dense(64, activation = 'relu',input_shape=(128,))) #prosthetw layer
new_model.compile(optimizer=optimizers.RMSprop(0.01),loss=losses.CategoricalCrossentropy(),metrics=[metrics.CategoricalAccuracy()])
new_model.summary()
new_model.layers[0].set_weights(weights) #vazw ta weights
interm_output = new_model.predict(data,batch_size=32)
outname='new_representations2.csv'
with open(outname, 'w') as filehandle:
for i in range(len(data.index)):
y_str=data.index[i]
for j in range(len(interm_output[i])):
y_str=y_str+','+str(interm_output[i][j])
input_shape=(img_dims, img_dims, 3),
activation='relu'))
model.add(Layr.MaxPooling2D((2, 2)))
model.add(Layr.Conv2D(128, (3, 3), activation='relu'))
model.add(Layr.MaxPooling2D((2, 2)))
model.add(Layr.Conv2D(256, (3, 3), activation='relu'))
model.add(Layr.MaxPooling2D((2, 2)))
model.add(Layr.Conv2D(512, (3, 3), activation='relu'))
model.add(Layr.MaxPooling2D((2, 2)))
model.add(Layr.Flatten())
model.add(Layr.Dense(units=img_dims * 2, activation='relu'))
# ensure this matches the number of folders in the ImageDataGenerator
model.add(Layr.Dense(units=classes, activation='softmax')) # number of classes
optimizer = Opti.SGD(learning_rate=0.002, momentum=0.9, nesterov=True)
loss = Loss.CategoricalCrossentropy(label_smoothing=0.2, from_logits=True)
model.compile(loss=loss, optimizer=optimizer, metrics=['accuracy'])
# Fit model to data
h = model.fit(
training_set,
steps_per_epoch=1081 // batch_size, # number of training set images, 729
epochs=epochs,
validation_data=test_set,
validation_steps=470 // batch_size) # number of test set images, 229
model.save('models.tmp/model_multiclass10.h5') # save model
# key check
print(h.history.keys())
# Plot cool stuff with matlab
