def test_scalar_weighted(self): cce_obj = losses.CategoricalCrossentropy() y_true = K.constant([[1, 0, 0], [0, 1, 0], [0, 0, 1]]) y_pred = K.constant([[.9, .05, .05], [.5, .89, .6], [.05, .01, .94]]) loss = cce_obj(y_true, y_pred, sample_weight=2.3) assert np.isclose(K.eval(loss), .7449, atol=1e-3) # Test with logits. logits = K.constant([[8., 1., 1.], [0., 9., 1.], [2., 3., 5.]]) cce_obj = losses.CategoricalCrossentropy(from_logits=True) loss = cce_obj(y_true, logits, sample_weight=2.3) assert np.isclose(K.eval(loss), .132, atol=1e-3)
def test_all_correct_unweighted(self): y_true = K.constant([[1, 0, 0], [0, 1, 0], [0, 0, 1]]) y_pred = K.constant([[1., 0., 0.], [0., 1., 0.], [0., 0., 1.]]) cce_obj = losses.CategoricalCrossentropy() loss = cce_obj(y_true, y_pred) assert np.isclose(K.eval(loss), 0.0, atol=1e-3) # Test with logits. logits = K.constant([[10., 0., 0.], [0., 10., 0.], [0., 0., 10.]]) cce_obj = losses.CategoricalCrossentropy(from_logits=True) loss = cce_obj(y_true, logits) assert np.isclose(K.eval(loss), 0.0, atol=1e-3)
def test_sample_weighted(self): cce_obj = losses.CategoricalCrossentropy() y_true = K.constant([[1, 0, 0], [0, 1, 0], [0, 0, 1]]) y_pred = K.constant([[.9, .05, .05], [.5, .89, .6], [.05, .01, .94]]) sample_weight = K.constant([[1.2], [3.4], [5.6]], shape=(3, 1)) loss = cce_obj(y_true, y_pred, sample_weight=sample_weight) assert np.isclose(K.eval(loss), 1.0696, atol=1e-3) # Test with logits. logits = K.constant([[8., 1., 1.], [0., 9., 1.], [2., 3., 5.]]) cce_obj = losses.CategoricalCrossentropy(from_logits=True) loss = cce_obj(y_true, logits, sample_weight=sample_weight) assert np.isclose(K.eval(loss), 0.31829, atol=1e-3)
def train_mnist_model_batch_sharded(model, optimizer, mesh, num_epochs, steps_per_epoch, global_batch_size): dataset, _ = get_mnist_datasets(NUM_CLASS, global_batch_size) input_image_layout = dtensor.Layout.batch_sharded(mesh, "batch", rank=4) input_label_layout = dtensor.Layout.batch_sharded(mesh, "batch", rank=2) loss_obj = losses.CategoricalCrossentropy() num_local_devices = mesh.num_local_devices() iterator = iter(dataset) train_losses = [] for epoch in range(num_epochs): total_loss = 0.00 for _ in range(steps_per_epoch): images, labels = next(iterator) images = tf.split(images, num_local_devices) labels = tf.split(labels, num_local_devices) d_images = dtensor.pack(images, input_image_layout) d_labels = dtensor.pack(labels, input_label_layout) total_loss += train_step(model, d_images, d_labels, loss_obj, optimizer) train_loss = tf.reduce_mean(total_loss / steps_per_epoch) logging.info("Epoch %d, Loss: %f", epoch, train_loss) train_losses.append(train_loss) return train_losses
def cnn_lstm_optimization(x_train, y_train, x_test, y_test, params): """Randomized search to optimize parameters of Neural Network.""" optimization_model = models.Sequential() optimization_model.add( layers.Conv1D(params['convolutional'], params['kernel'], activation=params['conv_activation'])) optimization_model.add(layers.MaxPooling1D(pool_size=1)) optimization_model.add(layers.LSTM(params['units'], return_sequences=True)) optimization_model.add(layers.LSTM(params['units'], return_sequences=False)) optimization_model.add(layers.Dropout(0.5)) optimization_model.add(layers.Dense(params['dense'], activation=relu)) optimization_model.add(layers.Dense(params['dense'], activation=relu)) optimization_model.add(layers.Dense(int(num_classes), activation='softmax')) optimization_model.compile( optimizer=params['optimizer'], loss=losses.CategoricalCrossentropy(), metrics=['accuracy', talos.utils.metrics.f1score]) history = optimization_model.fit(x_train, y_train, batch_size=None, epochs=params['epoch'], validation_data=(x_test, y_test)) return history, optimization_model
def test_no_reduction(self): y_true = K.constant([[1, 0, 0], [0, 1, 0], [0, 0, 1]]) logits = K.constant([[8., 1., 1.], [0., 9., 1.], [2., 3., 5.]]) cce_obj = losses.CategoricalCrossentropy( from_logits=True, reduction=Reduction.NONE) loss = cce_obj(y_true, logits) assert np.allclose(K.eval(loss), (0.001822, 0.000459, 0.169846), atol=1e-3)
def test_label_smoothing(self): logits = K.constant([[100.0, -100.0, -100.0]]) y_true = K.constant([[1, 0, 0]]) label_smoothing = 0.1 # Softmax Cross Entropy Loss: -\sum_i p_i \log q_i # where for a softmax activation # \log q_i = x_i - \log \sum_j \exp x_j # = x_i - x_max - \log \sum_j \exp (x_j - x_max) # For our activations, [100, -100, -100] # \log ( exp(0) + exp(-200) + exp(-200) ) = 0 # so our log softmaxes become: [0, -200, -200] # Label smoothing: z' = z * (1 - L) + L/n # 1 = 1 - L + L/n # 0 = L/n # Applying the above two fns to the given input: # -0 * (1 - L + L/n) + 200 * L/n + 200 * L/n = 400 L/n cce_obj = losses.CategoricalCrossentropy( from_logits=True, label_smoothing=label_smoothing) loss = cce_obj(y_true, logits) expected_value = 400.0 * label_smoothing / 3.0 assert np.isclose(K.eval(loss), expected_value, atol=1e-3)
kf = StratifiedKFold(n_splits=5, shuffle=True, random_state=2020) fold_num = 1 for train_ind, val_ind in kf.split(np.zeros(len(X_train)), Y_train): x_train = X_train[train_ind] y_train = keras.utils.to_categorical(Y_train[train_ind], len(classes)) x_val = X_train[val_ind] y_val = keras.utils.to_categorical(Y_train[val_ind], len(classes)) ntrain = len(x_train) nval = len(x_val) train_gen = idg_aug.flow(x_train, y_train, BATCH_SIZE) val_gen = idg.flow(x_val, y_val, BATCH_SIZE) model = build_model() model.compile(loss=losses.CategoricalCrossentropy(from_logits=True), optimizer=optimizers.Adam(LEARNING_RATE), metrics=['accuracy']) #save best model output_dir = save_dir+str(fold_num)+'.h5' checkpoint = ModelCheckpoint(output_dir, monitor='val_accuracy', save_best_only=True, mode='max') callbacks = [checkpoint] history = model.fit(train_gen, steps_per_epoch=ntrain // BATCH_SIZE, epochs=EPOCHS, callbacks=callbacks, validation_data = val_gen,
padding='valid')(seq_embedded) pooled_feat = layers.GlobalMaxPooling1D()(conv_feat) #MaxPooling conv_models.append(pooled_feat) conv_merged = layers.concatenate( conv_models, axis=1) #filter size가 2,3,4,5인 결과들 Concatenation model_dropout = layers.Dropout(0.5)(conv_merged) logits = layers.Dense(8, activation='softmax')(model_dropout) model = Model(seq_input, logits) #(입력,출력) model.compile(optimizer='adam', loss=losses.CategoricalCrossentropy(), metrics=['accuracy']) model.summary() #학습 시작 history = model.fit(X_train, TRAIN_LABELS, epochs=50, verbose=True, validation_data=(X_test, TEST_LABELS), batch_size=512) print(history.history['loss']) print(history.history['accuracy']) from keras.models import load_model model.save('KU_NLP') #모델 저장하기
def test_config(self): cce_obj = losses.CategoricalCrossentropy( reduction=losses_utils.Reduction.SUM, name='bce_1') assert cce_obj.name == 'bce_1' assert cce_obj.reduction == losses_utils.Reduction.SUM
# 首先通过 compile 函数指定网络使用的优化器对象,损失函数,评价指标等: # 导入优化器,损失函数模块 # 采用 Adam 优化器,学习率为 0.01;采用交叉熵损失函数,包含 Softmax # 创建 5 层的全连接层网络 network = Sequential([ layers.Dense(256, activation='relu'), layers.Dense(128, activation='relu'), layers.Dense(64, activation='relu'), layers.Dense(32, activation='relu'), layers.Dense(10) ]) network.build(input_shape=(None, 28 * 28)) network.summary() network.compile( optimizer=optimizers.Adam(lr=0.01), loss=losses.CategoricalCrossentropy(from_logits=True), metrics=['accuracy'] # 设置测量指标为准确率 ) # 8.2.2模型训练 # 模型装配完成后,即可通过 fit()函数送入待训练的数据和验证用的数据集 # 指定训练集为 train_db,验证集为 val_db,训练 5 个 epochs,每 2 个 epoch 验证一次 # 返回训练信息保存在 history 中 history = network.fit(train_db, epochs=5, validation_data=val_db, validation_freq=2) # 其中 train_db 为 tf.data.Dataset 对象,也可以传入 Numpy Array 类型的数据;epochs 指定训 练迭代的 epochs 数;validation_data 指定用于验证(测试)的数据集和验证的频率 validation_freq。 运行上述代码即可实现网络的训练与验证的功能,fit 函数会返回训练过程的数据记录 history,其中 history.history 为字典对象,包含了训练过程中的 loss,测量指标等记录项: history.history # 打印训练记录 # 加载一个 batch 的测试数据 x, y = next(iter(db_test))
model.summary() #read input data = pd.read_csv(filename,header=None,index_col=0) weights = model.layers[0].get_weights() #ftiaxnw to neo neurwniko, sto opoio tha valw ta varh tou dwsmenou new_model = keras.Sequential() new_model.add(layers.Dense(64, activation = 'relu',input_shape=(128,))) #prosthetw layer new_model.compile(optimizer=optimizers.RMSprop(0.01),loss=losses.CategoricalCrossentropy(),metrics=[metrics.CategoricalAccuracy()]) new_model.summary() new_model.layers[0].set_weights(weights) #vazw ta weights interm_output = new_model.predict(data,batch_size=32) outname='new_representations2.csv' with open(outname, 'w') as filehandle: for i in range(len(data.index)): y_str=data.index[i] for j in range(len(interm_output[i])): y_str=y_str+','+str(interm_output[i][j])
input_shape=(img_dims, img_dims, 3), activation='relu')) model.add(Layr.MaxPooling2D((2, 2))) model.add(Layr.Conv2D(128, (3, 3), activation='relu')) model.add(Layr.MaxPooling2D((2, 2))) model.add(Layr.Conv2D(256, (3, 3), activation='relu')) model.add(Layr.MaxPooling2D((2, 2))) model.add(Layr.Conv2D(512, (3, 3), activation='relu')) model.add(Layr.MaxPooling2D((2, 2))) model.add(Layr.Flatten()) model.add(Layr.Dense(units=img_dims * 2, activation='relu')) # ensure this matches the number of folders in the ImageDataGenerator model.add(Layr.Dense(units=classes, activation='softmax')) # number of classes optimizer = Opti.SGD(learning_rate=0.002, momentum=0.9, nesterov=True) loss = Loss.CategoricalCrossentropy(label_smoothing=0.2, from_logits=True) model.compile(loss=loss, optimizer=optimizer, metrics=['accuracy']) # Fit model to data h = model.fit( training_set, steps_per_epoch=1081 // batch_size, # number of training set images, 729 epochs=epochs, validation_data=test_set, validation_steps=470 // batch_size) # number of test set images, 229 model.save('models.tmp/model_multiclass10.h5') # save model # key check print(h.history.keys()) # Plot cool stuff with matlab