def train_classifier() -> tf.keras.Model:
model = get_classification_model()
model = compile_model(model)
train, val, test = create_datasets()
train = configure_dataset_for_performance(train, BATCH_SIZE)
val = configure_dataset_for_performance(val, BATCH_SIZE)
test = configure_dataset_for_performance(test, BATCH_SIZE)
log_dir = "logs/fit/" + datetime.now().strftime("%Y%m%d-%H%M%S")
callbacks = [
EarlyStopping(monitor="val_loss",
patience=5,
restore_best_weights=True),
TerminateOnNaN(),
TBCallback(log_dir=log_dir, histogram_freq=1),
PlotLossesKeras(),
]
fit_kwargs = dict(
epochs=20,
shuffle=True,
callbacks=callbacks,
)
history = model.fit(train, validation_data=val, **fit_kwargs)
plot_history(history.history)
scores = model.evaluate(test)
metric_dict = dict(zip(model.metrics_names, scores))
print("Test metrics:", metric_dict)
return model
def test_customization():
plot_history(pd.read_csv("tests/big_history.csv", index_col=0), path="plots/normal.png", customization_callback=callami)
plt.close()
plot_history("tests/big_history.csv")
plt.close()
plot_history(["tests/big_history.csv", "tests/big_history.csv"])
plt.close()
with pytest.raises(ValueError):
plot_history(78)
def test_plot_small_history():
plot_history(pd.read_json("tests/small_history.json"),
path="plots/small_history.png")
plt.close()
assert os.path.exists("plots/small_history.png")
plot_history("tests/small_history.json")
plt.close()
plot_history(["tests/small_history.json", "tests/small_history.json"])
plt.close()
plot_history(compress_json.load("tests/small_history.json"))
plt.close()
def test_plot_multi_history():
plot_history([
pd.read_csv("tests/history1.csv"),
pd.read_csv("tests/history2.csv"),
pd.read_csv("tests/history4.csv"),
pd.read_csv("tests/history3.csv")
],
path="plots/multiple.png",
interpolate=True,
max_epochs="min")
plt.close()
plot_history([
pd.read_csv("tests/history1.csv"),
pd.read_csv("tests/history2.csv"),
pd.read_csv("tests/history4.csv"),
pd.read_csv("tests/history3.csv")
],
path="plots/multiple.png",
interpolate=True,
max_epochs="max")
plt.close()
assert os.path.exists("plots/multiple.png")
def test_plot():
plot_history(pd.read_csv("tests/big_history.csv", index_col=0)[:16],
path="plots/interpolated.png",
interpolate=True)
plot_history(pd.read_csv("tests/big_history.csv", index_col=0)[:2],
path="plots/interpolated.png",
interpolate=True)
plot_history(pd.read_csv("tests/big_history.csv", index_col=0),
path="plots/interpolated.png",
interpolate=True)
plt.close()
assert os.path.exists("plots/interpolated.png")
def evaluate_model_TCN(x_tr,
y_tr,
x_te,
y_te,
gamma=2,
epochs=200,
verbose=0,
plot=0,
single_run=0):
"""Training function, to evaluate train set against test set or train set againts validation set
Arguments: (x_tr, y_tr) training data
(x_te, y_te) testing/validation data
gamma: focal loss parameter
epochs= number of times, it passes through the training dataset.
verbose: if true, print all the metrics
plot: if true, print built in plot
single_run: fix random seed to ensure reproducibility
Returns: loss: Last binary focal loss value on test/validation set
accuracy: accuracy on test/validation set
wf1: weighted F1 score
wf1_: custom weighted F1 score (with proportional weights)
mf1: macro F1 score
F1_tab: F1 score per label
Ptab: precision per label
Rtab: recall per label"""
batch_size = 1
w = class_weights(y_tr)
clear_session()
if single_run:
np.random.seed(123)
python_random.seed(123)
tf.random.set_seed(1234)
#-------------------------------------model definition---------------------------#
#Creation TCN object
Tcn = TCN(nb_filters=64,
kernel_size=2,
nb_stacks=1,
dilations=(8, 16, 32, 64, 128, 256, 512, 1024),
return_sequences=True,
activation=LeakyReLU(0.01),
kernel_initializer='he_normal')
i = Input(batch_shape=(1, None, x_tr.shape[2]))
o = Tcn(i)
o = Dense(200,
activation=LeakyReLU(0.01),
kernel_regularizer=keras.regularizers.l1_l2(0.00001))(o)
o = Dense(8, activation='sigmoid')(o)
model = Model(inputs=[i], outputs=[o])
model.compile(optimizer='adam',
loss=BinaryFocalLoss(gamma),
metrics=[
BinaryAccuracy(),
Precision(),
Recall(),
FalseNegatives(),
FalsePositives()
])
model.summary()
#---------------------fit network---------------------------------------------#
hist = model.fit(x_tr,
y_tr,
epochs=epochs,
batch_size=batch_size,
verbose=0,
validation_data=(x_te, y_te))
#---------------------------------evaluate model----------------------------------------------#
#evaluate model on test set (over all classes)
loss, accuracy, P, R, FN, FP = model.evaluate(x_te,
y_te,
batch_size=batch_size,
verbose=verbose)
#save model
if single_run:
model.save('Results/opt_TCN_model')
print("Model saved to Results")
y_pred = model.predict(x_te, batch_size=batch_size, verbose=0)
y_pred[y_pred < 0.5] = 0.
y_pred[y_pred > 0.5] = 1.
y_pred = reshape(y_pred, (y_pred.shape[0] * y_pred.shape[1], 8))
y_te = reshape(y_te, (y_te.shape[0] * y_te.shape[1], 8))
#evaluate F1 score for each label
F1_tab, Ptab, Rtab, wf1_ = F1_score(y_te, y_pred, w)
#evaluate accuracy per label
acc_tab = Acc(y_te, y_pred)
print("-> F1 score per label: ", F1_tab)
print("-> y_pred ", y_pred[:, 4])
#test f1 score built in
f = F1Score(8, threshold=0.5, average='weighted')
f.update_state(y_te, y_pred)
wf1 = f.result().numpy()
print("weighted F1 score built in: ", wf1)
f.reset_states()
f = F1Score(8, threshold=0.5, average='macro')
f.update_state(y_te, y_pred)
mf1 = f.result().numpy()
print("macro F1 score built in: ", mf1)
f.reset_states()
#-----------------------------------------print---------------------------------------------#
#print all
if verbose:
print(" -> Accuracy: ", accuracy, "; Mean of labelwise accuracy: ",
np.mean(acc_tab))
print("Per label accuracy: ", acc_tab)
print("-> Weighted F1 score: ", wf1_)
print("-> F1 score per label: ", F1_tab)
print("-> Precision: ", P, "; Recall: ", R)
print("-> Precision per label: ", Ptab)
print("-> Recall per label: ", Rtab)
print("-> Loss: ", loss)
if plot:
plot_history(hist.history)
return hist, loss, accuracy, wf1, wf1_, mf1, F1_tab, Ptab, Rtab
def evaluate_model(x_tr,
y_tr,
x_te,
y_te,
model_type=1,
gamma=2,
nodes_nb=600,
drop=0.1,
epochs=200,
reg=0,
verbose=0,
plot=0,
single_run=0):
"""Training function, to evaluate train set against test set or train set againts validation set
Arguments: (x_tr, y_tr) training data
(x_te, y_te) testing/validation data
model_type: type of model to train/evaluate (on LSTM layers)
0: 1 LSTM layer model
1: 1 bidirectional LSTM layer model
2: 2 bidirectional LSTM layer model
gamma: focal loss parameter
nodes_nb: number of neurons in the LSTM layers
drop: dropout value
verbose: if true, print all the metrics
plot: if true, print built in plot
single_run: fix random seed to ensure reproducibility
Returns: loss: Last binary focal loss value on test/validation set
accuracy: accuracy on test/validation set
wf1: weighted F1 score
wf1_: custom weighted F1 score (with proportional weights)
mf1: macro F1 score
F1_tab: F1 score per label
Ptab: precision per label
Rtab: recall per label"""
batch_size = 32
n_features, n_outputs = x_tr.shape[2], y_tr.shape[2]
w = class_weights(y_tr)
clear_session()
if single_run:
np.random.seed(2020)
python_random.seed(2020)
tf.random.set_seed(2020)
#-------------------------------------model definition-------------------------------------#
model = Sequential()
#model types
if model_type == 0:
model.add(
LSTM(nodes_nb,
input_shape=(None, n_features),
return_sequences=True))
model.add(Dropout(drop))
model.add(Dense(n_outputs, activation='sigmoid'))
if model_type == 1:
model.add(
Bidirectional(
LSTM(nodes_nb,
input_shape=(None, n_features),
return_sequences=True)))
model.add(Dropout(drop))
model.add(Dense(n_outputs, activation='sigmoid'))
if model_type == 2:
model.add(
Bidirectional(
LSTM(nodes_nb,
input_shape=(None, n_features),
return_sequences=True)))
model.add(Dropout(drop))
model.add(
Bidirectional(
LSTM(nodes_nb,
input_shape=(None, n_features),
return_sequences=True)))
model.add(Dropout(drop))
model.add(Dense(n_outputs, activation='sigmoid'))
model.compile(loss=BinaryFocalLoss(gamma),
optimizer='adam',
metrics=[
BinaryAccuracy(),
Precision(),
Recall(),
FalseNegatives(),
FalsePositives()
])
#------------------------------------fit network---------------------------------------------#
hist = model.fit(x_tr,
y_tr,
epochs=epochs,
batch_size=batch_size,
verbose=0,
validation_data=(x_te, y_te))
if verbose:
model.summary()
#---------------------------------evaluate model----------------------------------------------#
#evaluate model on test set (over all classes)
loss, accuracy, P, R, FN, FP = model.evaluate(x_te,
y_te,
batch_size=batch_size,
verbose=verbose)
#save model
if single_run:
model.save('Results/opt_LSTM_model')
print("Model saved to Results")
y_pred = model.predict(x_te, batch_size=batch_size, verbose=0)
y_pred = reshape(y_pred, (y_pred.shape[0] * y_pred.shape[1], 8))
y_te = reshape(y_te, (y_te.shape[0] * y_te.shape[1], 8))
#evaluate F1 score for each label
F1_tab, Ptab, Rtab, wf1_ = F1_score(y_te, y_pred, w)
#evaluate accuracy per label
acc_tab = Acc(y_te, y_pred)
#test f1 score built in
f = F1Score(8, threshold=0.5, average='weighted')
f.update_state(y_te, y_pred)
wf1 = f.result().numpy()
f.reset_states()
f = F1Score(8, threshold=0.5, average='macro')
f.update_state(y_te, y_pred)
mf1 = f.result().numpy()
f.reset_states()
#-----------------------------------------print---------------------------------------------#
#print all
if verbose:
print(" -> Accuracy: ", accuracy, "; Mean of labelwise accuracy: ",
np.mean(acc_tab))
print("Per label accuracy: ", acc_tab)
print("-> Proportional F1 score: ", wf1_, "; Weighted F1 score: ", wf1,
"; Macro F1 score: ", mf1)
print("-> F1 score per label: ", F1_tab)
print("-> Precision: ", P, "; Recall: ", R)
print("-> Precision per label: ", Ptab)
print("-> Recall per label: ", Rtab)
print("-> Loss: ", loss)
if plot:
plot_history(hist.history)
return hist, loss, accuracy, wf1, wf1_, mf1, F1_tab, Ptab, Rtab
def plot(self, *args, **kwargs):
plot_history(self.to_dataframe()[["score"]], *args, **kwargs)
gen_test_data()
preprocessed_data()
# 3. Load all dataset & normalize
X_train, y_train = load_audio_datafiles(conf, conf.X_train, conf.y_train, normalize=True)
X_test, y_test = load_audio_datafiles(conf, conf.X_test, conf.y_test, normalize=True)
print('Loaded train:test = {}:{} samples.'.format(len(X_train), len(X_test)))
# 4. Train folds
history, model = train_classifier(conf, fold=0,
dataset=[X_train, y_train, X_test, y_test],
model=None,
show_detail=False,
# init_weights=None, # from scratch
init_weights=conf.best_weight_file, # from scratch
)
# 5. Evaluate
evaluate_model(conf, model, X_test, y_test)
print('___ training finished ___')
fn = glob('weights/*.h5')[-1]
shutil.copy(fn, conf.best_weight_file)
os.system("python convert_keras_to_tf.py --model_type {} --keras_weight {} --out_prefix {}".format(conf.model, conf.best_weight_file, os.path.splitext(conf.best_weight_file)[0]))
print('___ trasfer xxx.h5 to xxx.pb finished ___')
plot_history(history.history, path="history.png")
plt.close()
def test_plot_singletons():
os.makedirs("plots/singletons", exist_ok=True)
plot_history(pd.read_json("tests/history.json"),
path="plots/singletons",
single_graphs=True)
plt.close()
def plot_model(model):
plot_history(history.history)
plt.show()
def kerasNet(x_data,
y_data,
NUNITS_INPUT = 64,
NUNITS = 32,
NHIDDEN = 2,
lr = 1e-3,
EPOCHS = 100,
BATCHSIZE = 32,
validation_split = 0.3,
VERBOSE = 2,
optimizer:str = "sgd",
loss = ['mean_squared_error'],
use_gpu:bool = True,
saveModel: bool = False,
plot_results:bool = True,
baseline:bool = False
):
"""
2 hidden layers, sigmoid tanh
"""
if use_gpu:
try:
if is_gpu_available():
print("Using gpu!")
except: print("No GPU")
model = Sequential()
model.add(Dense(NUNITS_INPUT, input_dim=(x_data.shape[1]), activation = "relu", name="First"))
for _ in range(NHIDDEN):
if baseline:
model.name = "Baseline"
model.add(Dense(NUNITS,
activation = "tanh"
))
else:
model.name = "Network1"
model.add(Dense(NUNITS,
activation = "tanh",
kernel_initializer = 'random_uniform',
#bias_initializer = 'random_normal',
#kernel_regularizer=regularizers.l2(0.01),
#activity_regularizer=regularizers.l1(0.01)
))
model.add(Dense(y_data.shape[1],
activation = 'linear',
name = "Final"))
model.compile(loss=loss,
optimizer=optimizer,
metrics = ['mse',rmse, max_error, 'mae']
)
print("X_Train : ", x_data.shape," and Y_Train: ", y_data.shape)
history = model.fit(x_data,
y_data,
validation_split = validation_split,
epochs = EPOCHS,
batch_size= BATCHSIZE,
verbose = VERBOSE
).history
print(model.summary())
plot_history(history)
if saveModel:
model.save(name + '.h5')
print("Saved Model")
return model
model.prepare_valid_positions(X_val[3]))
hist = model.fit(X_train,
Y_train,
validation_data=(X_val, Y_val),
epochs=1,
batch_size=batch_size)
if history:
history = {
key: history[key] + hist.history[key]
for key in hist.history
}
else:
history = hist.history
plot_history(history)
plt.show()
plt.close()
print('Saving ..')
if not os.path.exists(save_folder_path):
os.makedirs(save_folder_path)
print('Folder `%s` created' % save_folder_path)
model.save(save_folder_path)
with open(os.path.join(save_folder_path, 'tags_vectorizer.pkl'),
'wb') as handle:
pickle.dump(tags_vectorizer, handle, protocol=pickle.HIGHEST_PROTOCOL)
with open(os.path.join(save_folder_path, 'intents_label_encoder.pkl'),
'wb') as handle:
pickle.dump(intents_label_encoder,
handle,
def train_model(train_config_path, sess):
logging.basicConfig(stream=sys.stdout,
format='%(message)s',
level=logging.WARNING)
with open(os.path.join(train_config_path, 'train_config.json'),
'r') as json_file:
train_config = json.load(json_file)
data_folder_path = os.path.join(train_config['data_folder_path'],
'train')
save_folder_path = train_config['save_folder_path']
epochs = train_config['epochs']
batch_size = train_config['batch_size']
num_bert_fine_tune_layers = train_config['num_bert_fine_tune_layers']
model_hub_path = train_config['model_hub_path']
logging.log(logging.WARNING, 'Reading data ...')
text_arr, tags_arr, intents = JointBertModel.read_goo(data_folder_path)
logging.log(logging.WARNING, 'Vectorize data ...')
bert_vectorizer = BERTVectorizer(sess, model_hub_path)
input_ids, input_mask, segment_ids, valid_positions, sequence_lengths = bert_vectorizer.transform(
text_arr)
logging.log(logging.WARNING, 'Vectorize tags ...')
tags_vectorizer = TagsVectorizer()
tags_vectorizer.fit(tags_arr)
tags = tags_vectorizer.transform(tags_arr, valid_positions)
slots_num = len(tags_vectorizer.label_encoder.classes_)
logging.log(logging.WARNING, 'Encoding labels ...')
intents_label_encoder = LabelEncoder()
intents = intents_label_encoder.fit_transform(intents).astype(np.int32)
intents_num = len(intents_label_encoder.classes_)
model = JointBertModel(slots_num, intents_num, model_hub_path, sess,
num_bert_fine_tune_layers)
logging.log(logging.WARNING, 'Training model ...')
X = np.concatenate(
(input_ids, input_mask, segment_ids, valid_positions, tags),
axis=1)
Y = intents
split_width = input_ids.shape[1]
history = {}
i = 1
while i <= epochs:
folds = StratifiedKFold(n_splits=5, shuffle=True).split(X, Y)
for train_index, val_index in folds:
if i > epochs:
break
X_train, X_val = X[train_index], X[val_index]
Y_train, Y_val = Y[train_index], Y[val_index]
Y_train = [
X_train[:, 4 * split_width:5 * split_width], Y_train
]
X_train = [
X_train[:, 0:split_width],
X_train[:, split_width:2 * split_width],
X_train[:, 2 * split_width:3 * split_width],
X_train[:, 3 * split_width:4 * split_width]
]
Y_val = [X_val[:, 4 * split_width:5 * split_width], Y_val]
X_val = [
X_val[:,
0:split_width], X_val[:,
split_width:2 * split_width],
X_val[:, 2 * split_width:3 * split_width],
X_val[:, 3 * split_width:4 * split_width]
]
X_train = (X_train[0], X_train[1], X_train[2],
model.prepare_valid_positions(X_train[3]))
X_val = (X_val[0], X_val[1], X_val[2],
model.prepare_valid_positions(X_val[3]))
logging.log(logging.WARNING, 'Epoch %i/%i' % (i, epochs))
hist = model.fit(X_train,
Y_train,
validation_data=(X_val, Y_val),
epochs=1,
batch_size=batch_size)
if history:
history = {
key: history[key] + hist.history[key]
for key in hist.history
}
else:
history = hist.history
i += 1
plot_history(history)
plt.show()
plt.close()
logging.log(logging.WARNING, 'Saving model ...')
if not os.path.exists(save_folder_path):
os.makedirs(save_folder_path)
logging.info('Folder `%s` created' % save_folder_path)
model.save_model(save_folder_path)
with open(os.path.join(save_folder_path, 'tags_vectorizer.pkl'),
'wb') as handle:
pickle.dump(tags_vectorizer,
handle,
protocol=pickle.HIGHEST_PROTOCOL)
with open(os.path.join(save_folder_path, 'intents_label_encoder.pkl'),
'wb') as handle:
pickle.dump(intents_label_encoder,
handle,
protocol=pickle.HIGHEST_PROTOCOL)
return model
def calculate_models(epochs=1, neurons=None):
Path(STAT_DIR).mkdir(parents=True, exist_ok=True)
Path(MODEL_DIR).mkdir(parents=True, exist_ok=True)
timestamp = datetime.now().strftime("%Y_%m_%d_(%H-%M-%S)")
name_modifier = '_sample_' + str(SAMPLE) if SAMPLE is not None else '_sample_None'
name_modifier += '_epochs' + str(EPOCHS)
if globals()['USE_NORMED']:
name_modifier = '_normed_' + name_modifier
model_folder = MODEL_DIR + timestamp + name_modifier + '/'
Path(model_folder).mkdir(parents=True, exist_ok=True)
results = list()
rare_results = list()
if neurons is None:
min_value = to_categorical(globals()['VALIDATION_LABELS']).shape[1]
max_value = len(globals()['DATA'][0])+1
else:
min_value = neurons
max_value = neurons + 1
for x in range(min_value, max_value):
print("Training NN for " + str(x) + " hidden neurons...")
clear_session()
model = create_neural_network(x)
history = train_model(model, epochs)
model.save(model_folder + "model_" + str(x))
plot_history(history, path=model_folder + "model_" + str(x) + "/history.png", graphs_per_row=2)
pyplot.close()
plot_model(
model,
to_file=model_folder + "model_" + str(x) + "/model.png",
show_shapes=True,
show_layer_names=True,
rankdir="TB",
expand_nested=True,
dpi=100,
)
results.append([x] + validate_model(model))
temp = validate_model_for_all_classes(model)
to_append = [x]
for key in sorted(temp.keys()):
to_append += temp[key]
rare_results.append(to_append)
output_file = model_folder + "models" + name_modifier + "_" + timestamp + '.csv'
output_file = open(output_file, "w", encoding="utf-8")
output_file.write('neurons;loss;accuracy;recall;precision\n')
for r in results:
output_file.write(";".join(list(map(str, r))) + "\n")
output_file.close()
rare_output_file = model_folder + "models" + name_modifier + "_rare_" + timestamp + '.csv'
rare_output_file = open(rare_output_file, "w", encoding="utf-8")
rare_output_file.write('neurons;loss 1;accuracy 1;recall 1;precision 1;loss 2;accuracy 2;recall 2;precision 2;' +
'loss 4;accuracy 4;recall 4;precision 4;' +
'loss 6;accuracy 6;recall 6;precision 6;loss 7;accuracy 7;recall 7;precision 7;' +
'loss 8;accuracy 8;recall 8;precision 8;loss 9;accuracy 9;recall 9;precision 9\n')
for r in rare_results:
rare_output_file.write(";".join(list(map(str, r))) + "\n")
rare_output_file.close()
validation_data=(val_x, val_y)).history
pd.DataFrame(history).to_csv("historydota2.csv")
# lets assume `model` is main model
#model_json = model.to_json()
#with open("model_in_json.json", "w") as json_file:
# json.dump(model_json, json_file)
model.save("bobot_model1.h5")
model.save_weights("bobot_model2.h5")
#model.save_weights('weight.h5')
# Predict all Validation data
predict = model.predict(val_x)
plot_model(model, show_shapes=True, expand_nested=True, to_file='model.png')
# Visualize Prediction
df = pd.DataFrame(predict)
df.columns = ['Strength', 'Agility', 'Intelligent']
df.index = val_data[:, 0]
print(df)
plot_history(history,
style='-',
path='singleton',
single_graphs=True,
side=12,
graphs_per_row=1)
#plot_history(history)
plt.show()
def test_plot():
plot_history(pd.read_csv("tests/big_history.csv", index_col=0),
path="plots/normal.png")
plt.close()
assert os.path.exists("plots/normal.png")
