def train(input_dataframe):
# Split the dataset to train and test
# Since the fold_num is 5, every train data would be around 80% of the dataset and test data would be around 20% of the dataset.
fold_num = 5
kf = KFold(n_splits=fold_num)
# Store the best model for prediction
best_score = 0
best_model = None
history_score = []
# This is used for showing the current fold number
cnt = 1
# Training part
for train_idx, test_idx in kf.split(input_dataframe):
# Split the training and testing part from input dataframe.
# It is based on the index
train = input_dataframe.iloc[train_idx[:data_used_for_training]]
test = input_dataframe.iloc[test_idx[:data_used_for_testing]]
# Init the model class
model = MyModel()
# Prepare the data for training the model
X = train.loc[:, 'overview']
y = train.loc[:, 'genres']
# Train the model
model.fit(X, y)
# Prepare the ground truth and prediction for evaluating the performance.
truth = test.loc[:, 'genres']
prediction = model.predict(test.loc[:, 'overview'])
# Compute the score
score = evaluation(truth, prediction)
# Store all the score in this list
history_score.append(score)
# Store the best model and score
if score > best_score:
best_score = score
best_model = model
# Print the current states
print('Accuracy of fold %d: %.2f' % (cnt, score))
cnt += 1
# Print the contents
print('Best score: ' + str(best_score))
print('Worst score: ' + str(min(history_score)))
print('Average score: ' + str(np.array(history_score).mean()))
# Save the best model
best_model.save_weights(model_name)
def create_confusion_matrix():
''' creates a confusion matrix '''
X_text, X_num, y = main()
X_text_train, X_text_test, X_num_train, X_num_test, y_train, y_test = train_test_split(
X_text, X_num, y)
model = MyModel()
model.fit(X_text_train, X_num_train, y_train)
predictions = model.predict(X_text_test, X_num_test)
return confusion_matrix(y_test, predictions)
def build_and_train_gru_model(inputs, labels, num_of_labels, do_train=True):
num_of_features = len(inputs[0][0])
model = MyModel(input_len=model_config.input_vec_size, num_features=num_of_features, num_labels=num_of_labels)
if model_config.continue_train_existing_model:
weights_path = model_config.model_file_name
if model_config.many_to_many:
model.build_model("GRU", [350, 300, 250], [0.05, 0.05, 0.05], weights_path)
if do_train:
model.fit(inputs, labels, model_path="model", early_stopping_patience=40, val_percentage=validation_perc,
batch_size=batch_size, num_epochs=num_epochs)
else:
# NOTE: train - many inputs to one label
labels_many_to_one = Utils.get_many_to_one_labels(labels, num_of_labels)
model.build_model("GRU_1", [350, 300, 250], [0.05, 0.05, 0.05], weights_path)
if do_train:
model.fit(inputs, labels_many_to_one, model_path="model", early_stopping_patience=40, val_percentage=validation_perc,
batch_size=batch_size, num_epochs=num_epochs)
return model
def build_and_train_conv1_model(inputs, labels, num_of_labels, do_train=True, validation_data=None):
inputs_transpose, labels_many_to_one, input_len, num_of_features = Utils.convert_to_cnn_inputs_and_labels(inputs,
labels)
model = MyModel(input_len=input_len, num_features=num_of_features, num_labels=num_of_labels)
weights_path = None
if model_config.continue_train_existing_model:
weights_path = model_config.model_file_name
model.build_model("CONV1", [], [], weights_path=weights_path)
if do_train:
if validation_data is not None:
val_inputs_transpose, val_labels_many_to_one, input_len, num_of_features = Utils.convert_to_cnn_inputs_and_labels(
validation_data[0],
validation_data[1])
model.fit(inputs_transpose, labels_many_to_one, model_path=model_config.model_file_name,val_percentage=validation_perc,
early_stopping_patience=10, validation_data=(val_inputs_transpose,val_labels_many_to_one), batch_size=batch_size, num_epochs=num_epochs)
else:
model.fit(inputs_transpose, labels_many_to_one, model_path=model_config.model_file_name, early_stopping_patience=10,
val_percentage=validation_perc, batch_size=batch_size, num_epochs=num_epochs)
return model
optimizer=tf.keras.optimizers.Adam(learning_rate=lr_schedule),
loss=Loss(),
metrics=[PSNR(), SSIM()],
)
# resume checkpoint
if config['resume']:
model.build((None, None, None, 3))
model.load_weights(checkpoint_path)
# save the best model checkpoint
model_checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(
filepath=checkpoint_path,
save_weights_only=True,
monitor='val_loss',
mode='min',
save_best_only=True)
# tensorboard visulization
tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir,
histogram_freq=1)
# model fit
model.fit(trainset,
epochs=config['epochs'],
validation_data=valset,
validation_freq=1,
callbacks=[model_checkpoint_callback, tensorboard_callback])
# model summary
print(model.summary())
save_best_only=False,
mode='max')
model = MyModel(n_classes).model
# define optimizers
model.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
model.summary()
print(Config)
# training
hist = model.fit(train_generator,
validation_data=valid_generator,
epochs=Config['num_epochs'],
callbacks=[checkpoint])
loss = model.evaluate(test_generator)
plt.plot(hist.history['accuracy'])
plt.plot(hist.history['val_accuracy'])
plt.title('Test accuracy: ' + str(loss[1]))
plt.ylabel('Accuracy')
plt.xlabel('Epoch')
plt.legend(['Train', 'valid'], loc='upper left')
plt.savefig(Config['checkpoint_path'] + '/' + name + '.png')
print(loss)
if Config['shutown']:
import os
os.system("sudo shutdown -P now")
#sys.exit()
#model = DeepLabV3Plus(image_shape[0], image_shape[1], nclasses=4)
model = MyModel(4)
model.load_weights(weight_path+'fcn_20191021.ckpt')
#optimizer = tf.keras.optimizers.SGD(learning_rate=learning_rate, decay=0.0001)
optimizer = tf.keras.optimizers.Adam(
learning_rate=learning_rate, beta_1=0.9, beta_2=0.999, epsilon=1e-07, amsgrad=False,
name='Adam'
)
#mean_IOU = tf.keras.metrics.MeanIoU(num_classes=4)
model.compile(
optimizer=optimizer,
loss=tf.compat.v2.nn.softmax_cross_entropy_with_logits,
metrics=['accuracy']
)
config = tf.compat.v1.ConfigProto()
config.gpu_options.allow_growth=True
sess = tf.compat.v1.Session(config=config)
model.fit(train_dataset, epochs=num_epochs, callbacks=[tensorboard, checkpoint])
model.summary()
'venue_latitude', 'venue_longitude'], axis=1)
return X, y
def clean_raw_data():
df = pd.DataFrame(list(db['raw_collections'].find()))
#df = pd.get_dummies(df, columns=['currency', 'country'])
df.drop(df.select_dtypes(['object']), inplace=True, axis=1)
X = df.drop(['delivery_method', 'event_published','org_facebook',
'org_twitter', 'sale_duration','venue_latitude', 'venue_longitude'], axis=1)
return X
if __name__ == '__main__':
X, y = get_data('data/data.json')
model = MyModel()
model.fit(X, y)
with open('model.pkl', 'wb') as f:
# Write the model to a file.
pickle.dump(model, f)
with open('model.pkl', 'rb') as f:
model = pickle.load(f)
probs = model.predict_proba(x_test)
#DATA BASE
db = mc['events']
raw_collections = db['row']
client = EventAPIClient()
client.collect()
