def train(input_dataframe): # Split the dataset to train and test # Since the fold_num is 5, every train data would be around 80% of the dataset and test data would be around 20% of the dataset. fold_num = 5 kf = KFold(n_splits=fold_num) # Store the best model for prediction best_score = 0 best_model = None history_score = [] # This is used for showing the current fold number cnt = 1 # Training part for train_idx, test_idx in kf.split(input_dataframe): # Split the training and testing part from input dataframe. # It is based on the index train = input_dataframe.iloc[train_idx[:data_used_for_training]] test = input_dataframe.iloc[test_idx[:data_used_for_testing]] # Init the model class model = MyModel() # Prepare the data for training the model X = train.loc[:, 'overview'] y = train.loc[:, 'genres'] # Train the model model.fit(X, y) # Prepare the ground truth and prediction for evaluating the performance. truth = test.loc[:, 'genres'] prediction = model.predict(test.loc[:, 'overview']) # Compute the score score = evaluation(truth, prediction) # Store all the score in this list history_score.append(score) # Store the best model and score if score > best_score: best_score = score best_model = model # Print the current states print('Accuracy of fold %d: %.2f' % (cnt, score)) cnt += 1 # Print the contents print('Best score: ' + str(best_score)) print('Worst score: ' + str(min(history_score))) print('Average score: ' + str(np.array(history_score).mean())) # Save the best model best_model.save_weights(model_name)
def create_confusion_matrix(): ''' creates a confusion matrix ''' X_text, X_num, y = main() X_text_train, X_text_test, X_num_train, X_num_test, y_train, y_test = train_test_split( X_text, X_num, y) model = MyModel() model.fit(X_text_train, X_num_train, y_train) predictions = model.predict(X_text_test, X_num_test) return confusion_matrix(y_test, predictions)
def build_and_train_gru_model(inputs, labels, num_of_labels, do_train=True): num_of_features = len(inputs[0][0]) model = MyModel(input_len=model_config.input_vec_size, num_features=num_of_features, num_labels=num_of_labels) if model_config.continue_train_existing_model: weights_path = model_config.model_file_name if model_config.many_to_many: model.build_model("GRU", [350, 300, 250], [0.05, 0.05, 0.05], weights_path) if do_train: model.fit(inputs, labels, model_path="model", early_stopping_patience=40, val_percentage=validation_perc, batch_size=batch_size, num_epochs=num_epochs) else: # NOTE: train - many inputs to one label labels_many_to_one = Utils.get_many_to_one_labels(labels, num_of_labels) model.build_model("GRU_1", [350, 300, 250], [0.05, 0.05, 0.05], weights_path) if do_train: model.fit(inputs, labels_many_to_one, model_path="model", early_stopping_patience=40, val_percentage=validation_perc, batch_size=batch_size, num_epochs=num_epochs) return model
def build_and_train_conv1_model(inputs, labels, num_of_labels, do_train=True, validation_data=None): inputs_transpose, labels_many_to_one, input_len, num_of_features = Utils.convert_to_cnn_inputs_and_labels(inputs, labels) model = MyModel(input_len=input_len, num_features=num_of_features, num_labels=num_of_labels) weights_path = None if model_config.continue_train_existing_model: weights_path = model_config.model_file_name model.build_model("CONV1", [], [], weights_path=weights_path) if do_train: if validation_data is not None: val_inputs_transpose, val_labels_many_to_one, input_len, num_of_features = Utils.convert_to_cnn_inputs_and_labels( validation_data[0], validation_data[1]) model.fit(inputs_transpose, labels_many_to_one, model_path=model_config.model_file_name,val_percentage=validation_perc, early_stopping_patience=10, validation_data=(val_inputs_transpose,val_labels_many_to_one), batch_size=batch_size, num_epochs=num_epochs) else: model.fit(inputs_transpose, labels_many_to_one, model_path=model_config.model_file_name, early_stopping_patience=10, val_percentage=validation_perc, batch_size=batch_size, num_epochs=num_epochs) return model
optimizer=tf.keras.optimizers.Adam(learning_rate=lr_schedule), loss=Loss(), metrics=[PSNR(), SSIM()], ) # resume checkpoint if config['resume']: model.build((None, None, None, 3)) model.load_weights(checkpoint_path) # save the best model checkpoint model_checkpoint_callback = tf.keras.callbacks.ModelCheckpoint( filepath=checkpoint_path, save_weights_only=True, monitor='val_loss', mode='min', save_best_only=True) # tensorboard visulization tensorboard_callback = tf.keras.callbacks.TensorBoard(log_dir=log_dir, histogram_freq=1) # model fit model.fit(trainset, epochs=config['epochs'], validation_data=valset, validation_freq=1, callbacks=[model_checkpoint_callback, tensorboard_callback]) # model summary print(model.summary())
save_best_only=False, mode='max') model = MyModel(n_classes).model # define optimizers model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy']) model.summary() print(Config) # training hist = model.fit(train_generator, validation_data=valid_generator, epochs=Config['num_epochs'], callbacks=[checkpoint]) loss = model.evaluate(test_generator) plt.plot(hist.history['accuracy']) plt.plot(hist.history['val_accuracy']) plt.title('Test accuracy: ' + str(loss[1])) plt.ylabel('Accuracy') plt.xlabel('Epoch') plt.legend(['Train', 'valid'], loc='upper left') plt.savefig(Config['checkpoint_path'] + '/' + name + '.png') print(loss) if Config['shutown']: import os os.system("sudo shutdown -P now")
#sys.exit() #model = DeepLabV3Plus(image_shape[0], image_shape[1], nclasses=4) model = MyModel(4) model.load_weights(weight_path+'fcn_20191021.ckpt') #optimizer = tf.keras.optimizers.SGD(learning_rate=learning_rate, decay=0.0001) optimizer = tf.keras.optimizers.Adam( learning_rate=learning_rate, beta_1=0.9, beta_2=0.999, epsilon=1e-07, amsgrad=False, name='Adam' ) #mean_IOU = tf.keras.metrics.MeanIoU(num_classes=4) model.compile( optimizer=optimizer, loss=tf.compat.v2.nn.softmax_cross_entropy_with_logits, metrics=['accuracy'] ) config = tf.compat.v1.ConfigProto() config.gpu_options.allow_growth=True sess = tf.compat.v1.Session(config=config) model.fit(train_dataset, epochs=num_epochs, callbacks=[tensorboard, checkpoint]) model.summary()
'venue_latitude', 'venue_longitude'], axis=1) return X, y def clean_raw_data(): df = pd.DataFrame(list(db['raw_collections'].find())) #df = pd.get_dummies(df, columns=['currency', 'country']) df.drop(df.select_dtypes(['object']), inplace=True, axis=1) X = df.drop(['delivery_method', 'event_published','org_facebook', 'org_twitter', 'sale_duration','venue_latitude', 'venue_longitude'], axis=1) return X if __name__ == '__main__': X, y = get_data('data/data.json') model = MyModel() model.fit(X, y) with open('model.pkl', 'wb') as f: # Write the model to a file. pickle.dump(model, f) with open('model.pkl', 'rb') as f: model = pickle.load(f) probs = model.predict_proba(x_test) #DATA BASE db = mc['events'] raw_collections = db['row'] client = EventAPIClient() client.collect()