def calculate_t(dataset_no): print("Starting to find orignal_labels ans for dataset no", dataset_no) X, y = load_datasets(dataset_no) rows, col = X.shape kernel = 1.0 * RBF(1.0) ROW = int(Training_percent * rows) ROW = 800 print("Starting $") gpc = GaussianProcessClassifier(kernel=kernel, random_state=0).fit(X[:ROW, :], y[:ROW]) print("Successfully trained ", dataset_no) print("Starting predicting data for full length") orignal_probability = gpc.predict_proba(X[:ROW, :]) print("Orignal_probability array calculated for dataset_no", dataset_no) mrl = [None for _ in range(5)] current_pos = int(STARTING_FRACTION * col) while 1: gpc = GaussianProcessClassifier(kernel=kernel, random_state=0).fit( X[:ROW, :current_pos], y[:ROW]) new_probability = gpc.predict_proba(X[:ROW, :current_pos]) print("Probabilities calculated for current value of f = ", current_pos) for i in range(5): value_mrl = mrl[i] if not (value_mrl): if i == 4: temporary = 16 else: temporary = i + 1 if (check_probabilities_for_f(orignal_probability, new_probability, alpha, y, temporary)): mrl[i] = current_pos print("F for label", temporary, " is ", current_pos) # print("Saving model") # s = 'label_id' + str(i+1) + "component" + str(dataset_no) # filename = 'models/' + s + '.sav' # pickle.dump(gpc, open(filename, 'wb')) all_completed = 1 for value_mrl in mrl: if not (value_mrl): all_completed = 0 if all_completed: break current_pos = current_pos + 5 return mrl
def apply_gaussian_classifier(feature, col_required, array_to_predict): Main_X, Main_Y = load_datasets(feature) print("Starting Gausian") gpc = GaussianProcessClassifier(kernel=kernel, random_state=0).fit( Main_X[:ROWS, :col_required], Main_Y[:ROWS]) print("Successfully Trained :)") orignal_probability = gpc.predict_proba(array_to_predict) maxa = max(orignal_probability[0]) for j in range(5): if (orignal_probability[0][j] == maxa): index = j if (index == 0): return 1 elif (index == 1): return 2 elif (index == 2): return 3 elif (index == 3): return 4 else: return 16
dir_path = os.path.join(path, 'dataset_agument') ################################################# # use for make pickle data make_dir(dir_path , 'pickle_data') data_path = os.path.join(dir_path, 'pickle_data') for ver in version: make_dir(data_path , ver) file_path = os.path.join(raw_data_path, ver) for file in os.listdir(file_path): if file == '2018Y02M20D09H59m22s': continue loads = load_datasets(ver, file) dataset = loads.load_tracklets() datas = loads.load_INS() fd = classify_tracklets(dataset) xyz = TRtoTM(fd, datas, ver) fd = len_filter(xyz) emp = detect_emptynum(fd) ffd = fill_emptynum(fd, emp) ffd = len_filter(ffd, fp=True) save = save_datasets(data_path, ver, file) save.save_as_pkl(ffd)
from sklearn.model_selection import train_test_split from load_datasets import load_datasets import numpy as np X, y = load_datasets() def load_test_data(): X_test = X[650:] y_test = y[650:] return X_test, y_test def load_train_valid_data(test_split): X_tmp = X[:650] y_tmp = y[:650] X_train, X_valid, y_train, y_valid = train_test_split(X_tmp, y_tmp, test_size=test_split, random_state=1) return X_train, X_valid, y_train, y_valid
def training( network, learning_rate, hidden_units, epochs, gpu): train_dir = './flowers/train/' valid_dir = './flowers/valid/' test_dir = './flowers/test/' train_data, valid_data, test_data = load_datasets(train_dir, valid_dir, test_dir) trainloader, validloader, testloader = get_loaders(train_data, valid_data, test_data) model, classifier = get_model_classifier(network, get_class_tot(train_dir, valid_dir, test_dir), hidden_units) train_losses, valid_losses = [], [] if gpu == True: # use GPU device = torch.device('cuda') else: device = torch.device('cpu') criterion = nn.NLLLoss() optimizer = optim.Adam(model.classifier.parameters(), lr=0.001) model.to(device) for epoch in range(epochs): epoch_train_run_loss = 0 epoch_batches = 0 print(f"Epoch for {epoch+1}/{epochs}..") for images, labels in trainloader: epoch_batches += 1 images, labels = images.to(device), labels.to(device) optimizer.zero_grad() logps = model(images) loss = criterion(logps, labels) loss.backward() optimizer.step() epoch_train_run_loss += loss.item() with torch.no_grad(): model.eval() valid_loss, valid_accuracy = validation(model, validloader, criterion, device) valid_losses.append(valid_loss) model.train() train_losses.append(epoch_train_run_loss/epoch_batches) print(f"Final Train loss: {epoch_train_run_loss/epoch_batches:.4f}" f", Validation loss: {valid_loss:.4f} " f"and Validation accuracy: {valid_accuracy:.4f} for Epoch {epoch+1}/{epochs}") print("Validation Completed Successfully. Saving started") #saving checkpoint save_model_checkpoint(model, train_data, optimizer, epochs)
def main(): start_time = time.time() print("Loading data...") train_dataset, valid_dataset, eval_dataset = load_datasets() print("Done loading") train_data, train_labels = train_dataset train_labels = np.asarray(train_labels, dtype=np.int32) valid_data, valid_labels = valid_dataset valid_labels = np.asarray(valid_labels, dtype=np.int32) config = tf.estimator.RunConfig(model_dir = "Models/gray_scale_6.0", keep_checkpoint_max = 25, save_checkpoints_steps = 1000 ) params = tf.contrib.training.HParams( learning_rate = 0.001, dense_layers = 1, dense_layer_units = 1024, dropout_rate = 0.3, optimizer = "Adam", cross_entropy_weight = 1.0, ##### Higher level ####### color_channels = 1 ) # Create the Estimator face_classifier = tf.estimator.Estimator(model_fn=cnn_face_classification_fn, config = config, params = params) # Set up logging for predictions # Log the values in the "Softmax" tensor with label "probabilities" tensors_to_log = {"probabilities": "softmax_tensor"} logging_hook = tf.train.LoggingTensorHook( tensors=tensors_to_log, every_n_iter=1000) my_train = True # the old way if my_train: face_classifier.train(input_fn=lambda: imgs_input_fn(train_data, train_labels, perform_shuffle = True, batch_size = 25), hooks=[logging_hook], steps = 5000) train_results = face_classifier.evaluate(input_fn=lambda: imgs_input_fn(train_data, train_labels, perform_shuffle = False, batch_size = 25)) valid_results = face_classifier.evaluate(input_fn=lambda: imgs_input_fn(valid_data, valid_labels, perform_shuffle = False, batch_size = 25)) print("------------------------------------------") print(train_results) print(valid_results) """ # the new way train_spec = tf.estimator.TrainSpec(input_fn=lambda: imgs_input_fn(train_data, train_labels, perform_shuffle=True, batch_size=15), max_steps=50000, hooks=[logging_hook]) eval_spec = tf.estimator.EvalSpec(input_fn=lambda: imgs_input_fn(valid_data, valid_labels, perform_shuffle=False, batch_size=len(valid_data)), steps = 5000) tf.estimator.train_and_evaluate(face_classifier, train_spec, eval_spec) """ print("--- %s seconds ---" % (time.time() - start_time))