def test_sample_wrong_X(): """Test either if an error is raised when X is different at fitting and sampling""" # Create the object ada = ADASYN(random_state=RND_SEED) ada.fit(X, Y) assert_raises(RuntimeError, ada.sample, np.random.random((100, 40)), np.array([0] * 50 + [1] * 50))
def test_sample_wrong_X(): """Test either if an error is raised when X is different at fitting and sampling""" # Create the object ada = ADASYN(random_state=RND_SEED) ada.fit(X, Y) assert_raises(RuntimeError, ada.sample, np.random.random((100, 40)), np.array([0] * 50 + [1] * 50))
def test_ada_fit(): # Create the object ada = ADASYN(random_state=RND_SEED) # Fit the data ada.fit(X, Y) # Check if the data information have been computed assert_equal(ada.min_c_, 0) assert_equal(ada.maj_c_, 1) assert_equal(ada.stats_c_[0], 8) assert_equal(ada.stats_c_[1], 12)
def test_ada_fit(): """Test the fitting method""" # Create the object ada = ADASYN(random_state=RND_SEED) # Fit the data ada.fit(X, Y) # Check if the data information have been computed assert_equal(ada.min_c_, 0) assert_equal(ada.maj_c_, 1) assert_equal(ada.stats_c_[0], 8) assert_equal(ada.stats_c_[1], 12)
def test_ada_fit(): ada = ADASYN(random_state=RND_SEED) ada.fit(X, Y) assert ada.ratio_ == {0: 4, 1: 0}
def execute(trainfile, testfile, modeldir, logdir, epochs, batch_size, seed): print("--- Executing") print("Using trainfile: ", trainfile) print("Using testfile: ", testfile) print("Using modeldir: ", modeldir) print("Using logdir: ", logdir) print("Using epochs: ", epochs) print("Using batch_size: ", batch_size) print("Using seed: ", seed) print("--- Loading (transformed) data") data = Data.Data() df = data.load(trainfile) # print("df: ", df.shape) # For scoring (uses sample of training data) fraction = 0.1 X_validation = df.sample(frac=fraction) Y_validation = X_validation["is_attributed"].values X_validation.drop(["is_attributed"], 1, inplace=True) # print("X_validation: ", X_validation.shape) # For training, use the part of data NOT in the validation fraction X_train = df.loc[~df.index.isin(X_validation.index)] Y_train = X_train["is_attributed"].values X_train.drop(["is_attributed"], 1, inplace=True) del df; gc.collect() is_oversampled = False if is_oversampled: print("Loaded X_train: ", X_train.shape) print("---- BEFORE:\n", X_train.head()) print(Y_train) print("unique values: ", set(Y_train)) columns = X_train.columns.values total = len(Y_train) ones = np.sum(Y_train) zeros = total - ones nration = 1.0 nzeros = int(zeros) nones = int(nzeros * nration) ratio = {0:nzeros, 1:nones} print("ratio: ", ratio) oversampler = ADASYN(random_state=seed, ratio=ratio) oversampler.fit(X_train, Y_train) X_resampled, y_resampled = oversampler.sample(X_train, Y_train) X_resampled = X_resampled.astype(int) y_resampled = y_resampled.astype(int) X_train = pd.DataFrame(data=X_resampled, columns=columns) Y_train = y_resampled del X_resampled; del y_resampled; gc.collect() print("Oversampled (Random), ratio: ", ratio, " X_train: ", X_train.shape) print("---- AFTER:\n", X_train.head()) print(Y_train) print("unique values: ", set(Y_train)) X_test = data.load(testfile) print("X_train shape: ", X_train.shape) print("Y_train shape: ", Y_train.shape) print("X_test shape: ", X_test.shape) print("X_validation shape: ", X_validation.shape) print("Y_validation shape: ", Y_validation.shape) print("--- Creating model") model = DenseModelFour.DenseModelFour() print("--- Configuring model") model.configure(X_train, X_test, X_validation) model.set_validation(X_validation, Y_validation) print("--- Training model") model.fit( X_train, Y_train, X_test, modeldir=modeldir, logdir=logdir, epochs=epochs, batch_size=batch_size) del X_train; gc.collect() del Y_train; gc.collect() del X_test; gc.collect() print("--- Scoring model") print("Fraction used for scoring: ", fraction) print("X_validation shape: ", X_validation.shape) print("Y_validation shape: ", Y_validation.shape) roc_auc, probabilities = model.score(X_validation, Y_validation) print("Score probabilities shape: ", probabilities.shape) roc_auc = "{:0.6f}".format(roc_auc) print("Score: ROC-AUC: ",roc_auc) print("--- Saving model") modelfile = modeldir + "/" + "dense-model-final-" + roc_auc + ".h5" model.save(modelfile) print("Model saved to: ", modelfile)
def test_ada_fit(): ada = ADASYN(random_state=RND_SEED) ada.fit(X, Y) assert ada.ratio_ == {0: 4, 1: 0}
def oversample(method, pose_feats, d_list, labels): """Normalize data""" pose_feats_n, d_list_n = norm_feats(pose_feats, d_list) """Extract class indecies and equalize""" idx0 = np.flatnonzero(labels == 0) idx1 = np.flatnonzero(labels == 1) idx2 = np.flatnonzero(labels == 2) dom = np.min([len(idx0), len(idx1), len(idx2)]) n_idx0 = idx0[0:dom - 2] n_idx1 = idx1[0:dom - 2] n_idx2 = idx2[0:dom - 2] n_pose_feats0 = pose_feats_n[n_idx0] n_pose_feats1 = pose_feats_n[n_idx1] n_pose_feats2 = pose_feats_n[n_idx2] pose_feats_n = np.concatenate( [n_pose_feats0, n_pose_feats1, n_pose_feats2]) d_list_n = np.concatenate( [d_list_n[n_idx0], d_list_n[n_idx1], d_list_n[n_idx2]]) labels_n = np.concatenate([labels[n_idx0], labels[n_idx1], labels[n_idx2]]) nidx = np.concatenate([n_idx0, n_idx1, n_idx2]) """Randomize the equalized data""" # Generate the permutation index array. permutation = np.random.permutation(pose_feats_n.shape[0]) # Shuffle the arrays by giving the permutation in the square brackets. pose_feats_n = pose_feats_n[permutation] labels_n = labels_n[permutation] d_list_n = d_list_n[permutation] nidx = nidx[permutation] test = np.zeros([int(np.floor(len(pose_feats_n) / 4)), 66], dtype=np.float64) gt_test = np.zeros([int(np.floor(len(pose_feats_n) / 4))]) depth_test = np.zeros([int(np.floor(len(d_list_n) / 4)), 6], dtype=np.float64) nidx_test = np.zeros([int(np.floor(len(nidx) / 4))]) """Save validation data""" test[:, :] = np.array(pose_feats_n[0:int(np.floor(len(pose_feats_n) / 4)), :]) depth_test[:, :] = np.array(d_list_n[0:int(np.floor(len(pose_feats_n) / 4)), :]) gt_test = np.transpose( np.array(labels_n[0:int(np.floor(len(pose_feats_n) / 4))])) nidx_test = np.transpose( np.array(nidx[0:int(np.floor(len(pose_feats_n) / 4))])) """Extract the indecies used for validation""" pose_feats = np.delete(pose_feats, nidx_test, axis=0) d_list = np.delete(d_list, nidx_test, axis=0) labels = np.delete(labels, nidx_test) labels = labels.astype(int) """For SMOTE or ADASYN on training data""" if method == 2: pose_feats = np.concatenate([pose_feats, d_list], axis=1) fm = ADASYN(ratio='all', n_neighbors=5) fm = fm.fit(pose_feats, labels) elif method == 1: pose_feats = np.concatenate([pose_feats, d_list], axis=1) fm = SMOTE(ratio='all', kind='regular', k_neighbors=5) fm = fm.fit(pose_feats, labels) pose_feats, labels_train = fm.sample(pose_feats, labels) d_list = pose_feats[:, 66:72] """Apply normalization to over-sampled training data""" pose_feats, d_list = norm_feats(pose_feats, d_list) # Generate the permutation index array. permutation = np.random.permutation(pose_feats.shape[0]) # Shuffle the arrays by giving the permutation in the square brackets. pose_feats = pose_feats[permutation] labels_train = labels_train[permutation] d_list = d_list[permutation] train = pose_feats gt_train = labels_train depth_train = d_list return test, train, gt_test, gt_train, depth_train, depth_test
def test_ada_fit(): ada = ADASYN(random_state=RND_SEED) ada.fit(X, Y) assert ada.sampling_strategy_ == {0: 4}