def test_lmnn(self):
lmnn = LMNN(n_neighbors=5, learn_rate=1e-6, verbose=False)
lmnn.fit(self.X, self.y)
res_1 = lmnn.transform(self.X)
lmnn = LMNN(n_neighbors=5, learn_rate=1e-6, verbose=False)
res_2 = lmnn.fit_transform(self.X, self.y)
assert_array_almost_equal(res_1, res_2)
def test_iris(self):
lmnn = LMNN(k=5, learn_rate=1e-6, verbose=False)
lmnn.fit(self.iris_points, self.iris_labels)
csep = class_separation(lmnn.transform(self.iris_points),
self.iris_labels)
self.assertLess(csep, 0.25)
def sandwich_demo():
x, y = sandwich_data()
knn = nearest_neighbors(x, k=2)
ax = plt.subplot(3, 1, 1) # take the whole top row
plot_sandwich_data(x, y, ax)
plot_neighborhood_graph(x, knn, y, ax)
ax.set_title('input space')
ax.set_aspect('equal')
ax.set_xticks([])
ax.set_yticks([])
mls = [
LMNN(),
ITML_Supervised(num_constraints=200),
SDML_Supervised(num_constraints=200),
LSML_Supervised(num_constraints=200),
]
for ax_num, ml in enumerate(mls, start=3):
ml.fit(x, y)
tx = ml.transform()
ml_knn = nearest_neighbors(tx, k=2)
ax = plt.subplot(3, 2, ax_num)
plot_sandwich_data(tx, y, axis=ax)
plot_neighborhood_graph(tx, ml_knn, y, axis=ax)
ax.set_title(ml.__class__.__name__)
ax.set_xticks([])
ax.set_yticks([])
plt.show()
def __init__(self, k=3, pca=None, train=True, mu=0.5):
self.k = k
self.train = train
self.pca = pca
self.pca_trasform = None
self.mu = mu
self.lmnn = LMNN(k=k, use_pca=False, max_iter=10000, regularization=mu)
def test_toy_ex_lmnn(X, y, loss):
"""Test that the loss give the right result on a toy example"""
L = np.array([[1]])
lmnn = LMNN(k=1, regularization=0.5)
k = lmnn.k
reg = lmnn.regularization
X, y = lmnn._prepare_inputs(X, y, dtype=float, ensure_min_samples=2)
num_pts, n_components = X.shape
unique_labels, label_inds = np.unique(y, return_inverse=True)
lmnn.labels_ = np.arange(len(unique_labels))
lmnn.components_ = np.eye(n_components)
target_neighbors = lmnn._select_targets(X, label_inds)
# sum outer products
dfG = _sum_outer_products(X, target_neighbors.flatten(),
np.repeat(np.arange(X.shape[0]), k))
# storage
a1 = [None] * k
a2 = [None] * k
for nn_idx in range(k):
a1[nn_idx] = np.array([])
a2[nn_idx] = np.array([])
# assert that the loss equals the one computed by hand
assert lmnn._loss_grad(X, L.reshape(-1, X.shape[1]), dfG, k, reg,
target_neighbors, label_inds)[1] == loss
def constructSimilartyMatrixLMNN(self, ks):
print('now doing LMNN for k= ', ks)
self.y_train = self.y_train.reshape(-1, )
lmnn = LMNN(k=ks, learn_rate=1e-7, max_iter=1000)
lmnn.fit(self.trainVectorsPCA, self.y_train)
self.L_lmnn = lmnn.transformer()
name = 'lmnn/LMNN transformer matrix with dataset shape ' + str(
self.trainVectorsPCA.shape)
np.save(name, self.L_lmnn)
print('L.shape is ', self.L_lmnn.shape, '\n\n')
# Input data transformed to the metric space by X*L.T
self.transformedTrainLMNN = copy(lmnn.transform(self.trainVectorsPCA))
self.transformedTestLMNN = copy(lmnn.transform(self.testVectorsPCA))
self.transformedAllLMNN = copy(lmnn.transform(
self.allDataPCA)) #we compute the pairwise distance on this now
projectedDigits = TSNE(random_state=randomState).fit_transform(
self.transformedAllLMNN)
self.pwdis = copy(
pairwise_distances(self.transformedAllLMNN, metric='euclidean'))
self.D = np.zeros(self.pwdis.shape)
for i in range(0, self.pwdis.shape[0]):
l1 = self.pwdis[i].tolist()
#print 'l1 is ',l1,'\n\n'
allnearestNeighbours = sorted(range(len(l1)), key=lambda i: l1[i])
#now set the all the weights except for k+1 to 0
self.pwdis[i, allnearestNeighbours[ks:]] = 0
self.D[i, i] = sum(self.pwdis[i])
print('accuracy for LMNN for k= ', ks, '\n')
self.labelPropogation()
def sandwich_demo():
x, y = sandwich_data()
knn = nearest_neighbors(x, k=2)
ax = pyplot.subplot(3, 1, 1) # take the whole top row
plot_sandwich_data(x, y, ax)
plot_neighborhood_graph(x, knn, y, ax)
ax.set_title('input space')
ax.set_aspect('equal')
ax.set_xticks([])
ax.set_yticks([])
num_constraints = 60
mls = [(LMNN(), (x, y)),
(ITML(), (x, ITML.prepare_constraints(y, len(x), num_constraints))),
(SDML(), (x, SDML.prepare_constraints(y, len(x), num_constraints))),
(LSML(), (x, LSML.prepare_constraints(y, num_constraints)))]
for ax_num, (ml, args) in zip(xrange(3, 7), mls):
ml.fit(*args)
tx = ml.transform()
ml_knn = nearest_neighbors(tx, k=2)
ax = pyplot.subplot(3, 2, ax_num)
plot_sandwich_data(tx, y, ax)
plot_neighborhood_graph(tx, ml_knn, y, ax)
ax.set_title('%s space' % ml.__class__.__name__)
ax.set_xticks([])
ax.set_yticks([])
pyplot.show()
def process_lmnn(self, **option):
'''Metric Learning algorithm: LMNN'''
GeneExp = self.GeneExp_train
Label = self.Label_train
lmnn = LMNN(**option)
lmnn.fit(GeneExp, Label)
self.Trans['LMNN'] = lmnn.transformer()
def test_convergence_simple_example(capsys):
# LMNN should converge on this simple example, which it did not with
# this issue: https://github.com/scikit-learn-contrib/metric-learn/issues/88
X, y = make_classification(random_state=0)
lmnn = LMNN(verbose=True)
lmnn.fit(X, y)
out, _ = capsys.readouterr()
assert "LMNN converged with objective" in out
def LMNN(self):
print "Warning, the features will be transformed"
lmnn = LMNN(k=5, learn_rate = 1e-6)
lmnn.fit(self.features, targets)
self.features = lmnn.transform(self.features)
self.prepare_for_testing()
self.nearest_neighbors("LMNN + KNN")
def runLMNN(X_train, X_test, y_train, t_test, k):
transformer = LMNN(k=k, learn_rate=1e-6, convergence_tol=0.1, verbose=True)
transformer.fit(X_train, y_train)
X_train_proj = transformer.transform(X_train)
X_test_proj = transformer.transform(X_test)
np.save('X_train_LMNN_' + str(k), X_train_proj)
np.save('X_test_LMNN_' + str(k), X_test_proj)
return X_train_proj, X_test_proj
def train_test(self, x_train, y_train, x_test=None, cuisines=None, k=15):
torch.cuda.empty_cache()
x_train = np.array(x_train)
testing = x_test is not None
if testing:
x_tr = torch.tensor(x_train)
y_tr = torch.tensor(y_train)
x_val = torch.tensor(x_test)
dist = self.get_dist(x_tr, x_val)
y_pred = self.predict(dist, y_tr, k)
ids = [cuisine.id for cuisine in cuisines]
pred_cuisines = [
self.dataset.id2cuisine[label] for label in y_pred
]
self._write2csv(ids, pred_cuisines)
else:
shuffle_idx = torch.randperm(x_train.shape[0])
x_train = torch.tensor(x_train).float()
y_train = torch.tensor(y_train)
x_train = x_train[shuffle_idx]
y_train = y_train[shuffle_idx]
x_val = x_train[35000:]
x_tr = x_train[:35000]
y_val = y_train[35000:]
y_tr = y_train[:35000]
use_DML = False
if use_DML:
x_val = x_train[5000:6000]
x_tr = x_train[:5000]
y_val = y_train[5000:6000]
y_tr = y_train[:20000]
x_tr, x_val = self.PCA(x_tr, x_val, 64)
lmnn = LMNN(k=15, learn_rate=1e-6, min_iter=50, max_iter=100)
lmnn.fit(x_tr.numpy(), y_tr.numpy())
M = lmnn.get_mahalanobis_matrix()
M = torch.tensor(M).float()
n, d = x_val.shape
m = x_tr.shape[0]
x0 = x_tr.unsqueeze(1).expand(-1, n,
-1).contiguous().view(-1, d)
x1 = x_val.unsqueeze(0).expand(m, -1,
-1).contiguous().view(-1, d)
x = x0 - x1
dist0 = torch.mm(M, x.t().contiguous())
dists = dist0.t().contiguous() * x
dist = dists.sum(1).view(m, n)
else:
x_tr, x_val = self.PCA(x_tr, x_val, 500)
dist = self.get_dist(x_tr, x_val).cpu()
for k in [1, 3, 5, 8, 10, 15, 20, 25, 30]:
y_pred = self.predict(dist, y_tr, k)
acc = (y_pred == y_val).sum().float().numpy() / y_val.shape[0]
print("K=", k, " acc=", acc)
torch.cuda.empty_cache()
def get_model(config):
if config is None:
return None
if 'model_kwargs' not in config:
model_kwargs = dict()
else:
model_kwargs = config['model_kwargs']
if config['model'] == 'svm':
model = svm.SVC(**model_kwargs)
elif config['model'] == 'rdf':
model = RandomForestClassifier(**model_kwargs)
elif config['model'] == 'adaboost':
if 'sub_model' in config:
base_estimator = get_model(config['sub_model'])
else:
base_estimator = None
model = AdaBoostClassifier(base_estimator=base_estimator, **model_kwargs)
elif config['model'] == 'gradient_boost':
model = GradientBoostingClassifier(**model_kwargs)
elif config['model'] == 'gaussion_bayes':
model = naive_bayes.GaussianNB(**model_kwargs)
elif config['model'] == 'mlp':
model = MLPClassifier(**model_kwargs)
elif config['model'] == 'k_neighbors':
model = KNeighborsClassifier(**model_kwargs)
elif config['model'] == 'decision_tree':
model = DecisionTreeClassifier(**model_kwargs)
elif config['model'] == 'voting':
sub_models = []
for sub_model in config['sub_model']:
sub_models.append([sub_model['model_name'], get_model(sub_model)])
model = VotingClassifier(sub_models, **model_kwargs)
elif config['model'] == 'stacking':
final_estimator = get_model(config.get('final_model', None))
sub_models = []
for sub_model in config['sub_model']:
sub_models.append((sub_model['model_name'], get_model(sub_model)))
model = StackingClassifier(estimators=sub_models, final_estimator=final_estimator, **model_kwargs)
elif config['model'] == 'bagging':
base_estimator = get_model(config.get('sub_model', None))
model = BaggingClassifier(base_estimator=base_estimator, **model_kwargs)
elif config['model'] in ['lfda', 'lmnn', 'mmc']:
if config['model'] == 'lfda':
metric_learner = LFDA(**model_kwargs)
elif config['model'] == 'lmnn':
metric_learner = LMNN(**model_kwargs)
elif config['model'] == 'mmc':
metric_learner = MMC_Supervised(**model_kwargs)
else:
raise AttributeError
if 'final_model' in config:
final_model = get_model(config['final_model'])
else:
final_model = KNeighborsClassifier()
model = Pipeline([('metric', metric_learner), ('final', final_model)])
else:
raise AttributeError('unrecognized model %s' % config['model'])
return model
def LMNN_Metric(datamatrix, datalabel):
Dis_Matrix = np.zeros((len(datalabel), len(datalabel)))
lmnn = LMNN(k=5, learn_rate=1e-6)
lmnn.fit(datamatrix, datalabel)
metric_func = lmnn.get_metric()
for i in range(len(datalabel)):
for j in range(len(datalabel)):
Dis_Matrix[i, j] = metric_func(datamatrix[i], datamatrix[j])
return Dis_Matrix
def draw_knn_with_lmnn(k, metric):
names = ['x', 'y', 'color']
df = pd.DataFrame(mapped_colors, columns=names)
# print(df.head())
X = np.array(df.ix[:, 0:2])
y = np.array(df['color'])
lmnn = LMNN(k=5, learn_rate=1e-6)
lmnn.fit(X, y)
X_lmnn = lmnn.transform()
X = X_lmnn
# print(X)
X_train, X_test, y_train, y_test = train_test_split(X,
y,
test_size=0.33,
random_state=42)
if metric == 'mahalanobis':
knn = KNeighborsClassifier(
n_neighbors=k,
metric=metric,
metric_params={'V': np.cov(np.transpose(X))})
else:
knn = KNeighborsClassifier(n_neighbors=k, metric=metric)
knn.fit(X_train, y_train)
pred = knn.predict(X_test)
err = 1 - accuracy_score(y_test, pred)
print('\nThe error is ' + str(err * 100))
h = .02
cmap_light = ListedColormap(['#FFAAAA', '#AAFFAA', '#AAAAFF'])
cmap_bold = ListedColormap(['#FF0000', '#00FF00', '#0000FF'])
x_min, x_max = X[:, 0].min() - 1, X[:, 0].max() + 1
y_min, y_max = X[:, 1].min() - 1, X[:, 1].max() + 1
xx, yy = np.meshgrid(np.arange(x_min, x_max, h),
np.arange(y_min, y_max, h))
Z = knn.predict(np.c_[xx.ravel(), yy.ravel()])
Z = Z.reshape(xx.shape)
plt.figure()
plt.pcolormesh(xx, yy, Z, cmap=cmap_light)
plt.scatter(X[:, 0], X[:, 1], c=y, cmap=cmap_bold, edgecolor='k', s=20)
plt.xlim(xx.min(), xx.max())
plt.ylim(yy.min(), yy.max())
plt.title("3-Class classification (k = %i)" % k)
def __init__(self, lmnn=False, max_iter=1000, lm_num=200):
# self.clf = LabelPropagation(kernel='knn',max_iter=1000,n_jobs=10,n_neighbors=25)
self.clf = LabelSpreading(kernel='knn',
n_neighbors=25,
max_iter=max_iter,
alpha=0.2,
n_jobs=-1)
self.lmnn = lmnn
self.lm_num = lm_num
if lmnn:
self.ml = LMNN(use_pca=False, max_iter=2000)
def test_no_twice_same_objective(capsys):
# test that the objective function never has twice the same value
# see https://github.com/scikit-learn-contrib/metric-learn/issues/88
X, y = make_classification(random_state=0)
lmnn = LMNN(verbose=True)
lmnn.fit(X, y)
out, _ = capsys.readouterr()
lines = re.split("\n+", out)
# we get only objectives from each line:
# the regexp matches a float that follows an integer (the iteration
# number), and which is followed by a (signed) float (delta obj). It
# matches for instance:
# 3 **1113.7665747189938** -3.182774197440267 46431.0200999999999998e-06
objectives = [re.search(r"\d* (?:(\d*.\d*))[ | -]\d*.\d*", s)
for s in lines]
objectives = [match.group(1) for match in objectives if match is not None]
# we remove the last element because it can be equal to the penultimate
# if the last gradient update is null
assert len(objectives[:-1]) == len(set(objectives[:-1]))
def test_loss_grad_lbfgs(self):
"""Test gradient of loss function
Assert that the gradient is almost equal to its finite differences
approximation.
"""
rng = np.random.RandomState(42)
X, y = make_classification(random_state=rng)
L = rng.randn(rng.randint(1, X.shape[1] + 1), X.shape[1])
lmnn = LMNN()
k = lmnn.k
reg = lmnn.regularization
X, y = lmnn._prepare_inputs(X, y, dtype=float,
ensure_min_samples=2)
num_pts, n_components = X.shape
unique_labels, label_inds = np.unique(y, return_inverse=True)
lmnn.labels_ = np.arange(len(unique_labels))
lmnn.components_ = np.eye(n_components)
target_neighbors = lmnn._select_targets(X, label_inds)
# sum outer products
dfG = _sum_outer_products(X, target_neighbors.flatten(),
np.repeat(np.arange(X.shape[0]), k))
# initialize L
def loss_grad(flat_L):
return lmnn._loss_grad(X, flat_L.reshape(-1, X.shape[1]), dfG,
k, reg, target_neighbors, label_inds)
def fun(x):
return loss_grad(x)[1]
def grad(x):
return loss_grad(x)[0].ravel()
# compute relative error
epsilon = np.sqrt(np.finfo(float).eps)
rel_diff = (check_grad(fun, grad, L.ravel()) /
np.linalg.norm(approx_fprime(L.ravel(), fun, epsilon)))
np.testing.assert_almost_equal(rel_diff, 0., decimal=5)
def baseline_model(X_train, y_train, X_test, y_test):
#dimension reduction
feature_selection = LinearSVC(C=1, penalty="l1", dual=False)
X_train_reduced = feature_selection.fit_transform(X_train, y_train)
X_test_reduced = feature_selection.transform(X_test)
#metrics learning
ml = LMNN(k=4, min_iter=50, max_iter=1000, learn_rate=1e-7)
ml.fit(X_train_reduced, y_train)
X_train_new = ml.transform(X_train_reduced)
X_test_new = ml.transform(X_test_reduced)
neigh = KNeighborsClassifier(n_neighbors=4)
neigh.fit(X_train_new, y_train)
predicted = neigh.predict(X_test_new)
#pickle.dump(ml, open('dist_metrics', 'w'))
return predicted
def __init__(self, init_with=0):
"""Initialize the transformation pipleine, with an optional list of transformation
modes. init_with can be any bitwise-or'd combination of the TRANSFORM_* modes."""
self.pipeline = []
self.supervised_pipeline = []
self.mink = 4
if init_with & self.TRANSFORM_SCALE:
self.add_standard_scaler()
if init_with & self.TRANSFORM_PCA:
print "Adding PCA"
self.add_pca()
if init_with & self.TRANSFORM_RANDOM_PCA:
print "Adding RandomPCA"
self.add_random_pca()
if init_with & self.TRANSFORM_KERNEL_PCA:
self.add_kernel_pca()
if init_with & self.TRANSFORM_LMNN:
self.supervised_pipeline.append(LMNN(k=self.mink))
if init_with & self.TRANSFORM_NCA:
self.supervised_pipeline.append(NCA())
def get_dist_func(
data: Array[np.float64], target: Array[np.float64]
) -> Callable[[Callable[[np.float64, np.float64], np.float64], np.int, np.int],
np.float64]:
"""
Get function that returns distances between examples in learned space.
Args:
data : Array[np.float64] - training data_trans
target : int - target variable values (classes of training examples)
Returns:
Callable[[Callable[[np.float64, np.float64], np.float64], np.int, np.int], np.float64] -- higher
order function that takes a matric function and returns a function that takes two indices of examples
and returns distance between examples in learned metric space.
"""
# Get transformed data.
data_trans: Array[np.float64] = LMNN().fit_transform(
StandardScaler().fit_transform(data), target)
# Computing distance:
def dist_func_res(metric: Callable[[np.float64, np.float64], np.float64],
i1: np.int, i2: np.int) -> np.float64:
"""
distance function that takes indices of examples in training set and returns distance
in learned space using specified distance metric.
Args:
i1 : int - index of first training example
i2 : int - index of second training example
Returns:
np.float64 - distance in learned metric space using specified metric
between specified training examples.
"""
# Compute distance in learned metric space using specified metric.
return metric(data_trans[i1, :], data_trans[i2, :])
return dist_func_res # Return distance function.
def lmnn_fit(X_train, Y_train, X_test, Y_test, color_map):
lmnn = LMNN(init='pca',
k=3,
learn_rate=5e-4,
max_iter=500000,
regularization=0.2)
lmnn.fit(X_train, Y_train)
X_train_transformed = lmnn.transform(X_train)
if (X_train.shape[1] == 2):
plt.figure()
plt.scatter(X_train_transformed[:, 0],
X_train_transformed[:, 1],
c=color_map[Y_train],
s=2)
plt.savefig("after_lmnn_transform_train.png", dpi=300)
X_test_transformed = lmnn.transform(X_test)
if (X_test.shape[1] == 2):
plt.figure()
plt.scatter(X_test_transformed[:, 0],
X_test_transformed[:, 1],
c=color_map[Y_test],
s=2)
plt.savefig("after_lmnn_transform_test.png", dpi=300)
return (X_train_transformed, X_test_transformed)
# #### Joint: Post-processing (Clustering) ################
mode = np.load('posterior_mode_multiply.npz')['joint_posterior_mode']
mode = mode[renewed, :]
print(mode.shape)
from abcpy.statistics import Identity
stat = Identity(degree=4, cross=True)
mode = stat.statistics([[mode[i, :]] for i in range(mode.shape[0])])
print(mode.shape)
label = [1 for i in range(16)] + [2 for i in range(16)
] + [3 for i in range(16)]
print(label)
from metric_learn import LMNN
metric = LMNN(init='auto',
k=6,
min_iter=10000,
max_iter=50000,
convergence_tol=1e-6,
learn_rate=1e-10,
regularization=.5,
n_components=2)
metric.fit(mode, label)
L = metric.components_
np.savez('L_all_3_cross_parameters.npz', L=L)
L = np.load('L_all_3_cross_parameters.npz')['L']
mode_lmnn = mode.dot(L.T)
print(mode_lmnn.shape)
import pylab as plt
plt.figure()
plt.plot(mode_lmnn[:16, 0],
mode_lmnn[:16, 1],
def test_lmnn(self): lmnn = LMNN(k=5, learn_rate=1e-6, verbose=False) lmnn.fit(self.X, self.y) L = lmnn.components_ assert_array_almost_equal(L.T.dot(L), lmnn.get_mahalanobis_matrix())
loader = DataLoader(dataset=trainset,
batch_size=args.batch_size,
shuffle=False,
num_workers=args.num_workers,
pin_memory=True)
embs = []
labels = []
with torch.no_grad():
for i, batch in tqdm(enumerate(loader, 1),
total=len(loader),
desc='embedding'):
if torch.cuda.is_available():
data, label = batch[0].cuda(), batch[1]
else:
data, label = batch
data_emb = model.encoder(data)
embs.append(data_emb)
labels.append(label)
embs = torch.cat(embs).cpu().numpy()
labels = torch.cat(labels).numpy()
lmnn = LMNN(verbose=True)
print('fitting data....')
lmnn.fit(embs, labels)
print('fitting data finished.')
directory = 'checkpoints/lmnn/'
if not osp.exists(directory):
os.makedirs(directory)
joblib.dump(lmnn, osp.join(directory, '%s.pkl' % args.filename))
def test_lmnn(self):
check_estimator(LMNN())
def lmnn_apply(x_input, y_input, k_input):
y_classified = classify_y(y_input)
lmnn = LMNN(k=k_input, learn_rate=1e-6)
x_output = lmnn.fit_transform(x_input, y_classified)
return x_output
def main(params):
initialize_results_dir(params.get('results_dir'))
backup_params(params, params.get('results_dir'))
print('>>> loading data...')
X_train, y_train, X_test, y_test = LoaderFactory().create(
name=params.get('dataset'),
root=params.get('dataset_dir'),
random=True,
seed=params.getint('split_seed'))()
print('<<< data loaded')
print('>>> computing psd matrix...')
if params.get('algorithm') == 'identity':
psd_matrix = np.identity(X_train.shape[1], dtype=X_train.dtype)
elif params.get('algorithm') == 'nca':
nca = NCA(init='auto',
verbose=True,
random_state=params.getint('algorithm_seed'))
nca.fit(X_train, y_train)
psd_matrix = nca.get_mahalanobis_matrix()
elif params.get('algorithm') == 'lmnn':
lmnn = LMNN(init='auto',
verbose=True,
random_state=params.getint('algorithm_seed'))
lmnn.fit(X_train, y_train)
psd_matrix = lmnn.get_mahalanobis_matrix()
elif params.get('algorithm') == 'itml':
itml = ITML_Supervised(verbose=True,
random_state=params.getint('algorithm_seed'))
itml.fit(X_train, y_train)
psd_matrix = itml.get_mahalanobis_matrix()
elif params.get('algorithm') == 'lfda':
lfda = LFDA()
lfda.fit(X_train, y_train)
psd_matrix = lfda.get_mahalanobis_matrix()
elif params.get('algorithm') == 'arml':
learner = TripleLearner(
optimizer=params.get('optimizer'),
optimizer_params={
'lr': params.getfloat('lr'),
'momentum': params.getfloat('momentum'),
'weight_decay': params.getfloat('weight_decay'),
},
criterion=params.get('criterion'),
criterion_params={'calibration': params.getfloat('calibration')},
n_epochs=params.getint('n_epochs'),
batch_size=params.getint('batch_size'),
random_initialization=params.getboolean('random_initialization',
fallback=False),
update_triple=params.getboolean('update_triple', fallback=False),
device=params.get('device'),
seed=params.getint('learner_seed'))
psd_matrix = learner(X_train,
y_train,
n_candidate_mins=params.getint('n_candidate_mins',
fallback=1))
else:
raise Exception('unsupported algorithm')
print('<<< psd matrix got')
np.savetxt(os.path.join(params.get('results_dir'), 'psd_matrix.txt'),
psd_matrix)
class MLPipe:
pipe = Pipeline([('scaling', StandardScaler()),
('feature_selection',
SelectFromModel(RandomForestClassifier(n_estimators=100, random_state=42), threshold='median')),
('metric_learning', None),
('classifier', SVC())])
save_path = './titanic/pipe_{}.bin'
feature_selection_param_grid = {
'SVC': [
{
'scaling': [StandardScaler(), None],
'metric_learning': [None, LMNN()],
'feature_selection': [SelectFromModel(RandomForestClassifier(n_estimators=100, random_state=42), threshold='median'), None],
# 'feature_selection__estimator': [RandomForestClassifier(n_estimators=100, random_state=42), SVC(C=1000), KNeighborsClassifier()],
'classifier': [SVC()],
'classifier__kernel': ['rbf'],
'classifier__C': [0.001, 0.01, 0.1, 1, 10, 100],
'classifier__gamma': [0.001, 0.01, 0.1, 1, 10, 100]
},
{
'scaling': [StandardScaler(), None],
'metric_learning': [None, LMNN()],
'feature_selection': [SelectFromModel(RandomForestClassifier(n_estimators=100, random_state=42), threshold='median'), None],
'classifier': [SVC()],
'classifier__kernel': ['linear'],
'classifier__C': [0.001, 0.01, 0.1, 1, 10, 100],
}
],
'rfc': [
{
#'scaling': [StandardScaler(), None],
'scaling': [None],
'metric_learning': [None, LMNN()],
'feature_selection': [SelectFromModel(RandomForestClassifier(n_estimators=100, random_state=42), threshold='median'), None],
'classifier': [RandomForestClassifier()],
'classifier__n_estimators': [10, 25, 50, 75, 100],
'classifier__max_depth': [None, 5, 10, 25],
'classifier__min_samples_split': [5, 10, 15]
}
],
'knn': [
{
'scaling': [StandardScaler(), MinMaxScaler(), None],
'metric_learning': [None, LMNN(), ITML_Supervised(num_constraints=200)],
'feature_selection': [
SelectFromModel(RandomForestClassifier(n_estimators=100, random_state=42), threshold='median'),
None],
'classifier': [KNeighborsClassifier()],
'classifier__n_neighbors': [2, 3, 4, 5],
'classifier__algorithm': ['auto', 'ball_tree', 'kd_tree']
}
],
'dt': [
{
'scaling': [StandardScaler(), MinMaxScaler(), None],
'metric_learning': [None],
'feature_selection': [
SelectFromModel(RandomForestClassifier(n_estimators=100, random_state=42), threshold='median'),
None],
'classifier': [DecisionTreeClassifier()],
'classifier__criterion': ['gini', 'entropy'],
'classifier__max_features': ['auto', 'sqrt', 'log2'],
'classifier__max_depth': [None, 5, 10, 15]
}
],
'gbc': [
{
'scaling': [StandardScaler(), None],
'metric_learning': [None, LMNN()],
'feature_selection': [
SelectFromModel(RandomForestClassifier(n_estimators=100, random_state=42), threshold='median'),
None],
'classifier': [GradientBoostingClassifier()],
'classifier__loss': ['deviance'],
'classifier__learning_rate': [0.1, 1, 10],
'classifier__n_estimators': [10, 50, 100],
'classifier__criterion': ['friedman_mse'],
'classifier__max_features': ['auto'],
'classifier__max_depth': [None, 2, 5, 10, 15, 25],
'classifier__min_samples_split': [5, 10, 15]
}
],
'xgb': [
{
'scaling': [StandardScaler()],#, None],
'metric_learning': [None],#, LMNN(), ITML_Supervised(num_constraints=200)],
'feature_selection': [None],
'classifier': [xgb.XGBClassifier()],
'classifier__n_estimators': [500, 1000, 2000],
'classifier__max_depth': [4, 6, 8, 10],
'classifier__min_child_weight': [1, 2, 3],
'classifier__gamma': [0.4, 0.6, 0.8, 0.9, 1],
'classifier__subsample': [0.4, 0.6, 0.8, 1.0],
'classifier__colsample_bytree': [0.4, 0.6, 0.8, 1.0]
}
]
}
def __init__(self, model_name: str):
print('model_name: %s' % model_name)
self._model_name = model_name
self._param_grid = MLPipe.feature_selection_param_grid[model_name]
self._save_path = MLPipe.save_path.format(model_name)
self._save_best_path = self._save_path + '-best'
self._model = None
self._pipe = MLPipe.pipe
def fit_model(self, train_X: list, train_y: list):
model = load_model(self._save_path)
if not model:
# create model, if not loading file
grid_search = GridSearchCV(self._pipe, self._param_grid, cv=5, n_jobs=-1)
grid_search.fit(train_X, train_y)
save_model(self._save_path, grid_search)
model = grid_search
print('best score: {:.2f}'.format(model.best_score_))
print('best estimator \n{}'.format(model.best_estimator_))
self._model = model.best_estimator_
def predict(self, test_X: list) -> list:
test_y = self._model.predict(test_X).astype(int)
return test_y
def get_model(self) -> dict:
return self._model
def save_best_model(self):
save_model(self._save_best_path, self._model)
def load_best_model(self):
self._model = load_model(self._save_best_path)
def get_cv_failure_data(self, train_X: list, train_y: list):
ret_index = np.array([])
evaluate_model = self._model
kf = KFold(n_splits=5)
for train_index, test_index in kf.split(train_X):
evaluate_model.fit(train_X[train_index], train_y[train_index])
evaluate_y = evaluate_model.predict(train_X[test_index])
correct_eval_y = train_y[test_index]
ret_index = np.concatenate((ret_index, np.array(test_index)[evaluate_y != train_y[test_index]]))
return list(ret_index.astype(int))
print('Data Preparation Done', '\n')
#print(FSTrainData.max(axis=0) - FSTrainData.min(axis=0))
#print(len(FSTrainData[0]))
#print(len(FSTestData[0]))
#print(len(FSTestData))
#print(len(TestData))
#print(TrainData)
#print(type(TrainData))
#print(TrainLabels)
#print(type(TrainLabels))
if Method == 'LMNN':
print("Method: LMNN", '\n')
lmnn = LMNN(k=3, learn_rate=1e-6, verbose=False)
x = lmnn.fit(FSTrainData, TrainLabels)
TFSTestData = x.transform(FSTestData)
print('Transformation Done', '\n')
elif Method == 'COV':
print("Method: COV", '\n')
cov = Covariance().fit(FSTrainData)
TFSTestData = cov.transform(FSTestData)
print('Transformation Done', '\n')
elif Method == 'ITML':
print("Method: ITML", '\n')
itml = ITML_Supervised(num_constraints=200, A0=None)
x = itml.fit(FSTrainData, TrainLabels)
TFSTestData = x.transform(FSTestData)