def test_quantile_boundaries_detection(self):
"""...Test boundaries detection for method `quantile`
"""
n_cuts = 3
binarizer = FeaturesBinarizer(method='quantile',
n_cuts=n_cuts,
detect_column_type="column_names",
remove_first=False)
# only for the two continuous features
boundaries_0 = binarizer._get_boundaries(self.columns[0],
self.features[:, 0],
fit=True)
np.testing.assert_array_almost_equal(
boundaries_0,
np.array([-np.inf, 0.009021, 0.271109, 0.473155, np.inf]))
boundaries_1 = binarizer._get_boundaries(self.columns[1],
self.features[:, 1],
fit=True)
np.testing.assert_array_almost_equal(
boundaries_1,
np.array([-np.inf, -0.718759, -0.191478, 0.445833, np.inf]))
def test_linspace_boundaries_detection(self):
"""...Test boundaries detection for method `linspace`
"""
n_cuts = 3
binarizer = FeaturesBinarizer(method='linspace',
n_cuts=n_cuts,
detect_column_type="column_names",
remove_first=False)
# only for the two continuous features
boundaries_0 = binarizer._get_boundaries(self.columns[0],
self.features[:, 0],
fit=True)
np.testing.assert_array_almost_equal(
boundaries_0,
np.array([-np.inf, -0.230951, 0.527193, 1.285336, np.inf]))
boundaries_1 = binarizer._get_boundaries(self.columns[1],
self.features[:, 1],
fit=True)
np.testing.assert_array_almost_equal(
boundaries_1,
np.array([-np.inf, -0.495156, 0.124166, 0.743488, np.inf]))
def test_binarizer_remove_first(self):
"""...Test binarizer fit when remove_first=True
"""
n_cuts = 3
one_hot_encoder = OneHotEncoder(sparse=True)
expected_binarization = one_hot_encoder.fit_transform(
self.default_expected_intervals)
binarizer = FeaturesBinarizer(method='quantile',
n_cuts=n_cuts,
detect_column_type="auto",
remove_first=True)
binarizer.fit(self.features)
binarized_array = binarizer.transform(self.features)
self.assertEqual(binarized_array.__class__, csr.csr_matrix)
expected_binarization_without_first = \
np.delete(expected_binarization.toarray(), [0, 4, 8, 10], 1)
np.testing.assert_array_equal(expected_binarization_without_first,
binarized_array.toarray())
return
def test_CoxRegression_fit(self):
"""...Test CoxRegression fit with different solvers and penalties
"""
raw_features, times, censoring = Test.get_train_data()
coeffs_pen = {
'none':
np.array([
-0.03068462, 0.03940001, 0.16758354, -0.24838003, 0.16940664,
0.9650363, -0.14818724, -0.0802245, -1.52869811, 0.0414509
]),
'l2':
np.array([
-0.02403681, 0.03455527, 0.13470436, -0.21654892, 0.16637723,
0.83125941, -0.08555382, -0.12914753, -1.35294435, 0.02312935
]),
'l1':
np.array([
0., 1.48439371e-02, 1.03806171e-01, -1.57313537e-01,
1.40448847e-01, 8.05306416e-01, -5.41296030e-02,
-1.07753576e-01, -1.37612207e+00, 6.43289248e-05
]),
'elasticnet':
np.array([
0., 0.01011823, 0.10530518, -0.16885214, 0.14373715,
0.82547312, -0.06122141, -0.09479487, -1.39454662, 0.00312597
]),
'tv':
np.array([
0.03017556, 0.03714465, 0.0385349, -0.10169967, 0.15783755,
0.64860815, -0.00617636, -0.22235137, -1.07938977, -0.07181225
]),
'binarsity':
np.array([
0.03794176, -0.04473702, 0.00339763, 0.00339763, -0.16493989,
0.05497996, 0.05497996, 0.05497996, -0.08457476, -0.08457476,
0.0294825, 0.13966702, 0.10251257, 0.02550264, -0.07207419,
-0.05594102, -0.10018038, -0.10018038, 0.10018038, 0.10018038,
-0.47859686, -0.06685181, -0.00850803, 0.55395669, 0.00556327,
-0.00185442, -0.00185442, -0.00185442, 0.26010429, 0.09752455,
-0.17881442, -0.17881442, 0.932516, 0.32095387, -0.49766315,
-0.75580671, 0.0593833, -0.01433773, 0.01077109, -0.05581666
])
}
for penalty in self.penalties:
if penalty == 'binarsity':
# binarize features
n_cuts = 3
binarizer = FeaturesBinarizer(n_cuts=n_cuts)
features = binarizer.fit_transform(raw_features)
else:
features = raw_features
for solver in self.solvers:
solver_kwargs = {
'penalty': penalty,
'tol': 0,
'solver': solver,
'verbose': False,
'max_iter': 10
}
if penalty != 'none':
solver_kwargs['C'] = 50
if penalty == 'binarsity':
solver_kwargs['blocks_start'] = \
binarizer.feature_indices[:-1, ]
solver_kwargs['blocks_length'] = binarizer.n_values
learner = CoxRegression(**solver_kwargs)
learner.fit(features, times, censoring)
np.testing.assert_array_almost_equal(coeffs_pen[penalty],
learner.coeffs,
decimal=1)
def _simulate(self):
# The features matrix already exists, and is created by the
# super class
features = self.features
n_samples, n_features = features.shape
# Simulation of cut-points
n_cut_points = self.n_cut_points
n_cut_points_factor = self.n_cut_points_factor
sparsity = self.sparsity
s = round(n_features * sparsity)
# sparsity index set
S = np.random.choice(n_features, s, replace=False)
if n_cut_points is None:
n_cut_points = np.random.geometric(n_cut_points_factor, n_features)
else:
n_cut_points = np.repeat(n_cut_points, n_features)
cut_points = {}
coeffs_binarized = np.array([])
for j in range(n_features):
feature_j = features[:, j]
quantile_cuts = np.linspace(10, 90, 10)
candidates = np.percentile(feature_j,
quantile_cuts,
interpolation="nearest")
cut_points_j = np.random.choice(candidates,
n_cut_points[j],
replace=False)
cut_points_j = np.sort(cut_points_j)
cut_points_j = np.insert(cut_points_j, 0, -np.inf)
cut_points_j = np.append(cut_points_j, np.inf)
cut_points[str(j)] = cut_points_j
# generate beta star
if j in S:
coeffs_block = np.zeros(n_cut_points[j] + 1)
else:
coeffs_block = np.random.normal(1, .5, n_cut_points[j] + 1)
# make sure 2 consecutive coeffs are different enough
coeffs_block = np.abs(coeffs_block)
coeffs_block[::2] *= -1
# sum-to-zero constraint in each block
coeffs_block = coeffs_block - coeffs_block.mean()
coeffs_binarized = np.append(coeffs_binarized, coeffs_block)
binarizer = FeaturesBinarizer(method='given',
bins_boundaries=cut_points)
binarized_features = binarizer.fit_transform(features)
u = binarized_features.dot(coeffs_binarized)
# Simulation of true times
E = np.random.exponential(scale=1., size=n_samples)
E *= np.exp(-u)
scale = self.scale
shape = self.shape
if self.times_distribution == "weibull":
T = 1. / scale * E**(1. / shape)
else:
# There is not point in this test, but let's do it like that
# since we're likely to implement other distributions
T = 1. / scale * E**(1. / shape)
m = T.mean()
# Simulation of the censoring
c = self.censoring_factor
C = np.random.exponential(scale=c * m, size=n_samples)
# Observed time
self._set("times", np.minimum(T, C).astype(self.dtype))
# Censoring indicator: 1 if it is a time of failure, 0 if censoring.
censoring = (T <= C).astype(np.ushort)
self._set("censoring", censoring)
return self.features, self.times, self.censoring, cut_points, \
coeffs_binarized, S
def fit_and_score(features, features_bin, times, censoring, blocks_start,
blocks_length, boundaries, features_names, idx_train,
idx_test, validation_data, C):
if features_names is None:
features_names = [str(j) for j in range(features.shape[1])]
X_train, X_test = features_bin[idx_train], features_bin[idx_test]
Y_train, Y_test = times[idx_train], times[idx_test]
delta_train, delta_test = censoring[idx_train], censoring[idx_test]
learner = CoxRegression(penalty='binarsity',
tol=1e-5,
verbose=False,
max_iter=100,
step=0.3,
blocks_start=blocks_start,
blocks_length=blocks_length,
warm_start=True)
learner._solver_obj.linesearch = False
learner.C = C
learner.fit(X_train, Y_train, delta_train)
coeffs = learner.coeffs
cut_points_estimates = {}
for j, start in enumerate(blocks_start):
coeffs_j = coeffs[start:start + blocks_length[j]]
all_zeros = not np.any(coeffs_j)
if all_zeros:
cut_points_estimate_j = np.array([-np.inf, np.inf])
else:
groups_j = get_groups(coeffs_j)
jump_j = np.where(groups_j[1:] - groups_j[:-1] != 0)[0] + 1
if jump_j.size == 0:
cut_points_estimate_j = np.array([-np.inf, np.inf])
else:
cut_points_estimate_j = boundaries[features_names[j]][jump_j]
if cut_points_estimate_j[0] != -np.inf:
cut_points_estimate_j = np.insert(cut_points_estimate_j, 0,
-np.inf)
if cut_points_estimate_j[-1] != np.inf:
cut_points_estimate_j = np.append(cut_points_estimate_j,
np.inf)
cut_points_estimates[features_names[j]] = cut_points_estimate_j
binarizer = FeaturesBinarizer(method='given',
bins_boundaries=cut_points_estimates)
binarized_features = binarizer.fit_transform(features)
blocks_start = binarizer.blocks_start
blocks_length = binarizer.blocks_length
X_bin_train = binarized_features[idx_train]
X_bin_test = binarized_features[idx_test]
learner_ = CoxRegression(penalty='binarsity',
tol=1e-5,
verbose=False,
max_iter=100,
step=0.3,
blocks_start=blocks_start,
blocks_length=blocks_length,
warm_start=True,
C=1e10)
learner_._solver_obj.linesearch = False
learner_.fit(X_bin_train, Y_train, delta_train)
score = learner_.score(X_bin_test, Y_test, delta_test)
if validation_data is not None:
X_validation = validation_data[0]
X_bin_validation = binarizer.fit_transform(X_validation)
Y_validation = validation_data[1]
delta_validation = validation_data[2]
score_validation = learner_.score(X_bin_validation, Y_validation,
delta_validation)
else:
score_validation = None
return score, score_validation
def binarsity_reg(X, y, grid_C=np.logspace(-2, 2, 10), C=None, verbose=True):
"""implementation of the linear regression on binarized features with binarsity penalty.
Parameters:
X : pandas dataframe, shape=(1420, 4), columns=[ str(j)+":continuous" for j in range(len(X[0]))], dtype='float'
list of continuous features
y :numpy.ndarray, dtype='float'
list of labels
grid_C : list or numpy.ndarray, dtype='float'
this list of weigths associated to the binarsity penalty from which will be chosen
the final by cross-validation if C=None
C : float (positive) or None
weigth associated to the binarsity penalty
verbose: Boolean
If True, prints additional info
Returns :
cut_points_estimates : dict, length = dimension of observation space
final_coeffs : numpy.ndarray
blocks_start : numpy.ndarray
all_groups : list
coeffs : numpy.ndarray
regr.C : 'float'
"""
t0 = time.clock()
if verbose == True:
print("binarsity regression number of observations :", len(X))
n_cuts = 50
binarizer = FeaturesBinarizer(n_cuts=n_cuts,
detect_column_type="column_names")
X_bin = binarizer.fit_transform(X)
features_names = [X.columns[j] for j in range(len(X.columns))]
boundaries = binarizer.bins_boundaries
blocks_start = binarizer.blocks_start
blocks_length = binarizer.blocks_length
n_folds = 5
if C == None:
scores_cv = pd.DataFrame(columns=['C', 'scores_mean', 'scores_std'])
for i, C_i in enumerate(grid_C):
scores = compute_score(X,
X_bin,
y,
blocks_start,
blocks_length,
C=C_i,
n_folds=n_folds)
scores = [C_i] + scores
scores_cv.loc[i] = scores
if verbose == True:
print("cross_val scores :")
print(scores_cv.round(3))
idx_min = scores_cv.scores_mean.argmin()
C_best = grid_C[idx_min]
idx_chosen = min([
i for i, j in enumerate(
list(scores_cv.scores_mean <= scores_cv.scores_mean.min() +
scores_cv.scores_std[idx_min])) if j
])
C_chosen = grid_C[idx_chosen]
if verbose == True:
print("C_best :", "%.4g" % C_best)
print("C_chosen :", "%.4g" % C_chosen)
regr = linear_model.LinearRegression(penalty='binarsity',
blocks_start=blocks_start,
blocks_length=blocks_length,
warm_start=True)
if C == None:
regr.C = C_chosen
else:
regr.C = C
if verbose == True:
print("regr.C :", "%.4g" % regr.C)
regr.fit(X_bin, y)
coeffs = regr.weights
# computations of the cut-points
all_groups = list()
cut_points_estimates = {}
for j, start in enumerate(blocks_start):
coeffs_j = coeffs[start:start + blocks_length[j]]
all_zeros = not np.any(coeffs_j)
if all_zeros:
cut_points_estimate_j = np.array([-np.inf, np.inf])
groups_j = np.array(blocks_length[j] * [0])
else:
groups_j = get_groups(coeffs_j)
# print("group"+str(j), groups_j)
jump_j = np.where(groups_j[1:] - groups_j[:-1] != 0)[0] + 1
if jump_j.size == 0:
cut_points_estimate_j = np.array([-np.inf, np.inf])
else:
cut_points_estimate_j = boundaries[features_names[j]][jump_j]
if cut_points_estimate_j[0] != -np.inf:
cut_points_estimate_j = np.insert(cut_points_estimate_j, 0,
-np.inf)
if cut_points_estimate_j[-1] != np.inf:
cut_points_estimate_j = np.append(cut_points_estimate_j,
np.inf)
cut_points_estimates[features_names[j]] = cut_points_estimate_j
if j > 0:
groups_j += max(all_groups) + 1
all_groups += list(groups_j)
if verbose == True:
print("cutpoints :")
for j in range(len(cut_points_estimates)):
print(features_names[j], [
"%.4f" % cut_points_estimates[features_names[j]][i]
for i in range(len(cut_points_estimates[features_names[j]]))
])
# creation of final binarized X data for the computed cutpoints
binarizer2 = FeaturesBinarizer(method='given',
bins_boundaries=cut_points_estimates)
X_bin2 = binarizer2.fit_transform(X)
X_bin2 = np.array(X_bin2.todense())
blocks_start2 = binarizer2.blocks_start
blocks_length2 = binarizer2.blocks_length
X_bin2_train, X_bin2_test, y_train, y_test = train_test_split(
X_bin2, y, test_size=0.2)
# final re-fit:
regr3 = linear_model.LinearRegression(penalty='binarsity',
blocks_start=blocks_start2,
blocks_length=blocks_length2,
warm_start=True)
regr3.C = 1e10
regr3.fit(X_bin2_train, y_train)
if verbose == True:
print(
"R² score of final predictor on train data (80% of total data) :",
"%.4g" % regr3.score(X_bin2_train, y_train))
print("R² score of final predictor on test data (20% of total data) :",
"%.4g" % regr3.score(X_bin2_test, y_test))
final_coeffs = {"intercept": regr3.intercept, "weights": regr3.weights}
t1 = time.clock()
if verbose == True:
print("time elapsed for binarsity regression step:",
"%.4g" % (t1 - t0), "s")
return cut_points_estimates, final_coeffs, blocks_start, all_groups, coeffs, regr.C
c_index_continuous = max(c_index, 1 - c_index)
predictions = estim_proba(marker_cox, lp_train, Y_train, delta_train)
ibs_cox = integrated_brier_score(predictions['values'],
predictions['times'], Y_test,
delta_test, Y_train, delta_train)
# Binacox
print("Train Binacox screening_cox_topP...")
X_train_ = X_train[screening_cox_topP]
X_test_ = X_test[screening_cox_topP]
# binarize feature
n_cuts = 50
binarizer = FeaturesBinarizer(n_cuts=n_cuts)
binarizer.fit(pd.concat([X_train_, X_test_]))
X_bin_train = binarizer.transform(X_train_)
blocks_start = binarizer.blocks_start
blocks_length = binarizer.blocks_length
boundaries = binarizer.boundaries
# fit binacox
learner = CoxRegression(penalty='binarsity',
tol=1e-5,
solver='agd',
verbose=False,
max_iter=100,
step=0.3,
blocks_start=blocks_start,