def _kmeans_train_predict(table, input_cols, n_clusters=3, prediction_col='prediction', init='k-means++', n_init=10,
max_iter=300, tol=1e-4, precompute_distances='auto', seed=None,
n_jobs=1, algorithm='auto', n_samples=None):
inputarr = table[input_cols]
if n_samples is None:
n_samples = len(inputarr)
validate(greater_than_or_equal_to(n_clusters, 1, 'n_clusters'),
greater_than_or_equal_to(n_init, 1, 'n_init'),
greater_than_or_equal_to(max_iter, 1, 'max_iter'),
greater_than(tol, 0.0, 'tol'),
greater_than_or_equal_to(n_jobs, 1, 'n_jobs'),
greater_than_or_equal_to(n_samples, 0, 'n_samples'))
k_means = SKKMeans(n_clusters=n_clusters, init=init, n_init=n_init,
max_iter=max_iter, tol=tol, precompute_distances=precompute_distances,
verbose=0, random_state=seed, copy_x=True, n_jobs=n_jobs, algorithm=algorithm)
k_means.fit(inputarr)
params = {'input_cols':input_cols, 'n_clusters':n_clusters, 'init':init, 'n_init':n_init, 'max_iter':max_iter, 'tol':tol,
'precompute_distances':precompute_distances, 'seed':seed, 'n_jobs':n_jobs, 'algorithm':algorithm}
cluster_centers = k_means.cluster_centers_
labels = k_means.labels_
pca2_model = PCA(n_components=2).fit(inputarr)
pca2 = pca2_model.transform(inputarr)
fig_centers = _kmeans_centers_plot(input_cols, cluster_centers)
fig_samples = _kmeans_samples_plot(table, input_cols, n_samples, cluster_centers)
fig_pca = _kmeans_pca_plot(labels, cluster_centers, pca2_model, pca2)
rb = BrtcReprBuilder()
rb.addMD(strip_margin("""
| ## Kmeans Result
| - Number of iterations run: {n_iter_}.
| - Coordinates of cluster centers
| {fig_cluster_centers}
| - Samples
| {fig_pca}
| {fig_samples}
|
| ### Parameters
| {params}
""".format(n_iter_=k_means.n_iter_, fig_cluster_centers=fig_centers, fig_pca=fig_pca, fig_samples=fig_samples, params=dict2MD(params))))
model = _model_dict('kmeans')
model['model'] = k_means
model['input_cols'] = input_cols
model['_repr_brtc_'] = rb.get()
out_table = table.copy()
out_table[prediction_col] = labels
return {'out_table':out_table, 'model':model}
def _svm_classification_train(table,
feature_cols,
label_col,
c=1.0,
kernel='rbf',
degree=3,
gamma='auto',
coef0=0.0,
shrinking=True,
probability=True,
tol=1e-3,
max_iter=-1,
random_state=None):
validate(greater_than(c, 0.0, 'c'))
_table = table.copy()
_feature_cols = _table[feature_cols]
_label_col = _table[label_col]
if (sklearn_utils.multiclass.type_of_target(_label_col) == 'continuous'):
raise_runtime_error('''Label Column should not be continuous.''')
_svc = svm.SVC(C=c,
kernel=kernel,
degree=degree,
gamma=gamma,
coef0=coef0,
shrinking=shrinking,
probability=probability,
tol=tol,
max_iter=max_iter,
random_state=random_state)
_svc_model = _svc.fit(_feature_cols, _label_col)
get_param = _svc.get_params()
get_param['feature_cols'] = feature_cols
get_param['label_col'] = label_col
rb = BrtcReprBuilder()
rb.addMD(
strip_margin("""
| ## SVM Classification Result
| ### Parameters
| {table_parameter}
""".format(table_parameter=dict2MD(get_param))))
_model = _model_dict('svc_model')
_model['svc_model'] = _svc_model
_model['features'] = feature_cols
_model['_repr_brtc_'] = rb.get()
return {'model': _model}
def _random_sampling(table,
num_or_frac='num',
num=1,
frac=0.5,
replace=False,
seed=None):
validate(greater_than_or_equal_to(num, 1, 'num'))
if num_or_frac == 'num':
out_table = table.sample(n=num, replace=replace, random_state=seed)
else: # 'frac'
out_table = table.sample(frac=frac, replace=replace, random_state=seed)
return {'table': out_table}
def delete_missing_data(table, input_cols, how='any', thresh=None):
_table = table.copy()
if thresh is not None:
validate(greater_than_or_equal_to(thresh, 1, 'thresh'))
thresh = len(input_cols) - thresh + 1
_out_table = _table.dropna(subset=input_cols,
how=how,
axis='index',
thresh=thresh)
return {'out_table': _out_table}
def split_data(table,
train_ratio=7.0,
test_ratio=3.0,
random_state=None,
shuffle=True,
stratify=None):
validate(greater_than(train_ratio, 0.0, 'train_ratio'),
greater_than(test_ratio, 0.0, 'test_ratio'))
ratio = test_ratio / (train_ratio + test_ratio)
out_table_train, out_table_test = sktrain_test_split(
table,
test_size=ratio,
random_state=random_state,
shuffle=shuffle,
stratify=stratify)
return {
'train_table': out_table_train.reset_index(),
'test_table': out_table_test.reset_index()
}
def _profile_table(table,
bins=10,
check_correlation=False,
correlation_threshold=0.9,
correlation_overrides=None):
validate(greater_than_or_equal_to(bins, 1, 'bins'),
greater_than(correlation_threshold, 0.0, 'correlation_threshold'))
rb = BrtcReprBuilder()
profile = pd_profiling.ProfileReport(
table,
bins=bins,
check_correlation=check_correlation,
correlation_threshold=correlation_threshold,
correlation_overrides=correlation_overrides)
rb.addHTML(profile.html)
summary = dict()
summary['_repr_brtc_'] = rb.get()
return {'result': summary}
def _pairplot(table,
x_vars,
y_vars=None,
kind='scatter',
diag_kind='auto',
markers=None,
palette=None,
height=2.5,
aspect=1,
dropna=True,
hue=None):
validate(greater_than(height, 0, 'height'),
greater_than(aspect, 0, 'aspect'))
s_default = plt.rcParams['lines.markersize']**2.
plot_kws = {"s": s_default * height / 6.4}
if y_vars is None:
y_vars = x_vars
if kind == 'scatter':
g = sns.pairplot(table, x_vars=x_vars, y_vars=y_vars, kind=kind, diag_kind=diag_kind, markers=markers, height=height, aspect=aspect, \
dropna=dropna, hue=hue, palette=palette, plot_kws=plot_kws)
else:
scatter_kws = {'scatter_kws': plot_kws}
g = sns.pairplot(table, x_vars=x_vars, y_vars=y_vars, kind=kind, diag_kind=diag_kind, markers=markers, height=height, aspect=aspect, \
dropna=dropna, hue=hue, palette=palette, plot_kws=scatter_kws)
if height <= 2.5:
for ax in g.axes.flatten():
for label in ax.get_xticklabels():
label.set_rotation(90 * (2.5 - height))
rb = BrtcReprBuilder()
rb.addPlt(plt)
plt.clf()
return {'result': {'_repr_brtc_': rb.get()}}
def _decision_tree_regression_train(
table,
feature_cols,
label_col, # fig_size=np.array([6.4, 4.8]),
criterion='mse',
splitter='best',
max_depth=None,
min_samples_split=2,
min_samples_leaf=1,
min_weight_fraction_leaf=0.0,
max_features=None,
random_state=None,
max_leaf_nodes=None,
min_impurity_decrease=0.0,
min_impurity_split=None,
presort=False,
sample_weight=None,
check_input=True,
X_idx_sorted=None):
param_validation_check = [
greater_than_or_equal_to(min_samples_split, 2, 'min_samples_split'),
greater_than_or_equal_to(min_samples_leaf, 1, 'min_samples_leaf'),
greater_than_or_equal_to(min_weight_fraction_leaf, 0.0,
'min_weight_fraction_leaf')
]
if max_depth is not None:
param_validation_check.append(
greater_than_or_equal_to(max_depth, 1, 'max_depth'))
validate(*param_validation_check)
regressor = DecisionTreeRegressor(criterion, splitter, max_depth,
min_samples_split, min_samples_leaf,
min_weight_fraction_leaf, max_features,
random_state, max_leaf_nodes,
min_impurity_decrease,
min_impurity_split, presort)
regressor.fit(table[feature_cols], table[label_col], sample_weight,
check_input, X_idx_sorted)
try:
from sklearn.externals.six import StringIO
from sklearn.tree import export_graphviz
import pydotplus
dot_data = StringIO()
export_graphviz(regressor,
out_file=dot_data,
feature_names=feature_cols,
filled=True,
rounded=True,
special_characters=True)
graph = pydotplus.graph_from_dot_data(dot_data.getvalue())
from brightics.common.repr import png2MD
fig_tree = png2MD(graph.create_png())
except:
fig_tree = "Graphviz is needed to draw a Decision Tree graph. Please download it from http://graphviz.org/download/ and install it to your computer."
# json
model = _model_dict('decision_tree_regression_model')
model['feature_cols'] = feature_cols
model['label_col'] = label_col
feature_importance = regressor.feature_importances_
model['feature_importance'] = feature_importance
model['max_features'] = regressor.max_features_
model['n_features'] = regressor.n_features_
model['n_outputs'] = regressor.n_outputs_
model['tree'] = regressor.tree_
get_param = regressor.get_params()
model['parameters'] = get_param
model['regressor'] = regressor
# report
indices = np.argsort(feature_importance)
sorted_feature_cols = np.array(feature_cols)[indices]
plt.title('Feature Importances')
plt.barh(range(len(indices)),
feature_importance[indices],
color='b',
align='center')
for i, v in enumerate(feature_importance[indices]):
plt.text(v,
i,
" {:.2f}".format(v),
color='b',
va='center',
fontweight='bold')
plt.yticks(range(len(indices)), sorted_feature_cols)
plt.xlabel('Relative Importance')
plt.tight_layout()
fig_feature_importances = plt2MD(plt)
plt.clf()
params = dict2MD(get_param)
feature_importance_df = pd.DataFrame(data=feature_importance,
index=feature_cols).T
# Add tree plot
rb = BrtcReprBuilder()
rb.addMD(
strip_margin("""
| ## Decision Tree Regression Train Result
| ### Decision Tree
| {fig_tree}
|
| ### Feature Importance
| {fig_feature_importances}
|
| ### Parameters
| {list_parameters}
|
""".format(fig_tree=fig_tree,
fig_feature_importances=fig_feature_importances,
list_parameters=params)))
model['_repr_brtc_'] = rb.get()
return {'model': model}
def _xgb_regression_train(table,
feature_cols,
label_col,
max_depth=3,
learning_rate=0.1,
n_estimators=100,
silent=True,
objectibe='reg:linear',
booster='gbtree',
n_jobs=1,
nthread=None,
gamma=0,
min_child_weight=1,
max_delta_step=0,
subsample=1,
colsample_bytree=1,
colsample_bylevel=1,
reg_alpha=0,
reg_lambda=1,
scale_pos_weight=1,
base_score=0.5,
random_state=0,
seed=None,
missing=None,
sample_weight=None,
eval_set=None,
eval_metric=None,
early_stopping_rounds=None,
verbose=True,
xgb_model=None,
sample_weight_eval_set=None):
validate(greater_than_or_equal_to(max_depth, 1, 'max_depth'),
greater_than_or_equal_to(learning_rate, 0.0, 'learning_rate'),
greater_than_or_equal_to(n_estimators, 1, 'n_estimators'))
regressor = XGBRegressor(max_depth, learning_rate, n_estimators, silent,
objectibe, booster, n_jobs, nthread, gamma,
min_child_weight, max_delta_step, subsample,
colsample_bytree, colsample_bylevel, reg_alpha,
reg_lambda, scale_pos_weight, base_score,
random_state, seed, missing)
regressor.fit(table[feature_cols], table[label_col], sample_weight,
eval_set, eval_metric, early_stopping_rounds, verbose,
xgb_model, sample_weight_eval_set)
# json
get_param = regressor.get_params()
feature_importance = regressor.feature_importances_
# plt.rcdefaults()
plot_importance(regressor)
plt.tight_layout()
fig_plot_importance = plt2MD(plt)
plt.clf()
# plt.rcParams['figure.dpi'] = figure_dpi
# plot_tree(regressor)
# fig_plot_tree_UT = plt2MD(plt)
# plt.clf()
# plt.rcParams['figure.dpi'] = figure_dpi
# plot_tree(regressor, rankdir='LR')
# fig_plot_tree_LR = plt2MD(plt)
# plt.rcdefaults()
# plt.clf()
out_model = _model_dict('xgb_regression_model')
out_model['feature_cols'] = feature_cols
out_model['label_col'] = label_col
out_model['parameters'] = get_param
out_model['feature_importance'] = feature_importance
out_model['regressor'] = regressor
out_model['plot_importance'] = fig_plot_importance
# out_model['plot_tree_UT'] = fig_plot_tree_UT
# out_model['plot_tree_LR'] = fig_plot_tree_LR
# out_model['to_graphviz'] = md_to_graphviz
# report
get_param_list = []
get_param_list.append(['feature_cols', feature_cols])
get_param_list.append(['label_col', label_col])
for key, value in get_param.items():
temp = [key, value]
get_param_list.append(temp)
get_param_df = pd.DataFrame(data=get_param_list,
columns=['parameter', 'value'])
feature_importance_df = pd.DataFrame(data=feature_importance,
index=feature_cols).T
rb = BrtcReprBuilder()
rb.addMD(
strip_margin("""
| ## XGB Regression Result
|
| ### Plot Importance
| {image_importance}
|
| ### Feature Importance
| {table_feature_importance}
|
| ### Parameters
| {table_parameter}
|
""".format(image_importance=fig_plot_importance,
table_feature_importance=pandasDF2MD(feature_importance_df, 20),
table_parameter=pandasDF2MD(get_param_df))))
out_model['_repr_brtc_'] = rb.get()
return {'model': out_model}
def _pca(table,
input_cols,
new_column_name='projected_',
n_components=None,
copy=True,
whiten=False,
svd_solver='auto',
tol=0.0,
iterated_power='auto',
random_state=None,
hue=None,
alpha=0,
key_col=None):
num_feature_cols = len(input_cols)
if n_components is None:
n_components = num_feature_cols
validate(greater_than_or_equal_to(n_components, 1, 'n_components'))
pca = PCA(None, copy, whiten, svd_solver, tol, iterated_power,
random_state)
pca_model = pca.fit(table[input_cols])
column_names = []
for i in range(0, n_components):
column_names.append(new_column_name + str(i))
# print(column_names)
pca_result = pca_model.transform(table[input_cols])
out_df = pd.DataFrame(data=pca_result[:, :n_components],
columns=[column_names])
out_df = pd.concat([table.reset_index(drop=True), out_df], axis=1)
out_df.columns = table.columns.values.tolist() + column_names
res_components = pca_model.components_
res_components_df = pd.DataFrame(data=res_components[:n_components],
columns=[input_cols])
res_explained_variance = pca_model.explained_variance_
res_explained_variance_ratio = pca_model.explained_variance_ratio_
res_singular_values = pca_model.singular_values_
res_mean = pca_model.mean_
res_n_components = pca_model.n_components_
res_noise_variance = pca_model.noise_variance_
res_get_param = pca_model.get_params()
res_get_covariance = pca_model.get_covariance()
res_get_precision = pca_model.get_precision()
# visualization
plt.figure()
if n_components == 1:
sns.scatterplot(column_names[0], column_names[0], hue=hue, data=out_df)
plt_two = plt2MD(plt)
plt.clf()
else:
plt_two = _biplot(
0,
1,
pc_columns=column_names,
columns=input_cols,
singular_values=res_singular_values,
components=res_components,
explained_variance_ratio=res_explained_variance_ratio,
alpha=alpha,
hue=hue,
data=out_df,
ax=plt.gca(),
key_col=key_col)
plt.figure()
fig_scree = _screeplot(res_explained_variance,
res_explained_variance_ratio, n_components)
table_explained_variance = pd.DataFrame(res_explained_variance,
columns=['explained_variance'])
table_explained_variance[
'explained_variance_ratio'] = res_explained_variance_ratio
table_explained_variance[
'cum_explained_variance_ratio'] = res_explained_variance_ratio.cumsum(
)
rb = BrtcReprBuilder()
rb.addMD(
strip_margin("""
| ## PCA Result
| ### Plot
| {image1}
|
| ### Explained Variance
| {fig_scree}
| {table_explained_variance}
|
| ### Components
| {table2}
|
| ### Parameters
| {parameter1}
""".format(image1=plt_two,
fig_scree=fig_scree,
table_explained_variance=pandasDF2MD(table_explained_variance),
parameter1=dict2MD(res_get_param),
table2=pandasDF2MD(res_components_df))))
model = _model_dict('pca')
model['components'] = res_components
model['explained_variance'] = res_explained_variance
model['explained_variance_ratio'] = res_explained_variance_ratio
model['singular_values'] = res_singular_values
model['mean'] = res_mean
model['n_components'] = res_n_components
model['noise_variance'] = res_noise_variance
model['parameters'] = res_get_param
model['covariance'] = res_get_covariance
model['precision'] = res_get_precision
model['_repr_brtc_'] = rb.get()
model['pca_model'] = pca_model
model['input_cols'] = input_cols
return {'out_table': out_df, 'model': model}
def _kmeans_silhouette_train_predict(table, input_cols, n_clusters_list=range(2, 10), prediction_col='prediction', init='k-means++', n_init=10,
max_iter=300, tol=1e-4, precompute_distances='auto', seed=None,
n_jobs=1, algorithm='auto', n_samples=None):
if n_samples is None:
n_samples = len(table)
inputarr = table[input_cols]
validate(all_elements_greater_than(n_clusters_list, 1, 'n_clusters_list'),
greater_than_or_equal_to(n_init, 1, 'n_init'),
greater_than_or_equal_to(max_iter, 1, 'max_iter'),
greater_than(tol, 0.0, 'tol'),
greater_than_or_equal_to(n_jobs, 1, 'n_jobs'),
greater_than_or_equal_to(n_samples, 0, 'n_samples'))
pca2_model = PCA(n_components=2).fit(inputarr)
pca2 = pca2_model.transform(inputarr)
silhouette_list = []
silouette_samples_list = []
models = []
centers_list = []
images = []
for k in n_clusters_list:
k_means = SKKMeans(n_clusters=k, init=init, n_init=n_init,
max_iter=max_iter, tol=tol, precompute_distances=precompute_distances,
verbose=0, random_state=seed, copy_x=True, n_jobs=n_jobs, algorithm=algorithm)
k_means.fit(inputarr)
models.append(k_means)
predict = k_means.labels_
centersk = k_means.cluster_centers_
centers_list.append(centersk)
score = silhouette_score(inputarr, predict)
silhouette_list.append(score)
samples = silhouette_samples(inputarr, predict)
silouette_samples_list.append(samples)
pca2_centers = pca2_model.transform(centersk)
_, (ax1, ax2) = plt.subplots(1, 2)
colors = cm.nipy_spectral(np.arange(k).astype(float) / k)
y_lower = 0
for i, color in zip(range(k), colors):
si = samples[predict == i]
si.sort()
sizei = si.shape[0]
y_upper = y_lower + sizei
ax1.fill_betweenx(np.arange(y_lower, y_upper),
0, si,
facecolor=color, edgecolor=color, alpha=0.7)
y_lower = y_upper
ax2.scatter(pca2[:, 0][predict == i], pca2[:, 1][predict == i], color=color)
ax1.axvline(x=score, color="red")
ax2.scatter(pca2_centers[:, 0], pca2_centers[:, 1], marker='x', edgecolors=1, s=200, color=colors)
imagek = plt2MD(plt)
plt.clf()
images.append(imagek)
argmax = np.argmax(silhouette_list)
best_k = n_clusters_list[argmax]
best_model = models[argmax]
predict = best_model.predict(inputarr)
best_centers = best_model.cluster_centers_
best_labels = best_model.labels_
fig_centers = _kmeans_centers_plot(input_cols, best_centers)
fig_samples = _kmeans_samples_plot(table, input_cols, n_samples, best_centers)
fig_pca = _kmeans_pca_plot(predict, best_centers, pca2_model, pca2)
x_clusters = range(len(n_clusters_list))
plt.xticks(x_clusters, n_clusters_list)
plt.plot(x_clusters, silhouette_list, '.-')
fig_silhouette = plt2MD(plt)
plt.clf()
rb = BrtcReprBuilder()
rb.addMD(strip_margin("""
| ## Kmeans Silhouette Result
| - silloutte metrics:
| {fig_silhouette}
| - best K: {best_k}
| - best centers:
| {fig_pca}
| {fig_centers}
| {fig_samples}
|
""".format(fig_silhouette=fig_silhouette, best_k=best_k, fig_pca=fig_pca, fig_centers=fig_centers, fig_samples=fig_samples)))
for k, image in zip(n_clusters_list, images):
rb.addMD(strip_margin("""
| ### k = {k}
| {image}
|
""".format(k=k, image=image)))
model = _model_dict('kmeans_silhouette')
model['best_k'] = best_k
model['best_centers'] = best_centers
model['best_model'] = best_model
model['input_cols'] = input_cols
model['_repr_brtc_'] = rb.get()
out_table = table.copy()
out_table[prediction_col] = predict
return {'out_table':out_table, 'model':model}
def _correlation(table, vars, method='pearson', height=2.5, corr_prec=2):
validate(greater_than(height, 0, 'height'),
greater_than_or_equal_to(corr_prec, 1, 'corr_prec'))
size = len(vars)
s_default = plt.rcParams['lines.markersize']**2.
scatter_kws = {"s": s_default * height / 6.4}
corr_arr = np.ones((size, size)) # TODO variable name dict
p_arr = np.zeros((size, size))
for i in range(size):
for j in range(i):
if method == 'pearson':
r, p = stats.pearsonr(table[vars[i]], table[vars[j]])
elif method == 'spearman':
r, p = stats.spearmanr(table[vars[i]], table[vars[j]])
elif method == 'kendal':
r, p = stats.kendalltau(table[vars[i]], table[vars[j]])
corr_arr[i][j] = r
p_arr[i][j] = p
for i in range(size):
for j in range(i, size):
corr_arr[i][j] = corr_arr[j][i]
p_arr[i][j] = p_arr[j][i]
def corr(x, y, **kwargs):
if kwargs['method'] == 'pearson':
r, p = stats.pearsonr(x, y)
elif kwargs['method'] == 'spearman':
r, p = stats.spearmanr(x, y)
elif kwargs['method'] == 'kendal':
r, p = stats.kendalltau(x, y)
p_stars = ''
if p <= 0.05:
p_stars = '*'
if p <= 0.01:
p_stars = '**'
if p <= 0.001:
p_stars = '***'
corr_text = '{:.{prec}f}'.format(r, prec=corr_prec)
print(type(corr_prec))
font_size = abs(r) * 15 * 2 / corr_prec + 5
ax = plt.gca()
ax.annotate(corr_text, [
.5,
.5,
],
xycoords="axes fraction",
ha='center',
va='center',
fontsize=font_size * height)
ax.annotate(p_stars,
xy=(0.65, 0.6),
xycoords=ax.transAxes,
color='red',
fontsize=17 * height)
g = sns.PairGrid(table, vars=vars, height=height)
g.map_diag(sns.distplot)
if method == 'pearson':
g.map_lower(sns.regplot, scatter_kws=scatter_kws)
else:
g.map_lower(sns.regplot, lowess=True, scatter_kws=scatter_kws)
g.map_upper(corr, method=method)
rb = BrtcReprBuilder()
rb.addPlt(plt)
plt.clf()
params = {'vars': vars, 'method': method, 'height': height}
res = dict()
res['params'] = params
res['corr'] = corr_arr
res['pvalue'] = p_arr
res['_repr_brtc_'] = rb.get()
return {'result': res}