class f_regressionFWEPrim(primitive):
def __init__(self, random_state=0):
super(f_regressionFWEPrim, self).__init__(name='f_regressionFWE')
self.id = 39
self.PCA_LAPACK_Prim = []
self.type = 'feature selection'
self.description = "Select the p-values corresponding to Family-wise error rate with F-value between label/feature for regression tasks."
self.hyperparams_run = {'default': True}
self.selector = None
self.accept_type = 'c_r'
def can_accept(self, data):
return self.can_accept_c(data, 'Regression')
def is_needed(self, data):
if data['X'].shape[1] < 3:
return False
return True
def fit(self, data):
data = handle_data(data)
self.selector = SelectFwe(f_regression, alpha=0.05)
self.selector.fit(data['X'], data['Y'])
def produce(self, data):
output = handle_data(data)
cols = list(output['X'].columns)
try:
mask = self.selector.get_support(indices=False)
final_cols = list(compress(cols, mask))
output['X'] = pd.DataFrame(self.selector.transform(output['X']), columns=final_cols)
except Exception as e:
print(e)
final_output = {0: output}
return final_output
def main(args):
if args.train_dir is None:
# args.train_dir = '/a/fr-05/vol/protein/danofer/ProtFeat/feat_extract/chap/train/'
#args.train_dir = '/cs/prt3/danofer/ProtFeat/feat_extract/test_seq/NP/SPCleaved_NP-70+NEG-30_Big-V3/'
# args.train_dir = r'D:\SkyDrive\Dropbox\bioInf_lab\AA_info\CODE\feat_extract\test_seq\NP\SPCleaved_NP-70+NEG-30_Big-V3'
# args.train_dir = r'E:\Dropbox\Dropbox\bioInf_lab\AA_info\fastas\NP\SP_Cleaved+NP+Neg_Big'
args.train_dir = r'E:\Dropbox\Dropbox\bioInf_lab\AA_info\fastas\Benchmarks\Thermophiles'
print("Using default train_dir: %s" % args.train_dir)
pandas.set_option('display.max_columns', 10)
pandas.set_option('display.max_rows', 4)
# mpl.rc('title', labelsize=6)
mpl.rc('ytick', labelsize=7)
mpl.rc('xtick', labelsize=4)
os.chdir(args.train_dir)
dataName = 'Neuropeptides'
df = pandas.read_csv('trainingSetFeatures.csv')
feature_cols = [
col for col in df.columns
if col not in ['classname', 'Id', 'proteinname']
]
feature_cols = numpy.array(feature_cols)
X = df[feature_cols].values
y = df.classname.values
le = LabelEncoder()
y = le.fit_transform(y)
"Initial feature selection trimming"
print(X.shape)
Fwe = SelectFwe(alpha=0.01).fit(X, y)
X = Fwe.transform(X)
print("F-test -> ", X.shape)
feature_cols = feature_cols[Fwe.get_support()]
'''
FeatSelection_SVM = True
if FeatSelection_SVM == True:
svc_L1 = LinearSVC(C=50, penalty="l1", dual=False,class_weight='auto').fit(X, y)
X = svc_L1.transform(X, y)
print ("L1 SVM Transformed X:",X_L1.shape)
feature_cols=feature_cols[list(set(np.where(svc_L1.coef_ != 0)[-1]))]
'''
k = SelectKBest(k=255).fit(X, y)
X = k.transform(X)
feature_cols = feature_cols[k.get_support()]
param_dist = {
"max_depth": [6, 9, None],
"max_features": ['auto', 0.4],
"min_samples_leaf": [1, 2, 3],
"bootstrap": [True, False],
'min_samples_split': [2, 3],
"criterion": ["gini"],
"n_estimators": [100],
"n_jobs": [-1]
}
rf = RandomForestClassifierWithCoef(max_depth=7,
min_samples_split=1,
min_samples_leaf=2,
n_estimators=50,
n_jobs=2,
max_features="auto")
"WARNING! F1 Score as implemented by Default in binary classification (two classes) gives the score for 1 class."
scores = cross_validation.cross_val_score(
rf,
X,
y,
n_jobs=-1,
cv=cross_validation.StratifiedShuffleSplit(y, n_iter=8, test_size=0.2))
print("X RF Accuracy: %0.3f (+- %0.2f)" %
(scores.mean(), scores.std() * 2))
"Instead of scores_f1, we could also use precision, sensitivity, MCC (if binary), etc'."
scores_f1 = cross_validation.cross_val_score(
rf,
X,
y,
n_jobs=-1,
cv=cross_validation.StratifiedShuffleSplit(y, n_iter=8, test_size=0.2),
scoring='f1')
print("X RF f1: %0.3f (+- %0.2f)" %
(scores_f1.mean(), scores_f1.std() * 2))
# rfeSelect = RFE(estimator=rf,n_features_to_select=16, step=0.04)
rfeSelect = RFECV(estimator=rf, step=20, cv=2,
scoring='f1') #average_precision , recall
X_RFE = rfeSelect.fit_transform(X, y)
print(X_RFE.shape)
RFE_FeatureNames = feature_cols[rfeSelect.get_support()]
print(RFE_FeatureNames)
RFE_ScoreRatio = 100 * (cross_validation.cross_val_score(
rf,
X_RFE,
y,
n_jobs=-1,
cv=cross_validation.StratifiedShuffleSplit(y, n_iter=8, test_size=0.2),
scoring='f1').mean()) / scores_f1.mean()
print(
"Even with just", X_RFE.shape[1],
" features, we have %f performance! (f1 score ratio)" %
(RFE_ScoreRatio))
# PlotFeaturesImportance(X_RFE, y, RFE_FeatureNames, dataName)
print("Alt plot:")
altPlotFeaturesImportance(X_RFE, y, RFE_FeatureNames, dataName)
def main(args):
if args.train_dir is None:
# args.train_dir = '/a/fr-05/vol/protein/danofer/ProtFeat/feat_extract/chap/train/'
#args.train_dir = '/cs/prt3/danofer/ProtFeat/feat_extract/test_seq/NP/SPCleaved_NP-70+NEG-30_Big-V3/'
# args.train_dir = r'D:\SkyDrive\Dropbox\bioInf_lab\AA_info\CODE\feat_extract\test_seq\NP\SPCleaved_NP-70+NEG-30_Big-V3'
# args.train_dir = r'E:\Dropbox\Dropbox\bioInf_lab\AA_info\fastas\NP\SP_Cleaved+NP+Neg_Big'
args.train_dir = r'E:\Dropbox\Dropbox\bioInf_lab\AA_info\fastas\Benchmarks\Thermophiles'
print("Using default train_dir: %s" % args.train_dir)
pandas.set_option('display.max_columns', 10)
pandas.set_option('display.max_rows', 4)
# mpl.rc('title', labelsize=6)
mpl.rc('ytick', labelsize=7)
mpl.rc('xtick', labelsize=4)
os.chdir(args.train_dir)
dataName = 'Neuropeptides'
df = pandas.read_csv('trainingSetFeatures.csv')
feature_cols = [col for col in df.columns if col not in ['classname','Id','proteinname']]
feature_cols=numpy.array(feature_cols)
X = df[feature_cols].values
y = df.classname.values
le = LabelEncoder()
y = le.fit_transform(y)
"Initial feature selection trimming"
print(X.shape)
Fwe = SelectFwe(alpha=0.01).fit(X,y)
X=Fwe.transform(X)
print("F-test -> ",X.shape)
feature_cols=feature_cols[Fwe.get_support()]
'''
FeatSelection_SVM = True
if FeatSelection_SVM == True:
svc_L1 = LinearSVC(C=50, penalty="l1", dual=False,class_weight='auto').fit(X, y)
X = svc_L1.transform(X, y)
print ("L1 SVM Transformed X:",X_L1.shape)
feature_cols=feature_cols[list(set(np.where(svc_L1.coef_ != 0)[-1]))]
'''
k = SelectKBest(k=255).fit(X,y)
X=k.transform(X)
feature_cols=feature_cols[k.get_support()]
param_dist = {"max_depth": [6,9, None],
"max_features": ['auto',0.4],
"min_samples_leaf": [1,2,3],
"bootstrap": [True, False],
'min_samples_split':[2,3],
"criterion": [ "gini"],
"n_estimators":[100],
"n_jobs":[-1]}
rf = RandomForestClassifierWithCoef(max_depth= 7, min_samples_split= 1, min_samples_leaf= 2, n_estimators= 50, n_jobs= 2, max_features= "auto")
"WARNING! F1 Score as implemented by Default in binary classification (two classes) gives the score for 1 class."
scores = cross_validation.cross_val_score(rf,X,y,n_jobs=-1,cv=cross_validation.StratifiedShuffleSplit(y,n_iter=8,test_size=0.2))
print("X RF Accuracy: %0.3f (+- %0.2f)" % (scores.mean(), scores.std() * 2))
"Instead of scores_f1, we could also use precision, sensitivity, MCC (if binary), etc'."
scores_f1 = cross_validation.cross_val_score(rf,X,y,n_jobs=-1,cv=cross_validation.StratifiedShuffleSplit(y,n_iter=8,test_size=0.2),scoring='f1')
print("X RF f1: %0.3f (+- %0.2f)" % (scores_f1.mean(), scores_f1.std() * 2))
# rfeSelect = RFE(estimator=rf,n_features_to_select=16, step=0.04)
rfeSelect = RFECV(estimator=rf,step=20, cv=2,scoring='f1') #average_precision , recall
X_RFE = rfeSelect.fit_transform(X,y)
print(X_RFE.shape)
RFE_FeatureNames = feature_cols[rfeSelect.get_support()]
print(RFE_FeatureNames)
RFE_ScoreRatio = 100*(cross_validation.cross_val_score(rf,X_RFE,y,n_jobs=-1,cv=cross_validation.StratifiedShuffleSplit(y,n_iter=8,test_size=0.2),scoring='f1').mean())/scores_f1.mean()
print("Even with just",X_RFE.shape[1]," features, we have %f performance! (f1 score ratio)" %(RFE_ScoreRatio))
# PlotFeaturesImportance(X_RFE, y, RFE_FeatureNames, dataName)
print("Alt plot:")
altPlotFeaturesImportance(X_RFE, y, RFE_FeatureNames, dataName)
#splitting training and test set
x_train, x_test, y_train, y_test = train_test_split(X,
Y,
test_size=0.20,
random_state=0)
#Chi-Squared Analysis
sel = SelectPercentile(chi2, percentile=80)
sel.fit(x_train, y_train)
x_train = sel.transform(x_train)
x_test = sel.transform(x_test)
#Univariate Feature Selection
fs = SelectFwe(alpha=150.0)
x_train = fs.fit_transform(x_train, y_train)
x_test = fs.transform(x_test)
#Classifier Fitting
clf = svm.LinearSVC(C=10,
penalty='l2',
loss='l1',
dual=True,
fit_intercept=False,
class_weight='auto')
clf.fit(x_train, y_train)
###############################################
'''Printed Data Analysis'''
###############################################
print "Training Accuracy"
]
feature_cols = np.array(feature_cols)
# In[ ]:
X = df[feature_cols].values
y = df.classname.values
# In[ ]:
le = LabelEncoder()
y = le.fit_transform(y)
# In[ ]:
print("Orig X -> ", X.shape)
Fwe = SelectFwe(alpha=0.001).fit(X, y)
X = Fwe.transform(X)
print("F-test -> ", X.shape)
feature_cols = feature_cols[Fwe.get_support()]
# In[ ]:
rf = RandomForestClassifierWithCoef(max_depth=9,
min_samples_split=3,
min_samples_leaf=3,
n_estimators=650,
n_jobs=-1,
max_features="auto")
# In[ ]:
scores = cross_val_score(rf,
def GetAllPerf (filePaths=None):
if filePaths is None:
filePaths = list(find_files(directory='./test_seq', pattern='trainingSetFeatures.csv'))
#Sanity check:
# filePaths=['/a/fr-05/vol/protein/danofer/ProtFeat/feat_extract/test_seq/Thermophile']
# filePaths=['./test_seq/NP/NP2/Train/trainingSetFeatures.csv']
print("FilePaths: \n",filePaths)
fileNames=fileNameFromPaths (filePaths)
print("FileNames:",fileNames)
resDict = pd.DataFrame(index=fileNames,
columns=['Accuracy','Accuracy_SD',
'f1','f1_SD','dummy_freq:Accuracy','dummy_freq:f1',
'LargestClassPercent','Classes',
# 'TopRFE-Features','Best (f1) Model parameters',
'# Classes',
'Array-Acc-Scores' ,'Array-f1-Scores'
,'bestML-Acc','bestML-f1','dummy_freq_f1_weighted'])
#redDict holds results for each file/class, for saving to output-file
i=-1
for filePath in filePaths:
i +=1
'http://pythonconquerstheuniverse.wordpress.com/2008/06/04/gotcha-%E2%80%94-backslashes-in-windows-filenames/'
filePath = os.path.normpath(filePath)
print(filePath)
fileName=str(fileNames[i]) #Str added now 14.1
print("fileName: %s" %(fileName))
"resDict['Name']= fileName"
# filePath = str(argv[1])
# X, y, lb_encoder,featureNames = load_data(filePath+fileName, 'file') # X, y = features, labels
X, y, lb_encoder,featureNames = load_data(filePath, 'file') # X, y = features, labels
print(X.shape,"= (samples, features)")
y_inv = Counter(lb_encoder.inverse_transform(y))
MajorityPercent = round(100*y_inv.most_common()[0][1]/sum(y_inv.values()),1)
print("Classes:", lb_encoder.classes_)
print("MajorityClassPercent:", MajorityPercent)
resDict.LargestClassPercent[fileName] = MajorityPercent
resDict.Classes[fileName] = str(lb_encoder.classes_)
resDict["# Classes"][fileName]=len(lb_encoder.classes_)
KFilt=None
KFilt=350 #This is just temporary for the outputs - saves computation time. Barely filters compared to the model itself.
if KFilt is not None:
k = SelectKBest(k=KFilt).fit(X,y)
X=k.transform(X)
featureNames=featureNames[k.get_support()]
Fwe = SelectFwe(alpha=0.01).fit(X,y)
X=Fwe.transform(X)
featureNames=featureNames[Fwe.get_support()]
print("X reduced to K best features: ",X.shape)
FeatSelection_SVM=False #Feature Names need updating!!
FeatSelection_RandLogReg=False
if FeatSelection_RandLogReg == True:
LogRegFeats = RandomizedLogisticRegression(C=10, scaling=0.5,
sample_fraction=0.95, n_resampling=40, selection_threshold=0.2,n_jobs=-1).fit(X,y)
X_L1 = LogRegFeats.transform(X)
featureNames=featureNames[LogRegFeats.get_support()]
print("RandomizedLogisticRegression Feature Selection ->:",X_L1.shape)
elif FeatSelection_SVM == True:
svc_L1= LinearSVC(C=30, penalty="l2", dual=False,class_weight='auto').fit(X, y)
X_L1 = svc_L1.transform(X, y)
featureNames=featureNames[list(set(np.where(svc_L1.coef_ != 0)[-1]))]
print ("L1 SVM Transformed X:",X_L1.shape)
# X=X_L1
'''
print("Performance as a function of percent of features used:")
PlotPerfPercentFeatures(X,y,est=LinearSVC())
'''
'EG - graph best features; feature selection using RF, ensemble classifiers..'
'http://nbviewer.ipython.org/github/herrfz/dataanalysis/blob/master/assignment2/samsung_data_prediction_submitted.ipynb'
RFE_FeatsToKeep = 16
FeatSelection_RFE=False
FeatSelection_RFECV=False
if (FeatSelection_RFE or FeatSelection_RFECV) == True:
'RFE + - best feats'
'http://scikit-learn.org/stable/auto_examples/plot_rfe_with_cross_validation.html '
svc = LinearSVC(class_weight='auto')#,penalty='l1',dual=False)
# svc = LogisticRegression(class_weight='auto')#,C=1)
if FeatSelection_RFECV==True:
rfecv = RFECV(estimator=svc, step=RFE_FeatsToKeep,scoring='average_precision')
# ,cv=StratifiedShuffleSplit(y,n_iter=3,test_size=0.3))
#,scoring='f1',verbose=0) # " scoring='roc_auc','recall','f1',accuracy..."
else:
rfecv = RFE(estimator=svc,n_features_to_select=RFE_FeatsToKeep, step=0.03)
rfecv.fit(X, y)
if FeatSelection_RFECV==True:
print("RFE-CV selected %d features : " % (rfecv.n_features_))
print("RFE (%d features) scorer : " % (rfecv.n_features_),rfecv.score(X, y) )
rfe_featnames = featureNames[rfecv.get_support()]
featureNames = featureNames[rfecv.get_support()]
print("RFE selected feature names:",rfe_featnames)
X_RFE = rfecv.fit_transform(X, y)
print("X_RFE",X_RFE.shape)
resDict['TopRFE-Features'][fileName]=str(rfe_featnames)
'Set GetRFEPerf To true or by user, if perf. of reduced set wanted'
GetRFEPerf=False
# print("lb_encoder.classes_",lb_encoder.classes_)
'Blind score boxplot graphic example using Seaborn: http://nbviewer.ipython.org/github/cs109/2014/blob/master/homework-solutions/HW5-solutions.ipynb '
'Confusion matrixes + Dummies - http://bugra.github.io/work/notes/2014-11-22/an-introduction-to-supervised-learning-scikit-learn/'
'http://scikit-learn.org/stable/modules/model_evaluation.html#dummy-estimators'
"http://blog.yhathq.com/posts/predicting-customer-churn-with-sklearn.html"
print()
"Make custom F1 scorer. May not have fixed problem!"
from sklearn.metrics.score import make_scorer
f1_scorer = make_scorer(metrics.f1_score,
greater_is_better=True, average="micro") #Maybe another metric? May NOT be fixed!?. #weighted, micro, macro, none
# print("Dummy classifiers output:")
dummy_frequent = DummyClassifier(strategy='most_frequent',random_state=0)
y_dummyPred = Get_yPred(X,y,clf_class=dummy_frequent)
dummy_freq_acc = '{:.3}'.format(metrics.accuracy_score(y,y_dummyPred ))
dummy_freq_f1 = '{:.3}'.format(metrics.f1_score(y, y_dummyPred,average='weighted'))
dummy_freq_f1_weighted = '{:.3}'.format(f1_scorer(y, y_dummyPred))
#Get from ALL classes f1..
dummy_freq_f1_mean=(metrics.f1_score(y, y_dummyPred,average=None)).mean()
# print("Dummy, most frequent acc:",dummy_freq_acc)
# dummy_stratifiedRandom = DummyClassifier(strategy='stratified',random_state=0)
# dummy_strat2= '{:.3%}'.format(metrics.accuracy_score(y, Get_yPred(X,y,clf_class=dummy_frequent))) #,sample_weight=balance_weights(y)))
# 'print("Dummy, Stratified Random:",dummy_strat2)'
print()
resDict['dummy_freq:Accuracy'][fileName]=dummy_freq_acc
## resDict['dummy_freq:f1'][fileName]=dummy_freq_f1 dummy_freq_f1_mean
resDict['dummy_freq:f1'][fileName]=dummy_freq_f1_mean
resDict['dummy_freq_f1_weighted'][fileName]=dummy_freq_f1_weighted
# resDict.dummy_Stratfreq[fileName]=dummy_strat2
"We can get seperately the best model for Acc, and the best for f1!"
"WARNING!? In binary case - default F1 works for the 1 class, in sklearn 15. and lower"
# bestEst_f1,bestScore_f1 = ModelParam_GridSearch(X,y,cv=3,scoreParam = 'f1')
"Temporary workaround until next SKlearn update of F1 metric:"
# bestEst_f1,bestScore_f1 = ModelParam_GridSearch(X,y,cv=3,scoreParam = 'f1')f1_scorer
bestEst_f1,bestScore_f1 = ModelParam_GridSearch(X,y,cv=3,scoreParam = f1_scorer)
bestEst_acc,bestScore_acc = ModelParam_GridSearch(X,y,cv=2,scoreParam = 'accuracy')
print("bestEst (f1):",bestEst_f1)#,"best f1",bestScore_f1)
print("bestEst (f1):",bestEst_acc)#,"best acc",bestScore_acc)
#Temp
# bestEst_f1=bestEst_acc=bestEst = RandomForestClassifier(n_jobs=-1)
if GetRFEPerf==True:
bestEst_RFE,bestScore_RFE = ModelParam_GridSearch(X_RFE,y,cv=3,scoreParam = 'f1')
"Modified to get 2 estimators"
scores_acc = cross_val_score(estimator=bestEst_acc, X=X, y=y, cv=StratifiedShuffleSplit(y, n_iter=13, test_size=0.18), n_jobs=-1) #Accuracy
print("Accuracy: %0.3f (+- %0.2f)" % (scores_acc.mean(), scores_acc.std() * 2))
scores_f1 = cross_val_score(estimator=bestEst_f1, X=X, y=y, cv=StratifiedShuffleSplit(y, n_iter=13, test_size=0.18), n_jobs=-1, scoring='f1')
print("f1: %0.3f (+- %0.2f)" % (scores_f1.mean(), scores_f1.std() * 2))
resDict['Accuracy'][fileName]=round(scores_acc.mean(),4)
resDict['Accuracy_SD'][fileName]=round(scores_acc.std(),4)
resDict['f1'][fileName]=round(scores_f1.mean(),4)
resDict['f1_SD'][fileName]=round(scores_f1.std(),4)
resDict['Array-f1-Scores'][fileName]=(scores_f1)
resDict['Array-Acc-Scores'][fileName]=(scores_acc)
resDict['bestML-f1'][fileName]=(str(bestEst_f1))
resDict['bestML-Acc'][fileName]=(str(bestEst_acc))
#ORIG
# Acc,Acc_SD,f1,f1_SD = CV_multi_stats(X, y, bestEst,n=15)
# resDict['Accuracy'][fileName]=round(Acc,4)
# resDict['Accuracy_SD'][fileName]=round(Acc_SD,4)
# resDict['f1 score'][fileName]=round(f1,4)
# resDict['f1_SD'][fileName]=round(f1_SD,4)
# resDict['Best (f1) Model parameters'][fileName]= bestEst
print()
# print(fileName," Done")
print("Saving results to file")
resDict.to_csv("OutputData.tsv", sep=',')
# In[ ]:
X=df[feature_cols].values
y=df.classname.values
# In[ ]:
le = LabelEncoder()
y = le.fit_transform(y)
# In[ ]:
print("Orig X -> ",X.shape)
Fwe = SelectFwe(alpha=0.001).fit(X,y)
X=Fwe.transform(X)
print("F-test -> ",X.shape)
feature_cols=feature_cols[Fwe.get_support()]
# In[ ]:
rf = RandomForestClassifierWithCoef(max_depth= 9, min_samples_split= 3, min_samples_leaf= 3, n_estimators= 650, n_jobs= -1, max_features= "auto")
# In[ ]:
scores = cross_val_score(rf,X,y,n_jobs=-1,cv=StratifiedShuffleSplit(y,n_iter=7,test_size=0.3))
print("X RF Accuracy: %0.3f (+- %0.2f)" % (scores.mean(), scores.std() * 2))
# scores_f1 = cross_val_score(rf,X,y,n_jobs=-1,cv=StratifiedShuffleSplit(y,n_iter=10,test_size=0.22),scoring='f1')
# print("X RF f1: %0.3f (+- %0.2f)" % (scores_f1.mean(), scores_f1.std() * 2))
# plot the line, the points, and the nearest vectors to the plane
pl.plot(xx, yy, "k-")
pl.plot(xx, yy_down, "k--")
pl.plot(xx, yy_up, "k--")
pl.scatter(clf.support_vectors_[:, 0], clf.support_vectors_[:, 1], s=80, facecolors="none")
pl.scatter(X[:, 0], X[:, 1], c=Y, cmap=pl.cm.Paired)
pl.axis("tight")
pl.show()
print "Training Accuracy"
# print clf.decision_function(x_train)
print (classification_report(y_train, clf.predict(x_train), target_names=target_names))
x_test = fs.transform(x_test)
print "Testing Accuracy"
print (classification_report(y_test, clf.predict(x_test), target_names=target_names))
decisions = clf.decision_function(x_test)
print "DECISION", decisions.shape[1]
# print y_test
X = np.array(decisions[:, 0])
# print X
Y = np.array(decisions[:, 2])
Z = np.array(decisions[:, 1])
points = []
for i, val in enumerate(X):
# print X[i], Y[i], Z[i]
points.append((X[i], Y[i], Z[i]))
points = list(set(points))
def train_predict_and_test(model,
target_name,
train_features,
train_labels,
test_features,
test_labels,
feature_selection=None):
classification = (target_name == Phenotypes.DIAGNOSED_ASTHMA
or target_name == Phenotypes.BODY_MASS_INDEX_CATEGORICAL)
# Standardize data
standardized = False
if model == Models.MLP or model == Models.SVM:
print("Standardizing data..")
standardized = True
features_mean = train_features.mean()
features_std = train_features.std()
train_features = (train_features - features_mean) / features_std
test_features = (test_features - features_mean) / features_std
if not classification:
labels_mean = train_labels.mean()
labels_std = train_labels.std()
train_labels = (train_labels - labels_mean) / labels_std
test_labels = (test_labels - labels_mean) / labels_std
# Load optimized params
params = load_optimized_params(model, target_name)
# Features selection
feature_selector = VarianceThreshold(threshold=0).fit(
train_features) # Removing features with 0 variance
train_col, test_col = train_features.columns, test_features.columns
train_features = pd.DataFrame(feature_selector.transform(train_features),
columns=train_col)
test_features = pd.DataFrame(feature_selector.transform(test_features),
columns=test_col)
if feature_selection == "fwe":
print("Selecting features according to Familly Wise Error")
# alpha = 5e-2
alpha = 0.3
if params is not None:
try:
alpha = params['transformer_alpha']
except KeyError:
print(
"Cannot find parameter alpha for FWE feature selector. Using default value"
)
features_selector = SelectFwe(f_regression,
alpha=alpha).fit(train_features,
train_labels)
train_features = features_selector.transform(train_features)
test_features = features_selector.transform(test_features)
elif feature_selection == "kbest":
k = 150
if params is not None:
try:
k = params['k']
except KeyError:
print(
"Cannot find parameter k for k-best feature selector. Using default value: k=",
k)
print("Selecting k-best features:", k)
score_func = f_regression
if classification:
score_func = f_classif
features_selector = SelectKBest(score_func=score_func, k=k)
features_selector = features_selector.fit(train_features, train_labels)
train_features = features_selector.transform(train_features)
test_features = features_selector.transform(test_features)
elif feature_selection == "tree":
print("Selecting features from RF feature importance")
clf = RandomForestRegressor(n_estimators=100).fit(
train_features, train_labels)
if classification:
clf = RandomForestClassifier(n_estimators=100).fit(
train_features, train_labels)
features_selector = SelectFromModel(clf, prefit=True)
train_features = features_selector.transform(train_features)
test_features = features_selector.transform(test_features)
elif feature_selection == "corr":
threshold = 0.9 # Recommended default value
col_corr = set()
corr_matrix = train_features.corr()
for i in range(len(corr_matrix.columns)):
for j in range(i):
if abs(corr_matrix.iloc[i, j]) > threshold:
colname = corr_matrix.columns[i]
col_corr.add(colname)
train_features = train_features.drop(col_corr, axis=1)
test_features = test_features.drop(col_corr, axis=1)
# Oversampling
if classification and model != Models.SVM and model != Models.CART and model != Models.ELASTIC:
print("Oversampling features..")
if target_name == Phenotypes.DIAGNOSED_ASTHMA:
sampling_strat = 0.5
else:
sampling_strat = {
0: np.max(np.bincount(train_labels)) // 4,
1: np.max(np.bincount(train_labels)),
2: np.max(np.bincount(train_labels)),
3: np.max(np.bincount(train_labels)) // 2
}
oversampler = imblearn.over_sampling.RandomOverSampler(
sampling_strategy=sampling_strat, random_state=42)
# oversampler = imblearn.over_sampling.SMOTE(sampling_strategy=1.0,
# k_neighbors=5,
# random_state=42)
train_features, train_labels = oversampler.fit_resample(
train_features, train_labels)
if model == Models.RF:
if target_name == Phenotypes.BODY_MASS_INDEX_CATEGORICAL:
# Create validation set for threshold optimization
val_features, test_features, val_labels, test_labels = train_test_split(
test_features, test_labels, test_size=0.5, random_state=42)
model, predictions = _predict_rf(target_name, train_features,
train_labels, val_features,
val_labels)
else:
model, predictions = _predict_rf(target_name,
train_features,
train_labels,
test_features,
test_labels,
params=params)
elif model == Models.ELASTIC:
model, predictions = predict_elastic_net(target_name, train_features,
train_labels, test_features,
test_labels)
elif model == Models.XGB:
model, predictions = _predict_xgb(target_name,
train_features,
train_labels,
test_features,
test_labels,
params=params)
elif model == Models.MLP:
model, predictions = _predict_mlp(target_name,
train_features,
train_labels,
test_features,
test_labels,
params=params)
elif model == Models.SVM:
model, predictions = _predict_svm(target_name, train_features,
train_labels, test_features,
test_labels)
elif model == Models.CART:
model, predictions = _predict_cart(target_name, train_features,
train_labels, test_features,
test_labels)
elif model == Models.NAIVE:
if not (classification):
predictions = predict_naive(train_features, train_labels,
test_features, test_labels)
else:
raise SystemExit("Cannot use naive model on classification task")
else:
raise SystemExit("Unkwown model:", model)
# Destandardize results
if standardized and not (classification):
print("destandardize data..")
predictions = (predictions * labels_std) + labels_mean
test_labels = (test_labels * labels_std) + labels_mean
# Print results
if classification:
print_classification_metrics(ground_truth=test_labels,
predictions=predictions,
num_classes=test_labels.nunique())
else:
print_regression_metrics(ground_truth=test_labels,
predictions=predictions)
return model, predictions
def GetAllPerf (filePaths=None):
if filePaths is None:
filePaths = list(find_files(directory='./test_seq', pattern='trainingSetFeatures.csv'))
#Sanity check:
# filePaths=['/a/fr-05/vol/protein/danofer/ProtFeat/feat_extract/test_seq/Thermophile']
# filePaths=['./test_seq/NP/NP2/Train/trainingSetFeatures.csv']
print("FilePaths: \n",filePaths)
fileNames=fileNameFromPaths (filePaths)
print("FileNames:",fileNames)
resDict = pd.DataFrame(index=fileNames,
columns=['Accuracy','Accuracy_SD',
'f1','f1_SD','dummy_freq:Accuracy','dummy_freq:f1',
'LargestClassPercent','Classes',
# 'TopRFE-Features','Best (f1) Model parameters',
'# Classes',
'Array-Acc-Scores' ,'Array-f1-Scores'
,'bestML-Acc','bestML-f1','dummy_freq_f1_weighted'])
#redDict holds results for each file/class, for saving to output-file
i=-1
for filePath in filePaths:
i +=1
'http://pythonconquerstheuniverse.wordpress.com/2008/06/04/gotcha-%E2%80%94-backslashes-in-windows-filenames/'
filePath = os.path.normpath(filePath)
print(filePath)
fileName=str(fileNames[i]) #Str added now 14.1
print("fileName: %s" %(fileName))
"resDict['Name']= fileName"
# filePath = str(argv[1])
# X, y, lb_encoder,featureNames = load_data(filePath+fileName, 'file') # X, y = features, labels
X, y, lb_encoder,featureNames = load_data(filePath) # X, y = features, labels
print(X.shape,"= (samples, features)")
y_inv = Counter(lb_encoder.inverse_transform(y))
MajorityPercent = round(100*y_inv.most_common()[0][1]/sum(y_inv.values()),1)
print("Classes:", lb_encoder.classes_)
print("MajorityClassPercent:", MajorityPercent)
resDict.LargestClassPercent[fileName] = MajorityPercent
resDict.Classes[fileName] = str(lb_encoder.classes_)
resDict["# Classes"][fileName]=len(lb_encoder.classes_)
KFilt=None
KFilt=350 #This is just temporary for the outputs - saves computation time. Barely filters compared to the model itself.
if KFilt is not None:
k = SelectKBest(k=KFilt).fit(X,y)
X=k.transform(X)
featureNames=featureNames[k.get_support()]
Fwe = SelectFwe(alpha=0.01).fit(X,y)
X=Fwe.transform(X)
featureNames=featureNames[Fwe.get_support()]
print("X reduced to K best features: ",X.shape)
FeatSelection_SVM=False #Feature Names need updating!!
FeatSelection_RandLogReg=False
if FeatSelection_RandLogReg == True:
LogRegFeats = RandomizedLogisticRegression(C=10, scaling=0.5,
sample_fraction=0.95, n_resampling=40, selection_threshold=0.2,n_jobs=-1).fit(X,y)
X_L1 = LogRegFeats.transform(X)
featureNames=featureNames[LogRegFeats.get_support()]
print("RandomizedLogisticRegression Feature Selection ->:",X_L1.shape)
elif FeatSelection_SVM == True:
svc_L1= LinearSVC(C=30, penalty="l2", dual=False,class_weight='auto').fit(X, y)
X_L1 = svc_L1.transform(X, y)
featureNames=featureNames[list(set(np.where(svc_L1.coef_ != 0)[-1]))]
print ("L1 SVM Transformed X:",X_L1.shape)
# X=X_L1
'''
print("Performance as a function of percent of features used:")
PlotPerfPercentFeatures(X,y,est=LinearSVC())
'''
'EG - graph best features; feature selection using RF, ensemble classifiers..'
'http://nbviewer.ipython.org/github/herrfz/dataanalysis/blob/master/assignment2/samsung_data_prediction_submitted.ipynb'
RFE_FeatsToKeep = 16
FeatSelection_RFE=False
FeatSelection_RFECV=False
if (FeatSelection_RFE or FeatSelection_RFECV) == True:
'RFE + - best feats'
'http://scikit-learn.org/stable/auto_examples/plot_rfe_with_cross_validation.html '
svc = LinearSVC(class_weight='auto')#,penalty='l1',dual=False)
# svc = LogisticRegression(class_weight='auto')#,C=1)
if FeatSelection_RFECV==True:
rfecv = RFECV(estimator=svc, step=RFE_FeatsToKeep,scoring='average_precision')
# ,cv=StratifiedShuffleSplit(y,n_iter=3,test_size=0.3))
#,scoring='f1',verbose=0) # " scoring='roc_auc','recall','f1',accuracy..."
else:
rfecv = RFE(estimator=svc,n_features_to_select=RFE_FeatsToKeep, step=0.03)
rfecv.fit(X, y)
if FeatSelection_RFECV==True:
print("RFE-CV selected %d features : " % (rfecv.n_features_))
print("RFE (%d features) scorer : " % (rfecv.n_features_),rfecv.score(X, y) )
rfe_featnames = featureNames[rfecv.get_support()]
featureNames = featureNames[rfecv.get_support()]
print("RFE selected feature names:",rfe_featnames)
X_RFE = rfecv.fit_transform(X, y)
print("X_RFE",X_RFE.shape)
resDict['TopRFE-Features'][fileName]=str(rfe_featnames)
'Set GetRFEPerf To true or by user, if perf. of reduced set wanted'
GetRFEPerf=False
# print("lb_encoder.classes_",lb_encoder.classes_)
'Blind score boxplot graphic example using Seaborn: http://nbviewer.ipython.org/github/cs109/2014/blob/master/homework-solutions/HW5-solutions.ipynb '
'Confusion matrixes + Dummies - http://bugra.github.io/work/notes/2014-11-22/an-introduction-to-supervised-learning-scikit-learn/'
'http://scikit-learn.org/stable/modules/model_evaluation.html#dummy-estimators'
"http://blog.yhathq.com/posts/predicting-customer-churn-with-sklearn.html"
print()
"Make custom F1 scorer. May not have fixed problem!"
from sklearn.metrics.score import make_scorer
f1_scorer = make_scorer(metrics.f1_score,
greater_is_better=True, average="micro") #Maybe another metric? May NOT be fixed!?. #weighted, micro, macro, none
# print("Dummy classifiers output:")
dummy_frequent = DummyClassifier(strategy='most_frequent',random_state=0)
y_dummyPred = Get_yPred(X,y,clf_class=dummy_frequent)
dummy_freq_acc = '{:.3}'.format(metrics.accuracy_score(y,y_dummyPred ))
dummy_freq_f1 = '{:.3}'.format(metrics.f1_score(y, y_dummyPred,average='weighted'))
dummy_freq_f1_weighted = '{:.3}'.format(f1_scorer(y, y_dummyPred))
#Get from ALL classes f1..
dummy_freq_f1_mean=(metrics.f1_score(y, y_dummyPred,average=None)).mean()
# print("Dummy, most frequent acc:",dummy_freq_acc)
# dummy_stratifiedRandom = DummyClassifier(strategy='stratified',random_state=0)
# dummy_strat2= '{:.3%}'.format(metrics.accuracy_score(y, Get_yPred(X,y,clf_class=dummy_frequent))) #,sample_weight=balance_weights(y)))
# 'print("Dummy, Stratified Random:",dummy_strat2)'
print()
resDict['dummy_freq:Accuracy'][fileName]=dummy_freq_acc
## resDict['dummy_freq:f1'][fileName]=dummy_freq_f1 dummy_freq_f1_mean
resDict['dummy_freq:f1'][fileName]=dummy_freq_f1_mean
resDict['dummy_freq_f1_weighted'][fileName]=dummy_freq_f1_weighted
# resDict.dummy_Stratfreq[fileName]=dummy_strat2
"We can get seperately the best model for Acc, and the best for f1!"
"WARNING!? In binary case - default F1 works for the 1 class, in sklearn 15. and lower"
# bestEst_f1,bestScore_f1 = ModelParam_GridSearch(X,y,cv=3,scoreParam = 'f1')
"Temporary workaround until next SKlearn update of F1 metric:"
# bestEst_f1,bestScore_f1 = ModelParam_GridSearch(X,y,cv=3,scoreParam = 'f1')f1_scorer
bestEst_f1,bestScore_f1 = ModelParam_GridSearch(X,y,cv=3,scoreParam = f1_scorer)
bestEst_acc,bestScore_acc = ModelParam_GridSearch(X,y,cv=2,scoreParam = 'accuracy')
print("bestEst (f1):",bestEst_f1)#,"best f1",bestScore_f1)
print("bestEst (f1):",bestEst_acc)#,"best acc",bestScore_acc)
#Temp
# bestEst_f1=bestEst_acc=bestEst = RandomForestClassifier(n_jobs=-1)
if GetRFEPerf==True:
bestEst_RFE,bestScore_RFE = ModelParam_GridSearch(X_RFE,y,cv=3,scoreParam = 'f1')
"Modified to get 2 estimators"
scores_acc = cross_val_score(estimator=bestEst_acc, X=X, y=y, cv=StratifiedShuffleSplit(y, n_iter=13, test_size=0.18), n_jobs=-1) #Accuracy
print("Accuracy: %0.3f (+- %0.2f)" % (scores_acc.mean(), scores_acc.std() * 2))
scores_f1 = cross_val_score(estimator=bestEst_f1, X=X, y=y, cv=StratifiedShuffleSplit(y, n_iter=13, test_size=0.18), n_jobs=-1, scoring='f1')
print("f1: %0.3f (+- %0.2f)" % (scores_f1.mean(), scores_f1.std() * 2))
resDict['Accuracy'][fileName]=round(scores_acc.mean(),4)
resDict['Accuracy_SD'][fileName]=round(scores_acc.std(),4)
resDict['f1'][fileName]=round(scores_f1.mean(),4)
resDict['f1_SD'][fileName]=round(scores_f1.std(),4)
resDict['Array-f1-Scores'][fileName]=(scores_f1)
resDict['Array-Acc-Scores'][fileName]=(scores_acc)
resDict['bestML-f1'][fileName]=(str(bestEst_f1))
resDict['bestML-Acc'][fileName]=(str(bestEst_acc))
#ORIG
# Acc,Acc_SD,f1,f1_SD = CV_multi_stats(X, y, bestEst,n=15)
# resDict['Accuracy'][fileName]=round(Acc,4)
# resDict['Accuracy_SD'][fileName]=round(Acc_SD,4)
# resDict['f1 score'][fileName]=round(f1,4)
# resDict['f1_SD'][fileName]=round(f1_SD,4)
# resDict['Best (f1) Model parameters'][fileName]= bestEst
print()
# print(fileName," Done")
print("Saving results to file")
resDict.to_csv("OutputData.tsv", sep=',')
def run():
target_names = ["Self", "Another Person", "General Statement"]
tweets_and_labels = parse_labeled_data(filename)
#splitting training and test set
y_train, x_test, x_train = get_x_y(tweets_and_labels, testdata)
#Chi-Squared Analysis
sel = SelectPercentile(chi2, percentile=80)
sel.fit(x_train, y_train)
x_train = sel.transform(x_train)
x_test = sel.transform(x_test)
#Univariate Feature Selection
fs = SelectFwe(alpha=150.0)
x_train = fs.fit_transform(x_train, y_train)
x_test = fs.transform(x_test)
#Classifier Fitting
clf = svm.LinearSVC(C=10,
penalty='l2',
loss='l1',
dual=True,
fit_intercept=False,
class_weight='auto')
clf.fit(x_train, y_train)
returned = clf.predict(x_test)
print returned
#Print relevant usernames & tweets to .csv file
t = time.strftime("%d_%m_%Y")
output1 = 'classifications/' + t + '_self.csv'
output2 = 'classifications/' + t + '_another_person.csv'
with open(output1, 'w+') as o1:
wr = csv.writer(o1, quoting=csv.QUOTE_ALL)
for i, val in enumerate(returned):
if val == 0:
row = [testdata[i][1], testdata[i][0]]
wr.writerow(row)
with open(output2, 'w+') as o2:
wr = csv.writer(o2, quoting=csv.QUOTE_ALL)
for i, val in enumerate(returned):
if val == 1:
row = [testdata[i][1], testdata[i][0]]
wr.writerow(row)
########################################################################
'''Graphing of Data'''
'''Note, since there is no annotation for test data'''
'''This is a visual representation of output data, not model accuracy'''
########################################################################
graph = True
if (graph):
#Graph setup
X, Y, Z, new_y = graph_setup(clf, x_test, returned)
#graph Scatter Plot of training data
graph_scatter(x_train, y_train)
#Graph 3D Plot of test data
graph_3d(X, Y, Z, new_y)
#Graph 2-D Plot of test data
graph_2d(X, Y, new_y)
