def _update_metrics(self, y_true, y_pred,
onco_prob, tsg_prob):
# record which genes were predicted what
self.driver_gene_pred = pd.Series(y_pred, self.y.index)
self.driver_gene_score = pd.Series(onco_prob+tsg_prob, self.y.index)
# evaluate performance
prec, recall, fscore, support = metrics.precision_recall_fscore_support(y_true, y_pred,
average='macro')
cancer_gene_pred = ((onco_prob + tsg_prob)>.5).astype(int)
self.cancer_gene_count[self.num_pred] = np.sum(cancer_gene_pred)
self.precision[self.num_pred] = prec
self.recall[self.num_pred] = recall
self.f1_score[self.num_pred] = fscore
# compute Precision-Recall curve metrics
driver_prob = onco_prob + tsg_prob
driver_true = (y_true > 0).astype(int)
p, r, thresh = metrics.precision_recall_curve(driver_true, driver_prob)
p, r, thresh = p[::-1], r[::-1], thresh[::-1] # reverse order of results
thresh = np.insert(thresh, 0, 1.0)
self.driver_precision_array[self.num_pred, :] = interp(self.driver_recall_array, r, p)
self.driver_threshold_array[self.num_pred, :] = interp(self.driver_recall_array, r, thresh)
# calculate prediction summary statistics
prec, recall, fscore, support = metrics.precision_recall_fscore_support(driver_true, cancer_gene_pred)
self.driver_precision[self.num_pred] = prec[1]
self.driver_recall[self.num_pred] = recall[1]
# save driver metrics
fpr, tpr, thresholds = metrics.roc_curve(driver_true, driver_prob)
self.driver_tpr_array[self.num_pred, :] = interp(self.driver_fpr_array, fpr, tpr)
def test(self, a_trees, a_segments):
"""Estimate performance of segmenter model.
Args:
a_trees (list): BitPar trees
a_segments (list): corresponding gold segments for trees
Returns:
2-tuple: macro and micro-averaged F-scores
"""
if self.model is None:
return (0, 0)
segments = [self.model.predict(self.featgen(itree))[0]
for itree in a_trees]
a_segments = [str(s) for s in a_segments]
_, _, macro_f1, _ = precision_recall_fscore_support(a_segments,
segments,
average='macro',
warn_for=())
_, _, micro_f1, _ = precision_recall_fscore_support(a_segments,
segments,
average='micro',
warn_for=())
return (macro_f1, micro_f1)
def learnCART(self):
train_input_data = self.loadData(self.train_file)
target = [x[1] for x in train_input_data]
target = target[1:]
features = [x[2:] for x in train_input_data]
features = features[1:]
# feature selection
#features_new = self.doFeatureSelection(features,target)
model = self.classify(features,target)
test_input_data = self.loadData(self.test_file)
actualOutput = [x[1] for x in test_input_data]
actualOutput = actualOutput[1:]
features = [x[2:] for x in test_input_data]
features = features[1:]
predictedOutput = model.predict(features)
#print predictedOutput
#print actualOutput
self.computeAccuracy(predictedOutput,actualOutput)
print "Precision recall Fscore support metrics for CART "
print precision_recall_fscore_support(actualOutput,predictedOutput)
print "\nconfusion matrix\n"
print confusion_matrix(actualOutput,predictedOutput)
self.printDTRules(model)
X= []
Y=[]
for a in predictedOutput:
X.append(int(a))
for a in actualOutput:
Y.append(int(a))
self.plotROC(Y,X)
result = zip(Y,X)
self.write_To_File(result,"cart-predictions.csv")
def compare_dummy_classification(self):
""" Compares classifier to dummy classifiers. Return results (resultscores_tuple, N.A., N.A.)"""
X_train = self.train_vectors
y_train = self.train_tweetclasses
X_test = self.test_vectors
y_test = self.test_tweetclasses
dummy_results = []
dummy = DummyClassifier(strategy="most_frequent", random_state=0)
dummy.fit(X_train, y_train)
y_true, y_preddum = y_test, dummy.predict(X_test)
tuples = precision_recall_fscore_support(y_true, y_preddum)
dummy1 = DummyClassifier(strategy="stratified", random_state=0)
dummy1.fit(X_train, y_train)
y_true, y_preddum1 = y_test, dummy1.predict(X_test)
tuples1 = precision_recall_fscore_support(y_true, y_preddum1)
dummy2 = DummyClassifier(strategy="uniform", random_state=0)
dummy2.fit(X_train, y_train)
y_true, y_preddum2 = y_test, dummy2.predict(X_test)
tuples2 = precision_recall_fscore_support(y_true, y_preddum2)
resulttuple = ("dummy freq", "N.A.", "N.A.", "N.A.", "N.A.", tuples)
resulttuple1 = ("dummy strat", "N.A.", "N.A.", "N.A.", "N.A.", tuples1)
resulttuple2 = ("dummy uni", "N.A.", "N.A.", "N.A.", "N.A.", tuples2)
dummy_results.append(resulttuple)
dummy_results.append(resulttuple1)
dummy_results.append(resulttuple2)
return dummy_results
def main():
do_it = 1
# get data
global g_train, g_train_label, g_test, g_test_label, g_feature_name
g_train, g_train_label, g_test, g_test_label, g_feature_name = load_features_and_labels()
# do
if do_it == 0:
forest = cudaTreeRandomForestClassifier(n_estimators=50, verbose=True, bootstrap=False)
forest.fit(np.asarray(g_train), np.asarray(g_train_label), bfs_threshold=4196)
predictions = forest.predict(np.asarray(g_test))
print precision_recall_fscore_support(g_test_label, predictions, average='micro')
# do
if do_it == 0:
forest = hybridForestRandomForestClassifier(n_estimators=50,
n_gpus=2,
n_jobs=6,
bootstrap=False,
cpu_classifier=WiseRF)
forest.fit(np.asarray(g_train), np.asarray(g_train_label), bfs_threshold=4196)
predictions = forest.predict(np.asarray(g_test))
print precision_recall_fscore_support(g_test_label, predictions, average='micro')
def compare_dummy(self): """ Compares classifier to dummy classifiers""" #print "\nDetailed classification report:\n" #print "The model is trained on the full development set.\n" #print "The scores are computed on the full evaluation set.\n" X_train = self.train_vectors y_train = self.train_tweetclasses X_test = self.test_vectors y_test = self.test_tweetclasses dummy = DummyClassifier(strategy='most_frequent',random_state=0) dummy.fit(X_train, y_train) y_true, y_preddum = y_test, dummy.predict(X_test) tuples = precision_recall_fscore_support(y_true, y_preddum) dummy1 = DummyClassifier(strategy='stratified',random_state=0) dummy1.fit(X_train, y_train) y_true, y_preddum1 = y_test, dummy1.predict(X_test) tuples1 = precision_recall_fscore_support(y_true, y_preddum1) dummy2 = DummyClassifier(strategy='uniform',random_state=0) dummy2.fit(X_train, y_train) y_true, y_preddum2 = y_test, dummy2.predict(X_test) tuples2 = precision_recall_fscore_support(y_true, y_preddum2) return (tuples, tuples1,tuples2)
def compute_precision_recall_accuracy_thresholded_v3(threshold=0.5, sim_column=4):
"""
starting from 0 the sims (for supp-v2) are:
soundex
nysiis
metaphone
:param threshold:
:param sim_column:
:return:
"""
tab_strings, tab_values, ground_truth = read_in_sim_data_as_table()
y_true = list()
y_pred = list()
num_pos = 0
output_pos = 0
for i in range(len(tab_values)):
y_true.append(ground_truth[i])
if ground_truth[i] == 1:
num_pos += 1
if tab_values[i][sim_column] >= threshold:
y_pred.append(1)
output_pos += 1
else:
y_pred.append(0)
# precision, recall, thresholds = precision_recall_curve(np.array(y_true), np.array(y_pred))
print 'printing precision, recall, fscore, support:',
print precision_recall_fscore_support(np.array(y_true), np.array(y_pred), average='binary')
print 'accuracy score: ', accuracy_score(np.array(y_true), np.array(y_pred))
# print precision
# print recall
print 'number of positives output: ',
print num_pos
print 'number of positives in ground truth: ',
print output_pos
def compute_precision_recall_accuracy_thresholded_v2(threshold=0.5, sim_column=4):
"""
starting from 0 the sims (for supp-v2) are:
tri_gram_jaccard_similarity
jaro_winkler_similarity
levenshtein_similarity
needleman_wunsch_similarity
metaphone_similarity
:param threshold:
:param sim_column:
:return:
"""
tab_strings, tab_values, ground_truth = read_in_sim_data_as_table()
y_true = list()
y_pred = list()
num_pos = 0
output_pos = 0
for i in range(len(tab_values)):
y_true.append(ground_truth[i])
if ground_truth[i] == 1:
num_pos += 1
if tab_values[i][sim_column] >= threshold:
y_pred.append(1)
output_pos += 1
else:
y_pred.append(0)
# precision, recall, thresholds = precision_recall_curve(np.array(y_true), np.array(y_pred))
print precision_recall_fscore_support(np.array(y_true), np.array(y_pred), average='binary')
print 'accuracy score: ',accuracy_score(np.array(y_true), np.array(y_pred))
# print precision
# print recall
print num_pos
print output_pos
def test_precision_recall_f1_score_with_an_empty_prediction():
y_true = np.array([[0, 1, 0, 0], [1, 0, 0, 0], [0, 1, 1, 0]])
y_pred = np.array([[0, 0, 0, 0], [0, 0, 0, 1], [0, 1, 1, 0]])
# true_pos = [ 0. 1. 1. 0.]
# false_pos = [ 0. 0. 0. 1.]
# false_neg = [ 1. 1. 0. 0.]
p, r, f, s = precision_recall_fscore_support(y_true, y_pred,
average=None)
assert_array_almost_equal(p, [0.0, 1.0, 1.0, 0.0], 2)
assert_array_almost_equal(r, [0.0, 0.5, 1.0, 0.0], 2)
assert_array_almost_equal(f, [0.0, 1 / 1.5, 1, 0.0], 2)
assert_array_almost_equal(s, [1, 2, 1, 0], 2)
f2 = fbeta_score(y_true, y_pred, beta=2, average=None)
support = s
assert_array_almost_equal(f2, [0, 0.55, 1, 0], 2)
p, r, f, s = precision_recall_fscore_support(y_true, y_pred,
average="macro")
assert_almost_equal(p, 0.5)
assert_almost_equal(r, 1.5 / 4)
assert_almost_equal(f, 2.5 / (4 * 1.5))
assert_equal(s, None)
assert_almost_equal(fbeta_score(y_true, y_pred, beta=2,
average="macro"),
np.mean(f2))
p, r, f, s = precision_recall_fscore_support(y_true, y_pred,
average="micro")
assert_almost_equal(p, 2 / 3)
assert_almost_equal(r, 0.5)
assert_almost_equal(f, 2 / 3 / (2 / 3 + 0.5))
assert_equal(s, None)
assert_almost_equal(fbeta_score(y_true, y_pred, beta=2,
average="micro"),
(1 + 4) * p * r / (4 * p + r))
p, r, f, s = precision_recall_fscore_support(y_true, y_pred,
average="weighted")
assert_almost_equal(p, 3 / 4)
assert_almost_equal(r, 0.5)
assert_almost_equal(f, (2 / 1.5 + 1) / 4)
assert_equal(s, None)
assert_almost_equal(fbeta_score(y_true, y_pred, beta=2,
average="weighted"),
np.average(f2, weights=support))
p, r, f, s = precision_recall_fscore_support(y_true, y_pred,
average="samples")
# |h(x_i) inter y_i | = [0, 0, 2]
# |y_i| = [1, 1, 2]
# |h(x_i)| = [0, 1, 2]
assert_almost_equal(p, 1 / 3)
assert_almost_equal(r, 1 / 3)
assert_almost_equal(f, 1 / 3)
assert_equal(s, None)
assert_almost_equal(fbeta_score(y_true, y_pred, beta=2,
average="samples"),
0.333, 2)
def main():
model_file = '../../paper/data/srwe_model/wiki_small.w2v.model'
nytimes_file = '../gen_data/nytimes/news_corpus'
model = load_w2v_model(model_file, logging, nparray=True)
corpus_vec, corpus_label = load_nytimes(nytimes_file, model)
labels = list(set(corpus_label))
X_train, X_test, y_train, y_test = train_test_split(corpus_vec, corpus_label, test_size=0.2, random_state=42)
logging.info('train size: %d, test size:%d' % (len(y_train), len(y_test)))
clfs = {}
for label in labels:
clfs[label] = train(label, X_train, X_test, y_train, y_test)
y_pred = []
for each in X_test:
pred_res = []
for label in clfs:
pred_res.append((clfs[label].predict_proba(each.reshape(1, -1))[0][1], label))
sorted_pred = sorted(pred_res, key=lambda x: x[0], reverse=True)
y_pred.append(sorted_pred[0][1])
precision, recall, f_score, support, present_labels = precision_recall_fscore_support(y_test, y_pred)
for l, p, r, f in zip(present_labels, precision, recall, f_score):
print '%s\t%.4lf\t%.4lf\t%.4lf' % (l, p, r, f)
precision, recall, f_score, support, present_labels = precision_recall_fscore_support(y_test, y_pred, average='macro')
print 'Macro\t%.4lf\t%.4lf\t%.4lf' % (precision, recall, f_score)
precision, recall, f_score, support, present_labels = precision_recall_fscore_support(y_test, y_pred, average='micro')
print 'Micro\t%.4lf\t%.4lf\t%.4lf' % (precision, recall, f_score)
def evaluate_mutiple(ground_truth, prediction, find_max=False, f_beta = 1.0, avg_method=None):
"""
:param ground_truth: 1-d array, e.g. gt: [1, 1, 2, 2, 3]
:param prediction: 1-d array, e.g. prediction: [1, 1, 2, 2, 4]
:return: recall, precision, f-value
"""
prediction_indices = prediction
if find_max or len(prediction.shape) == 2:
prediction_indices = find_max_indices(prediction)
# Find Precision & Recall & F-value
precision, recall, f_value, support = None, None, None, None
if len(prediction.shape) == 2:
M = prediction.shape[1]
precision, recall, f_value, support \
= precision_recall_fscore_support(ground_truth,
prediction_indices,
beta=f_beta,
pos_label=M,
average=avg_method)
else:
precision, recall, f_value, support \
= precision_recall_fscore_support(ground_truth,
prediction_indices,
beta=f_beta,
average=avg_method)
return precision, recall, f_value
def clf_metrics(p_train, p_test, y_train, y_test):
""" Compute metrics on classifier predictions
Parameters
----------
p_train : np.array [n_samples]
predicted probabilities for training set
p_test : np.array [n_samples]
predicted probabilities for testing set
y_train : np.array [n_samples]
Training labels.
y_test : np.array [n_samples]
Testing labels.
Returns
-------
clf_scores : dict
classifier scores for training set
"""
y_pred_train = 1*(p_train >= 0.5)
y_pred_test = 1*(p_test >= 0.5)
train_scores = {}
test_scores = {}
train_scores['accuracy'] = metrics.accuracy_score(y_train, y_pred_train)
test_scores['accuracy'] = metrics.accuracy_score(y_test, y_pred_test)
train_scores['mcc'] = metrics.matthews_corrcoef(y_train, y_pred_train)
test_scores['mcc'] = metrics.matthews_corrcoef(y_test, y_pred_test)
(p, r, f, s) = metrics.precision_recall_fscore_support(y_train,
y_pred_train)
train_scores['precision'] = p
train_scores['recall'] = r
train_scores['f1'] = f
train_scores['support'] = s
(p, r, f, s) = metrics.precision_recall_fscore_support(y_test,
y_pred_test)
test_scores['precision'] = p
test_scores['recall'] = r
test_scores['f1'] = f
test_scores['support'] = s
train_scores['confusion matrix'] = \
metrics.confusion_matrix(y_train, y_pred_train, labels=[0, 1])
test_scores['confusion matrix'] = \
metrics.confusion_matrix(y_test, y_pred_test, labels=[0, 1])
train_scores['auc score'] = \
metrics.roc_auc_score(y_train, p_train + 1, average='weighted')
test_scores['auc score'] = \
metrics.roc_auc_score(y_test, p_test + 1, average='weighted')
clf_scores = {'train': train_scores, 'test': test_scores}
return clf_scores
def melodiness_metrics(m_train, m_test, y_train, y_test):
""" Compute metrics on melodiness score
Parameters
----------
m_train : np.array [n_samples]
melodiness scores for training set
m_test : np.array [n_samples]
melodiness scores for testing set
y_train : np.array [n_samples]
Training labels.
y_test : np.array [n_samples]
Testing labels.
Returns
-------
melodiness_scores : dict
melodiness scores for training set
"""
m_bin_train = 1*(m_train >= 1)
m_bin_test = 1*(m_test >= 1)
train_scores = {}
test_scores = {}
train_scores['accuracy'] = metrics.accuracy_score(y_train, m_bin_train)
test_scores['accuracy'] = metrics.accuracy_score(y_test, m_bin_test)
train_scores['mcc'] = metrics.matthews_corrcoef(y_train, m_bin_train)
test_scores['mcc'] = metrics.matthews_corrcoef(y_test, m_bin_test)
(p, r, f, s) = metrics.precision_recall_fscore_support(y_train,
m_bin_train)
train_scores['precision'] = p
train_scores['recall'] = r
train_scores['f1'] = f
train_scores['support'] = s
(p, r, f, s) = metrics.precision_recall_fscore_support(y_test,
m_bin_test)
test_scores['precision'] = p
test_scores['recall'] = r
test_scores['f1'] = f
test_scores['support'] = s
train_scores['confusion matrix'] = \
metrics.confusion_matrix(y_train, m_bin_train, labels=[0, 1])
test_scores['confusion matrix'] = \
metrics.confusion_matrix(y_test, m_bin_test, labels=[0, 1])
train_scores['auc score'] = \
metrics.roc_auc_score(y_train, m_train + 1, average='weighted')
test_scores['auc score'] = \
metrics.roc_auc_score(y_test, m_test + 1, average='weighted')
melodiness_scores = {'train': train_scores, 'test': test_scores}
return melodiness_scores
def eval_models(stream1, stream2, predictor1, predictor2):
source1 = multiplex_streams([stream1, stream2], [0.5, 0.5], 1000)
source2 = multiplex_streams([stream1, stream2], [0.1, 0.9], 1000)
for source in source1, source2:
data = source.next()
y_est1 = predictor1(data['x']).values()[0].argmax(axis=1)
y_est2 = predictor2(data['x']).values()[0].argmax(axis=1)
print metrics.precision_recall_fscore_support(data['y'], y_est1)
print metrics.precision_recall_fscore_support(data['y'], y_est2)
def metric(self,tag,rank):
precision,recall,fbeta,support = precision_recall_fscore_support(self.purchase,tag)
print "precision of purchase:",precision
print "recall of purchase:",recall
PAN,a,b,c = precision_recall_fscore_support(self.rating,rank)
print "P @ N :",PAN
def GBDT(x_train,y_train,x_test,y_test):
#####GBDT
clf = GradientBoostingClassifier()
clf.fit(x_train,y_train)
predict_y = clf.predict(x_test)
auc_score=metrics.roc_auc_score(y_test,predict_y)
print 'GBDT auc_score=',auc_score
print metrics.precision_recall_fscore_support(y_test,predict_y)
return auc_score
def AdaBoost(x_train,y_train,x_test,y_test):
#####AdaBoostClassifier
clf = AdaBoostClassifier()
clf.fit(x_train,y_train)
predict_y = clf.predict(x_test)
auc_score=metrics.roc_auc_score(y_test,predict_y)
print 'AdaBoost auc_score=',auc_score
print metrics.precision_recall_fscore_support(y_test,predict_y)
return auc_score
def RF(x_train,y_train,x_test,y_test):
#####RF
clf = RandomForestClassifier()
clf.fit(x_train,y_train)
predict_y = clf.predict(x_test)
auc_score=metrics.roc_auc_score(y_test,predict_y)
print 'RF auc_score=',auc_score
print metrics.precision_recall_fscore_support(y_test,predict_y)
return auc_score
def get_scores(y_pred, y_true):
scores = precision_recall_fscore_support(y_true=y_true, y_pred=y_pred, labels=[0,1])
average = precision_recall_fscore_support(y_true=y_true,
y_pred=y_pred,
average="macro",
pos_label=None,
labels=[0, 1])
return scores, average
def NBgauss(x_train,y_train,x_test,y_test):
####Naive Bayes (Gaussian likelihood)
clf = GaussianNB()
clf.fit(x_train,y_train)
predict_y = clf.predict(x_test)
auc_score=metrics.roc_auc_score(y_test,predict_y)
print 'GaussianNB auc_score=',auc_score
print metrics.precision_recall_fscore_support(y_test,predict_y)
return auc_score
def fmeasure(gl,pl):
from sklearn.metrics import precision_recall_fscore_support
y_gold=np.array(gl)
y_pred=np.array(pl)
assert len(y_gold)==len(y_pred)
# print y_gold[:10]
# print y_pred[:10]
prf=precision_recall_fscore_support(y_gold, y_pred, average='macro',pos_label=None)
acc=precision_recall_fscore_support(y_gold, y_pred, average='micro',pos_label=None)
return prf,acc[0]
def evaluateClassifier(testFolderName,clfFolderName,th=0):
clf = Classifier(clfFolderName)
batchGen = batchGenerator(50000,testFolderName,'Test')
pred = []
exp = []
for _,X,Y in batchGen:
pred += clf.predict(X)
exp += Y
print precision_recall_fscore_support(exp, pred, average='micro')
return pred,exp
def evaluate_results(self,expected,predicted):
if(len(expected)!=len(predicted)):
print "Looks like some values were not predicted properly"
print "Precision, Accuracy, Recall, FScore"
print precision_recall_fscore_support(expected, predicted,average='binary')
cm=confusion_matrix(expected, predicted)
print "Confusion matrix"
print cm
def apply_Model(temp_data, selectModel):
data = {};
data['X_train_ceil'] = temp_data['X_train_ceil'];
data['X_test_ceil'] = temp_data['X_test_ceil'];
data['y_train_ceil'] = temp_data['y_train_ceil'];
data['y_test_ceil'] = temp_data['y_test_ceil'];
data['ind_ceil'] = temp_data['ind_ceil']
# feature selection for the floor
data['X_train_floor'] = temp_data['X_train_floor'];
data['X_test_floor'] = temp_data['X_test_floor'];
data['y_train_floor'] = temp_data['y_train_floor'] ;
data['y_test_floor'] = temp_data['y_test_floor'];
data['ind_floor'] = temp_data['ind_floor'];
if(selectModel==1):
print "OneVsRest";
classifier_floor = OneVsRestClassifier(LinearSVC(random_state=0)).fit(data['X_train_floor'], data['y_train_floor'])
classifier_ceil = OneVsRestClassifier(LinearSVC(random_state=0)).fit(data['X_train_ceil'], data['y_train_ceil'])
if(selectModel==2):
print "Decision Tree";
classifier_floor = tree.DecisionTreeClassifier().fit(data['X_train_floor'], data['y_train_floor'])
classifier_ceil = tree.DecisionTreeClassifier().fit(data['X_train_ceil'], data['y_train_ceil'])
if(selectModel==3):
print "Nearest Centroid";
classifier_floor = NearestCentroid().fit(data['X_train_floor'], np.ravel(data['y_train_floor']));
classifier_ceil = NearestCentroid().fit(data['X_train_ceil'], np.ravel(data['y_train_ceil']));
if(selectModel==4):
print "SGD Classifier";
classifier_floor = SGDClassifier(loss="hinge", penalty="l2").fit(data['X_train_floor'], np.ravel(data['y_train_floor']));
classifier_ceil = SGDClassifier(loss="hinge", penalty="l2").fit(data['X_train_ceil'], np.ravel(data['y_train_ceil']));
train_predict_floor = classifier_floor.predict(data['X_train_floor']);
conf_mat_floor = confusion_matrix(train_predict_floor,data['y_train_floor']);
train_predict_ceil = classifier_ceil.predict(data['X_train_ceil']);
conf_mat_ceil = confusion_matrix(train_predict_ceil, data['y_train_ceil'])
y_predict_ceil = classifier_ceil.predict(data['X_test_ceil'])
result_ceil = confusion_matrix(y_predict_ceil,data['y_test_ceil'])
y_predict_floor = classifier_floor.predict(data['X_test_floor'])
result_floor = confusion_matrix(y_predict_floor,data['y_test_floor'])
precision_floor, recall_floor, _, _ = precision_recall_fscore_support(data['y_test_floor'], y_predict_floor)
precision_ceil, recall_ceil, _, _ = precision_recall_fscore_support(data['y_test_ceil'], y_predict_ceil)
acc_ceil = result_ceil.trace()*100/result_ceil.sum();
acc_ceil_train = conf_mat_ceil.trace()*100/conf_mat_ceil.sum();
acc_floor = result_floor.trace()*100/result_floor.sum();
acc_floor_train = conf_mat_floor.trace()*100/conf_mat_floor.sum();
data['acc_ceil_train'] = acc_ceil_train;
data['acc_floor_train'] = acc_floor_train;
return data, acc_ceil, acc_floor, recall_ceil, recall_floor, result_ceil, result_floor, conf_mat_floor, conf_mat_ceil;
def main(self):
dirname = 'features/'
rlst = ['radar1','radar2','radar3','radar4']
for rname in rlst:
filelist = get_file_list(dirname+rname+'/')
trfile = dirname+rname+'/'+rname+'_train.csv'
tsfile = dirname+rname+'/'+rname+'_test.csv'
tl = False
trl = False
test = np.array([])
train = np.array([])
for name in filelist:
file = dirname+rname+'/'+name
if search('p3',name):
if tl==False:
test = np.genfromtxt(file, delimiter=',')
tl = True
else:
tst = np.genfromtxt(file, delimiter=',')
test = np.vstack((test,tst))
# print 'test'
# print test.shape
else:
if trl==False:
train = np.genfromtxt(file, delimiter=',')
trl = True
else:
tr = np.genfromtxt(file, delimiter=',')
train = np.vstack((train,tr))
# print 'train'
# print train.shape
np.savetxt(trfile, train, delimiter=",")
np.savetxt(tsfile, test, delimiter=",")
print rname
print '______'
clf = Pipeline([
('feature_selection', LinearSVC(C=0.01, penalty="l1", dual=False)),
('classification', RandomForestClassifier())
])
X = train[:,0:25]
y = train[:,26]
# print y
clf.fit(X, y)
y_ = clf.predict(test[:,0:25])
print y_
# print clf.score(test[:,0:25],test[:,26])
# clf.predict()
y = test[:,26]
cm = confusion_matrix(y, y_)
print cm
print cm.shape
print clf.score(test[:,0:25],test[:,26])
print precision_recall_fscore_support(y,y_)
def load_digits_small():
# The digits dataset
digits = load_digits()
# The data that we are interested in is made of 8x8 images of digits, let's
# have a look at the first 3 images, stored in the `images` attribute of the
# dataset. If we were working from image files, we could load them using
# pylab.imread. Note that each image must have the same size. For these
# images, we know which digit they represent: it is given in the 'target' of
# the dataset.
images_and_labels = list(zip(digits.images, digits.target))
for index, (image, label) in enumerate(images_and_labels[:4]):
plt.subplot(2, 4, index + 1)
plt.axis('off')
plt.imshow(image, cmap=plt.cm.gray_r, interpolation='nearest')
plt.title('Training: %i' % label)
# To apply a classifier on this data, we need to flatten the image, to
# turn the data in a (samples, feature) matrix:
n_samples = len(digits.images)
data = digits.images.reshape((n_samples, -1))
target = digits.target
# only take some classes of digits
c1 = 1
c2 = [2]
target_bin_index = [i for i, t in enumerate(target) if t == c1 or t in c2]
bin_samples = len(target_bin_index)
binary_data = data[target_bin_index]
binary_target = target[target_bin_index]
binary_target[binary_target == c1] = 1
binary_target[binary_target != c1] = -1
nu_gamma_precision = [0, 0, 0]
nu_range = [0.1*i for i in range(1,10)]
gamma_range = [0.0001, 0.0003, 0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1, 3, 10, 30]
for nu in nu_range:
for gamma in gamma_range:
clf = ocsvm.OCSVM("rbf", nu=nu, gamma=gamma)
clf.fit(binary_data[:bin_samples/2])
expected = binary_target[:bin_samples/ 2:]
predicted = clf.predict(binary_data[:bin_samples/2:])
precision, recall, f1score, _ = precision_recall_fscore_support(expected, predicted, average='binary')
if precision > nu_gamma_precision[2]:
nu_gamma_precision = [nu, gamma, precision]
print "nu_gamma_precision: %s" % nu_gamma_precision
clf = ocsvm.OCSVM("rbf", nu=nu_gamma_precision[0], gamma=nu_gamma_precision[1])
clf.fit(binary_data[:bin_samples/2])
expected = binary_target[:bin_samples/ 2:]
predicted = clf.predict(binary_data[:bin_samples/2:])
print("Confusion matrix:\n%s" % confusion_matrix(expected, predicted))
precision, recall, f1score, support = precision_recall_fscore_support(expected, predicted, average='binary')
print "precision: %s, recall: %s, f1-score: %s" % (precision, recall, f1score)
def print_analysis(op_csv_list):
y_true = []
y_pred = []
for file in op_csv_list:
file_csv = pd.read_csv(file)
for i, row in enumerate(file_csv.values):
y_true.append(row[2])
y_pred.append(row[4])
print confusion_matrix(y_true, y_pred)
print precision_recall_fscore_support(y_true, y_pred, average="micro")
def print_analysis(pos_op_csv_lst, neg_op_csv_lst):
files = pos_op_csv_lst[:-1] + neg_op_csv_lst[:-1]
y_true = []
y_pred = []
for file in files:
file_csv = pd.read_csv(file)
for i, row in enumerate(file_csv.values):
y_true.append(row[2])
y_pred.append(row[4])
print confusion_matrix(y_true, y_pred)
print precision_recall_fscore_support(y_true, y_pred, average='micro')
def test_precision_recall_f1_score_multiclass():
"""Test Precision Recall and F1 Score for multiclass classification task"""
y_true, y_pred, _ = make_prediction(binary=False)
# compute scores with default labels introspection
p, r, f, s = precision_recall_fscore_support(y_true, y_pred, average=None)
assert_array_almost_equal(p, [0.83, 0.33, 0.42], 2)
assert_array_almost_equal(r, [0.79, 0.09, 0.90], 2)
assert_array_almost_equal(f, [0.81, 0.15, 0.57], 2)
assert_array_equal(s, [24, 31, 20])
# averaging tests
ps = precision_score(y_true, y_pred, pos_label=1, average='micro')
assert_array_almost_equal(ps, 0.53, 2)
rs = recall_score(y_true, y_pred, average='micro')
assert_array_almost_equal(rs, 0.53, 2)
fs = f1_score(y_true, y_pred, average='micro')
assert_array_almost_equal(fs, 0.53, 2)
ps = precision_score(y_true, y_pred, average='macro')
assert_array_almost_equal(ps, 0.53, 2)
rs = recall_score(y_true, y_pred, average='macro')
assert_array_almost_equal(rs, 0.60, 2)
fs = f1_score(y_true, y_pred, average='macro')
assert_array_almost_equal(fs, 0.51, 2)
ps = precision_score(y_true, y_pred, average='weighted')
assert_array_almost_equal(ps, 0.51, 2)
rs = recall_score(y_true, y_pred, average='weighted')
assert_array_almost_equal(rs, 0.53, 2)
fs = f1_score(y_true, y_pred, average='weighted')
assert_array_almost_equal(fs, 0.47, 2)
assert_raises(ValueError, precision_score, y_true, y_pred,
average="samples")
assert_raises(ValueError, recall_score, y_true, y_pred, average="samples")
assert_raises(ValueError, f1_score, y_true, y_pred, average="samples")
assert_raises(ValueError, fbeta_score, y_true, y_pred, average="samples",
beta=0.5)
# same prediction but with and explicit label ordering
p, r, f, s = precision_recall_fscore_support(
y_true, y_pred, labels=[0, 2, 1], average=None)
assert_array_almost_equal(p, [0.83, 0.41, 0.33], 2)
assert_array_almost_equal(r, [0.79, 0.90, 0.10], 2)
assert_array_almost_equal(f, [0.81, 0.57, 0.15], 2)
assert_array_equal(s, [24, 20, 31])
def compareResult(result):
result.sort()
data = []
with open(data3result,'rb') as csvfile:
for line in csvfile:
if len(line.strip()) > 0:
data.append(int(line) - 1)
data = np.array(data)
# print data
# print result
print precision_recall_fscore_support(data, result, average='micro')
plt.savefig('/Users/Mehraveh/Desktop/class_probs_leave_one_out.png', dpi=120)
sorted_ptms = sorted(range(len(imp)), key=lambda k: imp[k], reverse=True)
sorted_ptms1 = [x + 1 for x in sorted_ptms]
final = pd.DataFrame(sorted_ptms, label[sorted_ptms1])
plt.figure(1, figsize=(6, 6))
plt.figure(figsize=(12, 8))
xx = range(n_features)
plt.bar(xx, imp[sorted_ptms], 1, color="blue")
plt.xticks(xx, label[sorted_ptms1], rotation='vertical')
plt.savefig('/Users/Mehraveh/Desktop/impFeatures.png', dpi=300)
predicted = y_pred
y_test = y
precision, recall, fscore, support = precision_recall_fscore_support(
y_test, predicted)
### What are the three unknown classes?
X_test = data.ix[data[0] == 0, :]
X_test = X_test.drop(0, 1)
X_test = np.asmatrix(X_test)
y_test = data[0]
y_test = y_test.ix[data[0] == 0]
y = y.ravel()
X_train = X
y_train = y
clf.fit(X_train, y_train)
y_pred = clf.predict(X_test)
class_probs_unknowns = clf.predict_proba(X_test)
prob_loo_unknowns = class_probs_unknowns
def train():
model = ResNet().to(device)
# optimizer = torch.optim.AdamW(model.parameters(), lr=LEARNING_RATE, amsgrad=False, weight_decay=0.0005)
# scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 5, gamma=0.5)
optimizer = torch.optim.SGD(model.parameters(),
lr=LEARNING_RATE,
momentum=0.9)
scheduler = torch.optim.lr_scheduler.StepLR(optimizer, 10, gamma=0.5)
total_step = len(train_loader)
plot_x = []
plot_acc = []
plot_recall = []
plot_precision = []
plot_f1 = []
plot_x_loss = []
plot_loss = []
for epoch in range(EPOCH):
torch.cuda.empty_cache()
for step, (images, labels) in enumerate(train_loader):
images = images.to(device)
labels = labels.to(device)
outputs = model(images)
loss = loss_func(outputs, labels)
optimizer.zero_grad()
loss.backward()
optimizer.step()
if step % 20 == 0:
print('Epoch [{}/{}], Step [{}/{}], Loss: {:.7f}'.format(
epoch + 1, EPOCH, step + 1, total_step, loss.item()))
if step % 50 == 0:
plot_x_loss.append(step / 235 + epoch)
plot_loss.append(loss.item())
scheduler.step()
true_target = torch.FloatTensor()
pre_target = torch.FloatTensor()
correct_vali = 0
total_vali = 0
for images_vali, target in vali_loader:
images_vali = images_vali.to(device)
target = target.to(device)
with torch.no_grad():
pred = model(images_vali)
maxk = max((1, 5))
target_resize = target.view(-1, 1)
_, predicted = pred.topk(maxk, 1, True, True)
loss = loss_func(pred, target)
correct_vali += torch.eq(predicted, target_resize).sum().item()
total_vali += target.size(0)
temp = predicted.narrow(1, 0, 1)
temp = torch.squeeze(temp, 1)
temp = temp.cpu()
temp = temp.view(-1).float()
pre_target = torch.cat((pre_target, temp), 0)
temp = target.cpu()
temp = temp.view(-1).float()
true_target = torch.cat((true_target, temp), 0)
score_precision, score_recall, score_f1, _ = precision_recall_fscore_support(
true_target, pre_target, average='macro')
plot_x.append(epoch + 1)
plot_acc.append(correct_vali / total_vali)
plot_f1.append(score_f1)
plot_precision.append(score_precision)
plot_recall.append(score_recall)
print(
'Epoch {} Accuracy {:.4f} %, loss {:.4f}\nf1-score {:.4f} recall {:.4f} precision {:.4f}'
.format(epoch + 1, 100 * correct_vali / total_vali, loss.item(),
score_f1, score_recall, score_precision))
torch.save(model.state_dict(), './model&img/model.pt')
plotScore(plot_x, plot_acc, plot_recall, plot_precision, plot_f1,
plot_x_loss, plot_loss)
y,
test_size=0.25,
random_state=0)
# 2. Scaling
from sklearn.preprocessing import StandardScaler
scaler_X = StandardScaler()
X_train = scaler_X.fit_transform(X_train)
X_test = scaler_X.transform(X_test)
# Machine: Classifier | NB: Gaussian Naive Bayes
from sklearn.naive_bayes import GaussianNB
classifier = GaussianNB()
classifier.fit(X_train, y_train)
# Predictions
y_pred = classifier.predict(X_test)
# Validating Predictions using Confusion Matrix
from sklearn.metrics import confusion_matrix
cm = confusion_matrix(y_test, y_pred)
print(cm)
from sklearn.metrics import precision_recall_fscore_support
prf = precision_recall_fscore_support(y_test, y_pred)
print('\t\t\t\t ZERO\t\t\tONE')
print('Precision\t:', prf[0] * 100)
print('Recall\t\t:', prf[1] * 100)
print('F1 Measure\t:', prf[2] * 100)
print('Support\t\t:', prf[3])
def train(self, train_indices=None, test_indices=None):
#Read data
if self.read_data_batch:
self.read_rgbd_data_batch(self.read_data_ratio)
elif train_indices is not None:
self.read_grasp_data_from_indices(train_indices, test_indices)
else:
self.read_rgbd_data()
#Create the variables.
self.create_net_var()
#Call the network building function.
self.cost_function()
#Initialize the variables
init = tf.global_variables_initializer()
saver = tf.train.Saver()
tf.get_default_graph().finalize()
logs_path = self.logs_path + '_train'
# op to write logs to Tensorboard
summary_writer = tf.summary.FileWriter(logs_path,
graph=tf.get_default_graph())
start_time = time.time()
train_costs = []
train_pred_errors = []
test_pred_errors = []
learn_rate = 0.001
# Launch the graph
with tf.Session() as sess:
sess.run(init)
# Training cycle
for epoch in range(self.training_epochs):
if epoch % self.rate_dec_epochs == 0 and epoch != 0:
learn_rate *= 0.1
if self.read_data_batch and epoch % self.read_batch_epochs == 0 and epoch != 0:
self.read_rgbd_data_batch(self.read_data_ratio)
#[pool_1_out, fc_config_out] = sess.run([self.max_pool_1, self.fc_config], feed_dict=self.feed_dict_func(True, learn_rate))
#print np.array(pool_1_out).shape, np.array(fc_config_out).shape
[_, cost_output, pred_error_output, train_summary_output, f2vis_train_output, labels_train_output, pred_train_output] = \
sess.run([self.optimizer, self.cost, self.pred_error, self.train_summary, self.feature_to_vis,
self.holder_labels, self.pred], feed_dict=self.feed_dict_func(True, learn_rate))
# Write logs at every iteration
summary_writer.add_summary(train_summary_output, epoch)
# Display logs per epoch step
train_costs.append(cost_output)
train_pred_errors.append(pred_error_output)
[
pred_error_test_output, pred_err_test_sum_output,
f2vis_output_test, labels_test_output, pred_test_output
] = sess.run([
self.pred_error_test, self.pred_err_test_sum,
self.feature_to_vis, self.holder_labels, self.pred
],
feed_dict=self.feed_dict_func(False))
test_pred_errors.append(pred_error_test_output)
summary_writer.add_summary(pred_err_test_sum_output, epoch)
if epoch % self.display_step == 0:
print 'epoch: ', epoch
print 'labels_train_output:', labels_train_output
print 'pred_train_output', pred_train_output
print 'labels_test_output:', labels_test_output
print 'pred_test_output', pred_test_output
print 'train_cost: ', cost_output
print 'pred_error_train: ', pred_error_output
print 'train pred std: ', np.std(pred_train_output[:, 0])
print 'pred_error_test: ', pred_error_test_output
print 'test pred std: ', np.std(pred_test_output[:, 0])
pred_train_output[pred_train_output > 0.5] = 1.
pred_train_output[pred_train_output <= 0.5] = 0.
train_prfc = precision_recall_fscore_support(
labels_train_output, pred_train_output)
print 'training precision, recall, fscore, support:'
print train_prfc
pred_test_output[pred_test_output > 0.5] = 1.
pred_test_output[pred_test_output <= 0.5] = 0.
test_prfc = precision_recall_fscore_support(
labels_test_output, pred_test_output)
print 'testing precision, recall, fscore, support:'
print test_prfc
if epoch % (10 * self.display_step) == 0:
tsne_vis.tsne_vis(
f2vis_train_output, labels_train_output[:, 0],
'../models/tsne/train_tsne_' + str(epoch) + '_gt.png')
tsne_vis.tsne_vis(
f2vis_output_test, labels_test_output[:, 0],
'../models/tsne/test_tsne_' + str(epoch) + '_gt.png')
tsne_vis.tsne_vis(
f2vis_train_output, pred_train_output[:, 0],
'../models/tsne/train_tsne_' + str(epoch) +
'_pred.png')
tsne_vis.tsne_vis(
f2vis_output_test, pred_test_output[:, 0],
'../models/tsne/test_tsne_' + str(epoch) + '_pred.png')
print("Optimization Finished!")
saver.save(sess, self.cnn_model_path)
fig, ax = plt.subplots(figsize=(8, 8))
ax.plot(np.linspace(1, self.training_epochs, self.training_epochs),
train_costs, 'r')
fig.savefig('../models/train_costs.png')
plt.clf()
plt.close(fig)
fig, ax = plt.subplots(figsize=(8, 8))
ax.plot(np.linspace(1, self.training_epochs, self.training_epochs),
train_pred_errors, 'b')
fig.savefig('../models/train_pred_errors.png')
plt.clf()
plt.close(fig)
fig, ax = plt.subplots(figsize=(8, 8))
ax.plot(np.linspace(1, self.training_epochs, self.training_epochs),
test_pred_errors, 'k')
fig.savefig('../models/test_pred_errors.png')
plt.clf()
plt.close(fig)
elapsed_time = time.time() - start_time
print 'Total training elapsed_time: ', elapsed_time
if FLAGS.early_stopping > 0 and epoch > FLAGS.early_stopping and cost_val[-1] > np.mean(cost_val[-(FLAGS.early_stopping+1):-1]):
print("Early stopping...")
break
print("Optimization Finished!")
# Best results
print('Best epoch:', best_epoch)
print("Test set results:", "cost=", "{:.5f}".format(best_cost),
"accuracy=", "{:.5f}".format(best_acc))
print("Test Precision, Recall and F1-Score...")
print(metrics.classification_report(labels, preds, digits=4))
print("Macro average Test Precision, Recall and F1-Score...")
print(metrics.precision_recall_fscore_support(labels, preds, average='macro'))
print("Micro average Test Precision, Recall and F1-Score...")
print(metrics.precision_recall_fscore_support(labels, preds, average='micro'))
'''
# For visualization
doc_vectors = []
for i in range(len(test_doc_embeddings)):
doc_vector = test_doc_embeddings[i]
doc_vector_str = ' '.join([str(x) for x in doc_vector])
doc_vectors.append(str(np.argmax(test_y[i])) + ' ' + doc_vector_str)
doc_embeddings_str = '\n'.join(doc_vectors)
with open('data/' + FLAGS.dataset + '_doc_vectors.txt', 'w'):
f.write(doc_embeddings_str)
'''
c_com_weight, early_type, conf_threshold_1_vs_2
])
writer.writerow([
dataset_name, method, "cost_avg_baseline",
cost_baseline / len(dt_final), c_miss_weight,
c_action_weight, c_postpone_weight, c_com_weight,
early_type, conf_threshold_1_vs_2
])
dt_final.prediction = dt_final.prediction.replace(2, 1)
#print(dt_final.prediction)
# calculate precision, recall etc.
prec, rec, fscore, _ = precision_recall_fscore_support(
dt_final.actual,
dt_final.prediction,
pos_label=1,
average="binary")
tn, fp, fn, tp = confusion_matrix(dt_final.actual,
dt_final.prediction).ravel()
# calculate earliness based on the "true alarms" only
tmp = dt_final[(dt_final.prediction == 1)
& (dt_final.actual == 1)]
earliness = (1 - ((tmp.prefix_nr - 1) / tmp.case_length))
tmp = dt_final[(dt_final.prediction == 1)]
earliness_alarms = (1 -
((tmp.prefix_nr - 1) / tmp.case_length))
writer.writerow([
dataset_name, method, "fire_delay", myopic_param,
def cifar100(config, path, param):
print("START CIFAR100")
use_cuda = config.general.use_cuda and torch.cuda.is_available()
torch.manual_seed(config.general.seed)
device = torch.device("cuda" if use_cuda else "cpu")
print("START TRAINING TARGET MODEL")
data_train_target = custum_CIFAR100(True,
0,
config,
'../data',
train=True,
download=True,
transform=transforms.Compose([
transforms.ToTensor(),
transforms.Normalize(
(0.5, 0.5, 0.5),
(0.5, 0.5, 0.5))
]))
data_test_target = custum_CIFAR100(True,
0,
config,
'../data',
train=False,
transform=transforms.Compose([
transforms.ToTensor(),
transforms.Normalize(
(0.5, 0.5, 0.5),
(0.5, 0.5, 0.5))
]))
criterion = nn.CrossEntropyLoss()
train_loader_target = torch.utils.data.DataLoader(
data_train_target, batch_size=config.learning.batch_size, shuffle=True)
test_loader_target = torch.utils.data.DataLoader(
data_test_target, batch_size=config.learning.batch_size, shuffle=True)
dataloaders_target = {
"train": train_loader_target,
"val": test_loader_target
}
dataset_sizes_target = {
"train": len(data_train_target),
"val": len(data_test_target)
}
model_target = Net_cifar100().to(device)
optimizer = optim.SGD(model_target.parameters(),
lr=config.learning.learning_rate,
momentum=config.learning.momentum)
exp_lr_scheduler = lr_scheduler.StepLR(
optimizer,
step_size=config.learning.decrease_lr_factor,
gamma=config.learning.decrease_lr_every)
model_target, best_acc_target, data_test_set, label_test_set, class_test_set = train_model(
model_target,
criterion,
optimizer,
exp_lr_scheduler,
dataloaders_target,
dataset_sizes_target,
num_epochs=config.learning.epochs)
np.savetxt(path + "/res_train_target_" + str(param) + ".csv",
best_acc_target)
plot_title = 'CIFAR100'
drawLossAcc(best_acc_target, plot_title, path)
print("START TRAINING SHADOW MODEL")
all_shadow_models = []
all_dataloaders_shadow = []
data_train_set = []
label_train_set = []
class_train_set = []
for num_model_sahdow in range(config.general.number_shadow_model):
criterion = nn.CrossEntropyLoss()
data_train_shadow = custum_CIFAR100(False,
num_model_sahdow,
config,
'../data',
train=True,
download=True,
transform=transforms.Compose([
transforms.ToTensor(),
transforms.Normalize(
(0.5, 0.5, 0.5),
(0.5, 0.5, 0.5))
]))
data_test_shadow = custum_CIFAR100(False,
num_model_sahdow,
config,
'../data',
train=False,
transform=transforms.Compose([
transforms.ToTensor(),
transforms.Normalize(
(0.5, 0.5, 0.5),
(0.5, 0.5, 0.5))
]))
train_loader_shadow = torch.utils.data.DataLoader(
data_train_shadow,
batch_size=config.learning.batch_size,
shuffle=True)
test_loader_shadow = torch.utils.data.DataLoader(
data_test_shadow,
batch_size=config.learning.batch_size,
shuffle=True)
dataloaders_shadow = {
"train": train_loader_shadow,
"val": test_loader_shadow
}
dataset_sizes_shadow = {
"train": len(data_train_shadow),
"val": len(data_test_shadow)
}
model_shadow = Net_cifar100().to(device)
optimizer = optim.SGD(model_shadow.parameters(),
lr=config.learning.learning_rate,
momentum=config.learning.momentum)
exp_lr_scheduler = lr_scheduler.StepLR(
optimizer,
step_size=config.learning.decrease_lr_factor,
gamma=config.learning.decrease_lr_every)
model_shadow, best_acc_sh, data_train_set_unit, label_train_set_unit, class_train_set_unit = train_model(
model_shadow,
criterion,
optimizer,
exp_lr_scheduler,
dataloaders_target,
dataset_sizes_target,
num_epochs=config.learning.epochs)
data_train_set.append(data_train_set_unit)
label_train_set.append(label_train_set_unit)
class_train_set.append(class_train_set_unit)
np.savetxt(
path + "/res_train_shadow_" + str(num_model_sahdow) + "_" +
str(param) + ".csv", best_acc_sh)
all_shadow_models.append(model_shadow)
all_dataloaders_shadow.append(dataloaders_shadow)
print("START GETTING DATASET ATTACK MODEL")
data_train_set = np.concatenate(data_train_set)
label_train_set = np.concatenate(label_train_set)
class_train_set = np.concatenate(class_train_set)
#data_test_set, label_test_set, class_test_set = get_data_for_final_eval([model_target], [dataloaders_target], device)
#data_train_set, label_train_set, class_train_set = get_data_for_final_eval(all_shadow_models, all_dataloaders_shadow, device)
data_train_set, label_train_set, class_train_set = shuffle(
data_train_set,
label_train_set,
class_train_set,
random_state=config.general.seed)
data_test_set, label_test_set, class_test_set = shuffle(
data_test_set,
label_test_set,
class_test_set,
random_state=config.general.seed)
#print(data_train_set.shape, data_test_set.shape)
print("dataset train size", len(label_train_set))
print("dataset test size", len(label_test_set))
print("START FITTING ATTACK MODEL")
model = lgb.LGBMClassifier(objective='binary',
reg_lambda=config.learning.ml.reg_lambd,
n_estimators=config.learning.ml.n_estimators)
model.fit(data_train_set, label_train_set)
y_pred_lgbm = model.predict(data_test_set)
precision_general, recall_general, _, _ = precision_recall_fscore_support(
y_pred=y_pred_lgbm, y_true=label_test_set, average="macro")
accuracy_general = accuracy_score(y_true=label_test_set,
y_pred=y_pred_lgbm)
precision_per_class, recall_per_class, accuracy_per_class = [], [], []
for idx_class, classe in enumerate(data_train_target.classes):
all_index_class = np.where(class_test_set == idx_class)
precision, recall, _, _ = precision_recall_fscore_support(
y_pred=y_pred_lgbm[all_index_class],
y_true=label_test_set[all_index_class],
average="macro")
accuracy = accuracy_score(y_true=label_test_set[all_index_class],
y_pred=y_pred_lgbm[all_index_class])
precision_per_class.append(precision)
recall_per_class.append(recall)
accuracy_per_class.append(accuracy)
print("END CIFAR100")
return (precision_general, recall_general, accuracy_general,
precision_per_class, recall_per_class, accuracy_per_class)
def run_classifer(X_train, s_train, y_train, X_test, s_test, y_test):
s_train = np.array(s_train) # samples x features
s_test = np.array(s_test)
num_labels = 15
batch_size = 100
stemmer = sb.SnowballStemmer('english')
swlist = sw.words('english')
swlist += [stemmer.stem(w) for w in swlist]
swlist += [
"'d", "'s", 'abov', 'ani', 'becaus', 'befor', 'could', 'doe', 'dure',
'might', 'must', "n't", 'need', 'onc', 'onli', 'ourselv', 'sha',
'themselv', 'veri', 'whi', 'wo', 'would', 'yourselv'
] #complained about not having these as stop words
pubs = [
'buzzfe', 'buzzf', 'npr', 'cnn', 'vox', 'reuter', 'breitbart', 'fox',
'guardian', 'review', 'theatlant'
]
punct = [
] #[':', '..', '“', '@', '%', ';', '→', ')', '#', '(', '*', '&', '[', ']', '…', '?','—', '‘', '$'] #gonna leave these in for now
swlist += pubs
swlist += punct
if sys.argv[4].lower() == 'true':
tkzr = StemTokenizer()
else:
tkzr = None
if sys.argv[5].lower() != 'true':
swlist = []
#what features are we using?
if sys.argv[7].lower() == 'word':
count_vect = CountVectorizer(stop_words=swlist, tokenizer=tkzr)
count_vect.fit(X_train)
X_train = count_vect.transform(X_train)
X_test = count_vect.transform(X_test)
tfidf_transformer = TfidfTransformer()
tfidf_transformer.fit(X_train)
X_train = tfidf_transformer.transform(X_train)
X_test = tfidf_transformer.transform(X_test)
elif sys.argv[7].lower() == 'topic':
count_vect = CountVectorizer(stop_words=swlist, tokenizer=tkzr)
count_vect.fit(X_train)
X_train = count_vect.transform(X_train)
X_test = count_vect.transform(X_test)
lda_model = LatentDirichletAllocation(n_components=10)
lda_model.fit(X_train)
X_train = lda_model.transform(X_train)
X_test = lda_model.transform(X_test)
elif sys.argv[7].lower() == 'style':
X_train = csr_matrix(s_train)
X_test = csr_matrix(s_test)
elif sys.argv[7].lower() == 'all':
count_vect = CountVectorizer(stop_words=swlist, tokenizer=tkzr)
count_vect.fit(X_train)
X_train = count_vect.transform(X_train)
X_test = count_vect.transform(X_test)
tfidf_transformer = TfidfTransformer()
tfidf_transformer.fit(X_train)
X_train_tf = tfidf_transformer.transform(X_train)
X_test_tf = tfidf_transformer.transform(X_test)
print(type(X_train_tf))
lda_model = LatentDirichletAllocation(n_components=10)
lda_model.fit(X_train)
X_train_lda = lda_model.transform(X_train)
X_test_lda = lda_model.transform(X_test)
print(type(X_train_lda))
X_train = csr_matrix(
sparse.hstack(
[X_train_tf,
csr_matrix(X_train_lda),
csr_matrix(s_train)]))
X_test = csr_matrix(
sparse.hstack(
[X_test_tf,
csr_matrix(X_test_lda),
csr_matrix(s_test)]))
print(type(X_train))
# sparse.save_npz("X_train" + sys.argv[6] + ".npz", X_train)
# sparse.save_npz("X_test" + sys.argv[6] + ".npz", X_test)
else:
sys.exit('unknown features')
encoder = LabelBinarizer()
encoder.fit(y_train)
y_train = encoder.transform(y_train)
y_test = encoder.transform(y_test)
# np.save('X_train.npy', X_train)
# np.save('X_test.npy', X_test)
# np.save('y_train.npy', y_train)
# np.save('y_test.npy', y_test)
# sparse.save_npz("y_train" + sys.argv[6] + ".npz", y_train)
# sparse.save_npz("y_test" + sys.argv[6] + ".npz", y_test)
# load everything back
# X_train = sparse.load_npz("X_train.npz")
input_dim = X_train.shape[1]
model = Sequential()
model.add(Dense(512, input_shape=(input_dim, )))
model.add(Activation('relu'))
model.add(Dropout(0.3))
model.add(Dense(512))
model.add(Activation('relu'))
model.add(Dropout(0.3))
model.add(Dense(num_labels))
model.add(Activation('softmax'))
model.summary()
model.compile(loss='categorical_crossentropy',
optimizer='adam',
metrics=['accuracy'])
history = model.fit(X_train,
y_train,
batch_size=batch_size,
epochs=5,
verbose=1,
validation_split=0.1)
# model.model.save(sys.argv[6] + '.h5')
# X_train = np.load('X_train.npy')
# X_test = np.load('X_test.npy')
# y_train = np.load('y_train.npy')
# y_test = np.load('y_test.npy')
# model = keras.models.load_model(sys.argv[6] + '.h5')
score = model.evaluate(X_test, y_test, batch_size=batch_size, verbose=1)
print('Test accuracy:', score[1])
y_pred = model.predict(X_test, batch_size=batch_size, verbose=1)
predicted = np.argmax(y_pred, axis=1)
p, r, fs, s = precision_recall_fscore_support(np.argmax(y_test, axis=1),
predicted)
print(p, r, fs, s)
if (acc > max_acc):
cl_model.saver.save(cl_model.sess,
cl_save_path + 'cl_model.ckpt',
global_step=epoch_count)
max_acc = acc
# Create error case
# print("Create error case")
test_avg_loss, predict_set, true_set, train_prob = cl_model.predict(
train_batch_x, train_batch_y)
# print("Train true_set_len: ", len(true_set))
# print(train_prob[:10])
assert len(true_set) == len(train_prob)
acc = accuracy_score(true_set, predict_set)
performance = metrics.precision_recall_fscore_support(true_set,
predict_set,
average='binary')
precision = performance[0]
recall = performance[1]
f1 = performance[2]
print(
"Train test loss: {} accuracy: {} precision: {} recall: {} f1: {}".
format(test_avg_loss, round(acc, 6), round(precision, 6),
round(recall, 6), round(f1, 6)))
t1p0 = 0
t1p0_list = []
t1p0_prob = []
t0p1 = 0
t0p1_list = []
def train_10_fold_balanced(dv, tags):
Xl, Xr, y = load_dataset(dv, tags)
Xl, Xr, y = shuffle(Xl, Xr, y, random_state=0)
skf = StratifiedKFold(n_splits=10)
avg_accuracy = 0.
avg_recall = 0.
avg_precision = 0.
avg_f1_score = 0.
fold_index = 0
for train_idx, test_idx in skf.split(Xl, y):
t_beg = time.clock()
print ('*' * 40 + str(fold_index) + '*' * 40)
fold_path = os.path.join(vector_name, str(fold_index))
if os.path.exists(vector_name) is not True:
os.mkdir(vector_name)
if os.path.exists(fold_path) is not True:
os.mkdir(fold_path)
train_X_left = Xl[train_idx]
train_X_right = Xr[train_idx]
train_Y = y[train_idx]
train_X_left, train_X_right, train_Y = shuffle(train_X_left,
train_X_right, train_Y, random_state=0)
test_X_left = Xl[test_idx]
test_X_right = Xr[test_idx]
test_Y = y[test_idx]
validate_X_left = test_X_left[:256]
validate_X_right = test_X_right[:256]
validate_Y = test_Y[:256]
X_left = Input(shape=(bin_vec_dim, ))
X_right = Input(shape=(bin_vec_dim, ))
predictions = classification(X_left, X_right)
model = Model(inputs=[X_left, X_right], outputs=predictions)
model.compile(optimizer=K.optimizers.adam(lr=0.001),
loss=K.losses.binary_crossentropy,
metrics=['accuracy'])
model.fit([train_X_left, train_X_right], train_Y,
epochs=4,verbose=1, batch_size=batch_size)
t_end = time.clock()
print('Time cost: %.2f' % (t_end - t_beg))
model.save(filepath=os.path.join(fold_path, 'model.ckpt'))
y_pred = model.predict([test_X_left, test_X_right], verbose=1, batch_size=batch_size)
y_pred = np.round(y_pred)
accuracy = accuracy_score(test_Y, y_pred)
precision, recall, fscore, _ = precision_recall_fscore_support(test_Y,
y_pred, average='binary')
print("accuracy: %.4f, recall: %.4f, "
"precision: %.4f, f1 score: %.4f\n" % (
accuracy, recall, precision, fscore))
fout.write('*' * 80 + '\n')
fout.write('Fold %d:\n' % (fold_index))
fout.write('Time cost: %.2f\n' % (t_end - t_beg))
fout.write("Fold index: %d, accuracy: %.4f, recall: %.4f, "
"precision: %.4f, f1 score: %.4f\n" % (
fold_index, accuracy, recall, precision, fscore))
fout.flush()
avg_accuracy += accuracy
avg_precision += precision
avg_recall += recall
avg_f1_score += fscore
print('*' * 80)
fold_index += 1
avg_accuracy /= 10.0
avg_precision /= 10.0
avg_recall /= 10.0
avg_f1_score /= 10.0
print('Avg accuracy: %.4f, avg recall: %.4f, avg precision: %.4f, avg f1 '
'score: %.4f' % (
avg_accuracy, avg_recall, avg_precision, avg_f1_score))
fout.write('*' * 80 + '\n')
fout.write(
'Avg accuracy: %.4f, avg recall: %.4f, avg precision: %.4f, avg f1 '
'score: %.4f\n' % (avg_accuracy, avg_recall, avg_precision, avg_f1_score))
# View The Accuracy Of Our Limited Feature (2 Features) Model
print("Accuracy of limited features : ", end=" ")
print(accuracy_score(ytest, y_important_pred))
print()
print()
##print("training accuracy: {}".format(100*clf.score(xtrain,ytrain)))
##print("testing accuracy: {}".format(100*clf.score(xtest,ytest)))
y_true = ytest
y_pred = clf.predict(xtest)
from sklearn.metrics import precision_recall_fscore_support, accuracy_score
accuracy = accuracy_score(y_true, y_pred)
precision, recall, f1_score, _ = precision_recall_fscore_support(
y_true, y_pred, average='binary')
print("For full feature dataset...")
print()
print("Accuracy: ", accuracy)
print("Precision: ", precision)
print("Recall: ", recall)
print("F1 score: ", f1_score)
print()
print()
accuracy_imp = accuracy_score(y_true, y_important_pred)
precision_imp, recall_imp, f1_score_imp, _ = precision_recall_fscore_support(
y_true, y_important_pred, average='binary')
print("For limites feature dataset...")
def scores(key, paths, config):
import mapreduce
print(key)
values = [mapreduce.OutputCollector(p) for p in paths]
values = [item.load() for item in values]
y_true = [item["y_true"].ravel() for item in values]
y_pred = [item["y_pred"].ravel() for item in values]
y_true = np.concatenate(y_true)
y_pred = np.concatenate(y_pred)
prob_pred = [item["proba_pred"].ravel() for item in values]
prob_pred = np.concatenate(prob_pred)
p, r, f, s = precision_recall_fscore_support(y_true, y_pred, average=None)
auc = roc_auc_score(y_true, prob_pred) #area under curve score.
betas = np.hstack([item["beta"] for item in values]).T
# threshold betas to compute fleiss_kappa and DICE
betas_t = np.vstack([
array_utils.arr_threshold_from_norm2_ratio(betas[i, :], .99)[0]
for i in range(betas.shape[0])
])
#Compute pvalue
success = r * s
success = success.astype('int')
prob_class1 = np.count_nonzero(y_true) / float(len(y_true))
pvalue_recall0_true_prob = binom_test(success[0],
s[0],
1 - prob_class1,
alternative='greater')
pvalue_recall1_true_prob = binom_test(success[1],
s[1],
prob_class1,
alternative='greater')
pvalue_recall0_unknwon_prob = binom_test(success[0],
s[0],
0.5,
alternative='greater')
pvalue_recall1_unknown_prob = binom_test(success[1],
s[1],
0.5,
alternative='greater')
pvalue_recall_mean = binom_test(success[0] + success[1],
s[0] + s[1],
p=0.5,
alternative='greater')
scores = OrderedDict()
try:
a, l1, l2, tv = [float(par) for par in key.split("_")]
scores['a'] = a
scores['l1'] = l1
scores['l2'] = l2
scores['tv'] = tv
left = float(1 - tv)
if left == 0: left = 1.
scores['l1_ratio'] = float(l1) / left
except:
pass
scores['recall_0'] = r[0]
scores['recall_1'] = r[1]
scores['recall_mean'] = r.mean()
scores["auc"] = auc
scores['pvalue_recall0_true_prob_one_sided'] = pvalue_recall0_true_prob
scores['pvalue_recall1_true_prob_one_sided'] = pvalue_recall1_true_prob
scores[
'pvalue_recall0_unknwon_prob_one_sided'] = pvalue_recall0_unknwon_prob
scores[
'pvalue_recall1_unknown_prob_one_sided'] = pvalue_recall1_unknown_prob
scores['pvalue_recall_mean'] = pvalue_recall_mean
scores['prop_non_zeros_mean'] = float(np.count_nonzero(betas_t)) / \
float(np.prod(betas.shape))
scores['param_key'] = key
return scores
X = (X - np.min(X, 0)) / (np.max(X, 0) + 0.0001) # 0-1 scaling
classifier = joblib.load(PATH + '/data/' + sys.argv[1] + '/pkl/' +
sys.argv[1] + '_classifier.pkl')
logistic_classifier = joblib.load(PATH + '/data/' + sys.argv[1] + '/pkl/' +
sys.argv[1] + '_logistic_classifier.pkl')
CP = classifier.predict(X)
LCP = logistic_classifier.predict(X)
retCP = []
retLCP = []
for i in range(len(CP)):
retCP.append(reverse_dic[CP[i]])
retLCP.append(reverse_dic[LCP[i]])
cps, crs, cfs, cs = metrics.precision_recall_fscore_support(
CP, Y, average='weighted')
lcps, lcrs, lcfs, lcs = metrics.precision_recall_fscore_support(
LCP, Y, average='weighted')
test = open(
PATH + '/data/' + sys.argv[1] + '/test/' + sys.argv[2] + '.test', 'w')
json.dump(
{
'ntb': {
'samples': samples,
'score': classifier.score(X, Y),
'classifier_recall_score': crs,
'classifier_precision_score': cps,
'classifier_f1_score': cfs,
'logistic_classifier_recall_score': lcrs,
print(df_result)
# Making the Confusion Matrix
from sklearn.metrics import confusion_matrix,accuracy_score, precision_score
from sklearn.metrics import precision_recall_fscore_support
cm = confusion_matrix(y, y_pred_all)
print(cm)
print(cm[1][1])
#accuracy -number of instance correctly classified
acsc = accuracy_score(y, y_pred_all)
print("Accuracy:")
print(acsc)
#precision, recall, fscore, support
precision, recall, fscore, support = precision_recall_fscore_support(y, y_pred_all,average='weighted')
df_metrics = pd.DataFrame([[acsc, precision, recall, fscore]],
index=[0],
columns=['accuracy','precision', 'recall', 'fscore'])
print(df_metrics)
print("precision:")
print(precision)
print("recall:")
print(recall)
print("fscore:")
print(fscore)
combined_training_set_target = combined_training_set.iloc[:, -1]
testing_set_features = data_test.iloc[:, :-1]
testing_set_target = data_test.iloc[:, -1]
# Define the classifier used
classifier = GaussianNB()
# Fit accordingly to our combined training and validation set
classifier.fit(combined_training_set_features, combined_training_set_target)
# Finally, predict the test set the best estimator that GridSearch has scored during training
test_prediction = classifier.predict(testing_set_features)
# Metrics
test_confusion = confusion_matrix(testing_set_target, test_prediction)
p1, r1, f1, _ = precision_recall_fscore_support(testing_set_target, test_prediction, zero_division='warn')
p2, r2, f2, _ = precision_recall_fscore_support(testing_set_target, test_prediction, average='weighted')
p3, r3, f3, _ = precision_recall_fscore_support(testing_set_target, test_prediction, average='macro')
test_accuracy = accuracy_score(testing_set_target, test_prediction)
# Write results and metrics to file
file = open(output_results_file_path, 'w', encoding='utf8')
writer = csv.writer(file, quotechar='"', quoting=csv.QUOTE_ALL, lineterminator='\n')
count = 1
for x in test_prediction:
writer.writerow([count, x])
count += 1
writer.writerow("")
writer.writerow("")
writer.writerow(["Confusion Matrix"])
letter_header = letters.copy()
for x in args.FEAT:
if x not in feats_sizes:
raise Exception("Unidentified feat: {}".format(x))
folddir = args.folddir
outf = args.outf
model_name = args.modelname
max_len = args.max_len
epochs = args.epochs
batch_size = args.batch_size
FEAT = args.FEAT
VOCAB_SIZE = sum(map(feats_sizes.__getitem__, FEAT))
model = create_model()
preds = train_and_evaluate_model(model,
data_train_gen=Generator(
f"{folddir}/train", get_feats),
data_eval_folder=f"{folddir}/eval",
data_test_folder=f"{folddir}/test",
model_name=model_name,
epochs=epochs)
y_test, y_pred = preds
print(",".join(FEAT))
p, r, f1, _ = precision_recall_fscore_support(y_test, y_pred, average='macro')
print(" ==== Results ==== ")
print(f"{p}\t{r}\t{f1}")
print("=" * 20)
np.save(outf, preds)
loss, accuracy = model.evaluate(
{
'claims': test_claims,
'sentences': test_sents
}, test_model.test_data["label"])
print("test loss ", loss)
print("test accuracy ", accuracy)
batch_size = 32
y_pred = (np.asarray(
model.predict({
'claims': test_claims,
'sentences': test_sents
},
batch_size=batch_size))).round()
print(
"score of lstm ",
precision_recall_fscore_support(test_model.test_data["label"],
y_pred,
average="binary"))
f = open("lstm_sent_ret_results.txt", "w")
f.write("accuracy " + str(accuracy))
f.write("loss ", +str(loss))
f.write(
"precision recall f1 ", +str(
precision_recall_fscore_support(
test_model.test_data["label"], y_pred, average="binary")))
f.close()
def metric_evaluate(y_true, y_pred, model_name):
F1_score = f1_score(y_true, y_pred, average='macro')
acc = accuracy_score(y_true=y_true, y_pred=y_pred)
recall_score_ = recall_score(y_true, y_pred, average='macro')
cohen_kappa_score_ = cohen_kappa_score(y_true, y_pred)
confmat = confusion_matrix(y_true=y_true, y_pred=y_pred)
clf_report = classification_report(y_true, y_pred)
report = np.array(precision_recall_fscore_support(y_true, y_pred))
class1_report = report[:, 1]
class1_metric = list(class1_report[:-1])
metrics = []
def plot_confusion_matrix(cm,
classes,
normalize=False,
title='Confusion matrix',
cmap=plt.cm.Blues):
"""
This function prints and plots the confusion matrix.
Normalization can be applied by setting `normalize=True`.
"""
import itertools
if normalize:
cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
print("Normalized confusion matrix")
else:
print('Confusion matrix, without normalization')
print(cm)
plt.figure()
plt.imshow(cm, interpolation='nearest', cmap=cmap)
plt.title(title)
plt.colorbar()
tick_marks = np.arange(len(classes))
plt.xticks(tick_marks, classes, rotation=45)
plt.yticks(tick_marks, classes)
fmt = '.2f' if normalize else 'd'
thresh = cm.max() / 2.
for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
plt.text(j,
i,
format(cm[i, j], fmt),
horizontalalignment="center",
color="white" if cm[i, j] > thresh else "black")
plt.ylabel('True label')
plt.xlabel('Predicted label')
plt.tight_layout()
# Plot non-normalized confusion matrix
class_names = ['0', '1']
plot_confusion_matrix(confmat,
classes=class_names,
title='Confusion matrix, without normalization')
plt.savefig(model_name)
#plt.show()
#plt.close()
#print('f1_score:',F1_score,'ACC_score:',acc,'recall:',recall_score_)
#print ('cohen kappa score:', cohen_kappa_score_)
print('\n----class report ---:\n', clf_report)
metrics.extend([F1_score, acc, recall_score_, cohen_kappa_score_])
metrics.extend(class1_metric)
columns = [
'f1-score', 'acc', 'recall', 'cohen_kappa_score', 'precision-c1',
'recall-c1', 'f-score-c1'
]
metrics = dict(zip(columns, metrics))
return metrics
def save_results_to_csv(self, report_lines, all_iter_results):
filename = self._results_path
header = 'Total_Observations,Train_Size,Graph,Heuristic,K,Start_Date,End_Date,Duration,' \
'Train_Nodes_in_Graph,Test_Nodes_in_Graph,Number_of_Edges,Number_of_Random' \
'Guesses,Memory Consumption,Iteration'
if not self._results_averaged_on_report:
header += ', Precision Class 0, Precision Class 1, Recall Class 0, Recall Class 1, F1 Class 0, ' \
'F1 Class 1, Support Class 0, Support Class 1 '
else:
header += ',Precision,Recall,F1,Support,'
header += 'AUC,Avg_Precision,Avg_Recall,Avg_F1,Avg_AUC,Avg_Support,Std_Dev_Precision,' \
'STD_Dev_Recall,Sev_Dev_F1,Std_Dev_AUC,Std_Dev_Support \n'
with open(filename, "w") as text_file:
text_file.write(header)
for iteration, total_authors, train_size, graph_name, heuristic, k, prediction, actual, start_time, end_time, duration,\
train_nodes_in_graph, test_nodes_in_graph, num_random_guesses, memory_usage, num_edges in report_lines:
line = str(total_authors) + ',' + str(train_size) + ',' + str(graph_name) + ',' + heuristic + ',' \
+ str(k) + ',' + str(start_time) + ',' + str(end_time) + ',' + str(duration) + ',' \
+ str(train_nodes_in_graph) + ',' + str(test_nodes_in_graph) + ',' \
+ str(num_edges) + ',' + str(num_random_guesses) + ',' + str(memory_usage)+' ,'+str(iteration)
if not self._results_averaged_on_report:
results = precision_recall_fscore_support(
actual, prediction)
prec_a = results[0][0]
prec_b = results[0][1]
rec_a = results[1][0]
rec_b = results[1][1]
f1_a = results[2][0]
f1_b = results[2][1]
support_a = results[3][0]
support_b = results[3][1]
line += ','+str(prec_a) + ',' + str(prec_b) + ',' + str(rec_a) + ',' + str(rec_b) \
+ ',' + str(f1_a) + ', ' + str(f1_b) + ',' + str(support_a) + ',' + str(support_b)
else:
results = precision_recall_fscore_support(actual,
prediction,
average='binary')
prec = results[0]
rec = results[1]
f1 = results[2]
support = results[3]
line += ',' + str(prec) + ',' + str(rec) + ',' + str(
f1) + ', ' + str(support)
auc = roc_auc_score(actual, prediction)
avg_precision = np.mean(all_iter_results[(k, 'precision')])
avg_recall = np.mean(all_iter_results[(k, 'recall')])
avg_f1 = np.mean(all_iter_results[(k, 'f1')])
avg_auc = np.mean(all_iter_results[(k, 'auc')])
if all_iter_results[(k, 'support')] is not None \
and len(all_iter_results[(k, 'support')]) > 0:
avg_support = np.mean(all_iter_results[(k, 'support')])
stddev_support = np.std(all_iter_results[(k, 'support')])
else:
avg_support = 0.0
stddev_support = 0.0
stddev_precision = np.std(all_iter_results[(k, 'precision')])
stddev_recall = np.std(all_iter_results[(k, 'recall')])
stddev_f1 = np.std(all_iter_results[(k, 'f1')])
stddev_auc = np.std(all_iter_results[(k, 'auc')])
line += ',' + str(auc) + ',' + str(avg_precision) + ',' + str(avg_recall) + ',' + str(avg_f1) + ', '\
+ str(avg_auc) + ',' + str(avg_support) + ', '+str(stddev_precision)+', '+ str(stddev_recall)+', '+\
str(stddev_f1)+', '+str(stddev_auc)+','+str(stddev_support) + ' \n'
text_file.write(line)
('PreProcess', TextPreprocess()),
('features', StatFeatureVectorizer()
), #used to reduce dimensions pick this one or 'dense'
# ('dense', DenseTransformer()), #comment out if svd performed
('clf', LogisticRegression(n_jobs=-1, solver='lbfgs'))
])
#%% Running Logistic Regression cross validation
# LR_scores = cross_val_score(LR_pipeline, X_train, y_train, cv=2, scoring="f1_macro")
# print(LR_scores)
#error analysis
print('\n-------------- TFIDF LR -----------------')
CVLR = GridSearchCV(LR_pipeline, {}, scoring="f1_macro", cv=2, n_jobs=1)
CVLR.fit(X_train, y_train)
y_pred = CVLR.predict(X_valid)
printed = 0
for (i, label) in enumerate(y_pred):
if label != y_valid.iloc[i]:
if printed < 20:
print(X_valid.iloc[i].encode("utf-8"))
print('predicted: %s' % (y_pred[i]))
print('labeled: %s' % (y_valid.iloc[i]))
printed += 1
results = precision_recall_fscore_support(y_valid, y_pred, average='macro')
print('\n\nprecision, recall, f1:')
print(results) # precision , recall, fscore
#https://www.kaggle.com/evanmiller/pipelines-gridsearch-awesome-ml-pipelines
def run_epoch(self,
input_batches,
input_set,
input_count,
topic_params,
session,
input_labels=None,
optimizer=None):
loss_sum = 0.0
ppx_sum = 0.0
kld_sum = 0.0
supervised_loss_sum = 0.0
word_count = 0
doc_count = 0
doc_pred = []
doc_labels = []
for idx_batch in input_batches:
data_batch, count_batch, mask, label_batch = utils.fetch_data(
input_set,
input_count,
idx_batch,
self.vocab_size,
topic_params,
labels=input_labels)
#import pdb; pdb.set_trace()
input_feed = {
self.x.name: data_batch,
self.mask.name: mask,
self.label_ids.name: label_batch
}
if not optimizer is None:
_, (loss, kld, supervised_loss, prob) = session.run(
(optimizer, [
self.unsupervised_loss, self.kld, self.supervised_loss,
self.sup_prob
]), input_feed)
else:
loss, kld, supervised_loss, prob = session.run([
self.unsupervised_loss, self.kld, self.supervised_loss,
self.sup_prob
], input_feed)
if topic_params.multilabel:
prob_arr = np.asarray(prob)
multilabel_pred = np.where(prob_arr >= 0.5, 1, 0)
pred = np.ndarray.tolist(multilabel_pred)
else:
pred = np.argmax(prob, axis=1)
assert len(pred) == len(label_batch) == len(mask)
for i in range(len(mask)):
if mask[i] != 0.0:
doc_pred.append(pred[i])
doc_labels.append(label_batch[i])
loss_sum += np.sum(loss)
kld_sum += np.sum(kld) / np.sum(mask)
supervised_loss_sum += np.sum(supervised_loss) / np.sum(mask)
word_count += np.sum(count_batch)
# to avoid nan error
count_batch = np.add(count_batch, 1e-12)
# per document loss
ppx_sum += np.sum(np.divide(loss, count_batch))
doc_count += np.sum(mask)
assert -1 not in doc_labels
if topic_params.multilabel:
doc_labels = np.asarray(doc_labels)
doc_pred = np.asarray(doc_pred)
print_macro_prec, print_macro_recall, print_macro_f1_score, _ = precision_recall_fscore_support(
doc_labels, doc_pred, average="macro")
#print_micro_prec, print_micro_recall, print_micro_f1_score, _ = precision_recall_fscore_support(doc_labels, doc_pred, average = "micro")
print_acc = accuracy_score(doc_labels, doc_pred)
print_sup_loss = supervised_loss_sum / len(input_batches)
print_ppx = np.exp(loss_sum / word_count)
print_ppx_perdoc = np.exp(ppx_sum / doc_count)
print_kld = kld_sum / len(input_batches)
return print_ppx, print_ppx_perdoc, print_kld, print_sup_loss, print_macro_prec, print_macro_recall, print_macro_f1_score, print_acc
def test(self,
train_indices=None,
test_indices=None,
seen_or_unseen=None,
k_fold=None):
#Read data
if test_indices is not None:
self.read_grasp_data_from_indices(train_indices, test_indices)
else:
self.read_rgbd_data()
#Create the variables.
self.create_net_var()
#Call the network building function.
self.cost_function()
#Initialize the variables
init = tf.global_variables_initializer()
saver = tf.train.Saver()
tf.get_default_graph().finalize()
logs_path = self.logs_path + '_test'
start_time = time.time()
learn_rate = 0.001
#learn_rate = 0.0001
with tf.Session() as sess:
# Test model
saver.restore(sess, self.cnn_model_path)
pred_score_output = np.array([]).reshape(0, self.classes_num)
f2vis_output = np.zeros([
self.testing_samples_num,
self.fc1_neurons_num + self.fc2_neurons_num
])
for i in xrange(
np.ceil(float(self.testing_samples_num) /
self.batch_size).astype(int)):
[b_grasp, w_grasp, f2vis_batch_output, pred_batch_output
] = sess.run([
self.biases_grasp['out'], self.weights_grasp['out'],
self.feature_to_vis, self.pred
],
feed_dict=self.feed_dict_func(False, .0, i))
pred_score_output = np.concatenate(
(pred_score_output, pred_batch_output))
start_idx = i * self.batch_size
f2vis_output[start_idx:start_idx +
len(f2vis_batch_output), :] = f2vis_batch_output
print 'pred_score_output: ', pred_score_output
print 'self.testing_labels: ', self.testing_labels
pred_output = np.copy(pred_score_output)
pred_output[pred_output > 0.4] = 1.
pred_output[pred_output <= 0.4] = 0.
print 'binary pred_output: ', pred_output
print 'pred_output.shape: ', pred_output.shape
print 'self.testing_labels.shape: ', self.testing_labels.shape
pred_errors_all = np.abs(pred_output - self.testing_labels)
avg_pred_error_test = np.mean(pred_errors_all)
print('avg_pred_error_test:', avg_pred_error_test)
#print f2vis_output.shape, self.testing_labels.shape
#tsne_vis.tsne_vis(f2vis_output, self.testing_labels[:, 0], '../models/tsne/test_tsne_gt.png', True)
#tsne_vis.tsne_vis(f2vis_output, pred_output[:, 0], '../models/tsne/test_tsne_pred.png', True)
fig, ax = plt.subplots(figsize=(8, 8))
ax.plot(np.linspace(1, self.testing_samples_num,
self.testing_samples_num),
self.testing_labels,
'ro',
label='gt labels')
ax.plot(np.linspace(1, self.testing_samples_num,
self.testing_samples_num),
pred_output,
'bo',
label='pred labels')
ax.plot(np.linspace(1, self.testing_samples_num,
self.testing_samples_num),
pred_errors_all,
'k*',
label='pred errors')
#plt.legend(handles=[gt_plot, pred_plot, error_plot])
legend = ax.legend(loc='upper right', shadow=True)
frame = legend.get_frame()
frame.set_facecolor('0.90')
fig.savefig('../models/labels_test.png')
plt.clf()
plt.close(fig)
prfc = precision_recall_fscore_support(self.testing_labels,
pred_output)
print 'precision, recall, fscore, support:'
print prfc
gt_labels_file_name = '../cross_val/' + seen_or_unseen + '/gt_labels_fold_' + k_fold + '.txt'
np.savetxt(gt_labels_file_name, self.testing_labels)
pred_score_file_name = '../cross_val/' + seen_or_unseen + '/pred_score_fold_' + k_fold + '.txt'
np.savetxt(pred_score_file_name, pred_score_output)
roc_fig_name = '../cross_val/' + seen_or_unseen + '/config_net_roc' + k_fold + '.png'
plot_roc_pr_curve.plot_roc_curve(self.testing_labels,
pred_score_output, roc_fig_name)
pr_fig_name = '../cross_val/' + seen_or_unseen + '/config_net_pr' + k_fold + '.png'
plot_roc_pr_curve.plot_pr_curve(self.testing_labels, pred_score_output,
pr_fig_name)
elapsed_time = time.time() - start_time
print 'Total testing elapsed_time: ', elapsed_time
def eval_performance(y_true, y_pred, tagnames):
pre, rec, f1, support = metrics.precision_recall_fscore_support(y_true, y_pred)
print "=== Performance (omitting 'O' class) ==="
print "Mean precision: %.02f%%" % (100*sum(pre[1:] * support[1:])/sum(support[1:]))
print "Mean recall: %.02f%%" % (100*sum(rec[1:] * support[1:])/sum(support[1:]))
print "Mean F1: %.02f%%" % (100*sum(f1[1:] * support[1:])/sum(support[1:]))
def compute_results(self, heuristic, graph_name, iteration, k, predictions,
actual, title, duration, results,
anchors_percent_size):
print(title)
if heuristic is not None:
print('Link prediciton model: ' + heuristic)
print(' K: ' + str(k))
print(' Similarity Function: ' + graph_name)
print(' Iteration: ' + str(iteration))
print(' Duration: ' + str(duration))
if len(actual) > 0 and len(predictions) > 0:
#report = classification_report(actual, predictions, target_names=[' 0 good actor', '1 bad actor'])
report = classification_report(actual, predictions)
print(report)
print('AUC ' + str(roc_auc_score(actual, predictions)))
else:
print('No data')
auc = roc_auc_score(actual, predictions)
results[(k, 'auc')] += [auc]
targeted_class_field_name = self._targeted_class_field_name[0]
#current_classifier = self._results_dict[targeted_class_field_name][graph_name][k][heuristic]
self._result_container.set_result(auc, PerformanceMeasures.AUC,
targeted_class_field_name,
graph_name, k, heuristic,
anchors_percent_size) #original
#self._result_container.set_result(auc, PerformanceMeasures.AUC, targeted_class_field_name, graph_name, k, heuristic) #by Lior
#current_classifier[PerformanceMeasures.AUC] += auc
if not self._results_averaged_on_report:
performance = precision_recall_fscore_support(actual, predictions)
results[(k, 'precision')] += [
float((performance[0][0] + performance[0][1]) / 2)
]
results[(k, 'recall')] += [
float((performance[1][0] + performance[1][1]) / 2)
]
results[(k, 'f1')] += [
float((performance[2][0] + performance[2][1]) / 2)
]
results[(k, 'support')] += [
float((performance[3][0] + performance[3][1]) / 2)
]
else:
performance = precision_recall_fscore_support(actual,
predictions,
average='binary')
precision = performance[0]
results[(k, 'precision')] += [precision]
self._result_container.set_result(precision,
PerformanceMeasures.PRECISION,
targeted_class_field_name,
graph_name, k, heuristic,
anchors_percent_size) #original
#self._result_container.set_result(precision, PerformanceMeasures.PRECISION, targeted_class_field_name,
# graph_name, k,
# heuristic) #By Lior
recall = performance[1]
results[(k, 'recall')] += [recall]
self._result_container.set_result(recall,
PerformanceMeasures.RECALL,
targeted_class_field_name,
graph_name, k, heuristic,
anchors_percent_size) #original
#self._result_container.set_result(recall, PerformanceMeasures.RECALL, targeted_class_field_name,
# graph_name, k,
# heuristic) #by Lior
f1 = performance[2]
results[(k, 'f1')] += [f1]
results[(k, 'support')] = None
accuracy = accuracy_score(actual, predictions)
self._result_container.set_result(accuracy,
PerformanceMeasures.ACCURACY,
targeted_class_field_name,
graph_name, k, heuristic,
anchors_percent_size)
# self._result_container.set_result(accuracy, PerformanceMeasures.ACCURACY, targeted_class_field_name,
# graph_name, k,
# heuristic)
confusion_matrix_score = confusion_matrix(actual, predictions)
self._result_container.set_result(
confusion_matrix_score, PerformanceMeasures.CONFUSION_MATRIX,
targeted_class_field_name, graph_name, k, heuristic,
anchors_percent_size)
# self._result_container.set_result(confusion_matrix_score, PerformanceMeasures.CONFUSION_MATRIX,
# targeted_class_field_name,
# graph_name, k,
# heuristic)
num_of_correct_instances, num_of_incorrect_instances = self._result_container.calculate_correctly_and_not_correctly_instances(
confusion_matrix_score)
self._result_container.set_result(
num_of_correct_instances,
PerformanceMeasures.CORRECTLY_CLASSIFIED,
targeted_class_field_name, graph_name, k, heuristic,
anchors_percent_size)
# self._result_container.set_result(num_of_correct_instances, PerformanceMeasures.CORRECTLY_CLASSIFIED,
# targeted_class_field_name,
# graph_name, k,
# heuristic)
self._result_container.set_result(
num_of_incorrect_instances,
PerformanceMeasures.INCORRECTLY_CLASSIFIED,
targeted_class_field_name, graph_name, k, heuristic,
anchors_percent_size)
# self._result_container.set_result(num_of_incorrect_instances, PerformanceMeasures.INCORRECTLY_CLASSIFIED,
# targeted_class_field_name,
# graph_name, k,
# heuristic)
self._result_container.set_result(
num_of_incorrect_instances,
PerformanceMeasures.SELECTED_FEATURES,
targeted_class_field_name, graph_name, k, heuristic,
anchors_percent_size)
X_Test = sequence.pad_sequences(tokenisedTest, maxlen=max_review_length,padding='post')
#Radical
radicalModel = Sequential()
radicalModel.add(Embedding(vocabSize, 100, input_length=max_review_length,weights=[embedding_matrix], trainable=False))
radicalModel.add(LSTM(100, dropout=0.2, recurrent_dropout=0.2))
radicalModel.add(Dense(1, activation='sigmoid'))
radicalModel.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])
radicalModel.fit(X_Train,radicalTrain,epochs=10, batch_size=100)
radicalScore = radicalModel.evaluate(X_Test,radicalTest,verbose = 100)
accuRadicalLstm = radicalScore[1]
print("\nRadical Training Done for Iteration",iteration)
positiveRadical = [x for x in radicalTest if x == 1]
predictRadical = radicalModel.predict_classes(X_Test, verbose = 1)
positivePredRadical = [x for x in predictRadical if x > 0]
prec, recall, fscore, support = precision_recall_fscore_support(radicalTest, predictRadical)
print("Number of positive Examples : ",len(positiveRadical), "\nratio : ", (len(positiveRadical) / len(radicalTest)), "\nPositive Predicted : ", len(positivePredRadical), "\naccuracy : ", accuRadicalLstm, "\nwrongness : ", 1 - accuRadicalLstm,"\n\nPrecision : ",prec,"\nRecall : ", recall, "\nf1Score : ", fscore, "\nsupport : ", support )
gRadicalAccu += accuRadicalLstm
gPositivePredRadical += len(positivePredRadical)
gPrecision[0] += prec[0]
gPrecision[1] += prec[1]
gRecall[0] += recall[0]
gRecall[1] += recall[1]
gFScore[0] += fscore[0]
gFScore[1] += fscore[1]
with timer("final Output"):
def classification_report(classificator_type, y_true, y_pred):
''' Computes clasification metrics
:param y_true - original class label
:param y_pred - predicted class label
:return presicion, recall for each class; micro_f1 measure, macro_f1 measure
'''
report = classificator_type + '\n'
last_line_heading = 'avg / total'
final_line_heading = 'final score'
labels = unique_labels(y_true, y_pred)
width = len(last_line_heading)
target_names = ['{0}'.format(l) for l in labels]
headers = ["precision", "recall", "f1-score", "support"]
fmt = '%% %ds' % width # first column: class name
fmt += ' '
fmt += ' '.join(['% 9s' for _ in headers])
fmt += '\n'
headers = [""] + headers
report += fmt % tuple(headers)
report += '\n'
p, r, f1, s = precision_recall_fscore_support(y_true, y_pred,
labels=labels,
average=None)
f1_macro = 0
precision_macro = 0
recall_macro = 0
for i, label in enumerate(labels):
values = [target_names[i]]
f1_macro += f1[i]
precision_macro += p[i]
recall_macro += r[i]
for v in (p[i], r[i], f1[i]):
values += ["{0:0.5f}".format(v)]
values += ["{0}".format(s[i])]
report += fmt % tuple(values)
report += '\n'
# compute averages
values = [last_line_heading]
for v in (np.average(p, weights=s),
np.average(r, weights=s),
np.average(f1, weights=s)):
values += ["{0:0.5f}".format(v)]
values += ['{0}'.format(np.sum(s))]
report += fmt % tuple(values)
values = [final_line_heading]
for v in (precision_macro, recall_macro, f1_macro):
values += ["{0:0.5f}".format(v / labels.size)]
values += ['{0}'.format(np.sum(s))]
report += fmt % tuple(values)
return report
def run_graph(self):
# INPUTs
is_training = tf.placeholder(tf.bool, shape=(), name='bool_train')
x_char_ngram_indices = tf.placeholder(tf.int32,
shape=[None, self.params['max_domain_segments_len'],
self.max_num_charngrams],
name='embedding')
x_suffix = tf.placeholder(tf.float32,
shape=[None, self.params['num_suffix']],
name='suffix')
seq_len = tf.placeholder(tf.int32, shape=[None], name='length')
sample_weights = tf.placeholder(tf.float32, shape=[None], name='weight')
y = tf.placeholder(tf.int32, shape=[None], name='target') # Each entry in y must be an index in [0, num_classes)
# embedding layers
# Look up embeddings for inputs.
embeddings = tf.Variable(tf.random_uniform([len(self.charngram2index), embed_dimen], -1.0, 1.0))
embed = tf.nn.embedding_lookup(embeddings, x_char_ngram_indices)
mask = tf.placeholder(tf.float32, shape=[None, self.params['max_domain_segments_len'],
self.max_num_charngrams],
name='embed_mask')
mask = tf.expand_dims(mask, axis=-1)
mask = tf.tile(mask, [1,1,1,embed_dimen])
x_embed = tf.multiply(embed, mask) # x_embed.shape: (None, self.params['max_domain_segments_len'],
# self.max_num_charngrams, embed_dimen)
x_embed = tf.reduce_mean(x_embed, 2)
domain_vectors = []
if 'RNN' in type:
rnn_cell = tf.contrib.rnn.BasicRNNCell(n_rnn_neurons, activation=tf.nn.tanh)
# The shape of last_states should be [batch_size, n_lstm_neurons]
_, domain_vec_rnn = tf.nn.dynamic_rnn(rnn_cell, x_embed, sequence_length=seq_len, dtype=tf.float32, time_major=False)
domain_vec_rnn = tf.layers.dropout(domain_vec_rnn, dropout_rate, training=is_training)
domain_vectors.append(domain_vec_rnn)
if 'CNN' in type:
pooled_outputs = []
for filter_size in filter_sizes:
# Define and initialize filters
filter_shape = [filter_size, embed_dimen, 1, num_filters]
W_filter = tf.Variable(tf.truncated_normal(filter_shape, stddev=0.1)) # initialize the filters' weights
b_filter = tf.Variable(tf.constant(0.1, shape=[num_filters])) # initialize the filters' biases
# The conv2d operation expects a 4-D tensor with dimensions corresponding to batch, width, height and channel.
# The result of our embedding doesn’t contain the channel dimension
# So we add it manually, leaving us with a layer of shape [None, sequence_length, embedding_size, 1].
x_embed_expanded = tf.expand_dims(x_embed, -1)
conv = tf.nn.conv2d(x_embed_expanded, W_filter, strides=[1, 1, 1, 1], padding="VALID")
# Apply nonlinearity
h = tf.nn.relu(tf.nn.bias_add(conv, b_filter), name="relu")
pooled = tf.nn.max_pool(h, ksize=[1, self.params['max_domain_segments_len'] - filter_size + 1, 1, 1],
strides=[1, 1, 1, 1], padding='VALID')
pooled_outputs.append(pooled)
# Combine all the pooled features
h_pool = tf.concat(pooled_outputs, axis=3)
num_filters_total = num_filters * len(filter_sizes)
domain_vec_cnn = tf.reshape(h_pool, [-1, num_filters_total])
domain_vec_cnn = tf.layers.dropout(domain_vec_cnn, dropout_rate, training=is_training)
domain_vectors.append(domain_vec_cnn)
# concatenate suffix one-hot and the abstract representation of the domains segments
# The shape of cat_layer should be [batch_size, n_lstm_neurons+self.params['num_suffix']]
cat_layer = tf.concat(domain_vectors + [x_suffix], -1)
# print(cat_layer.get_shape())
logits = cat_layer
for _ in range(n_fc_layers):
logits = tf.contrib.layers.fully_connected(logits, num_outputs=n_rnn_neurons, activation_fn=act_fn)
logits = tf.layers.dropout(logits, dropout_rate, training=is_training)
logits = tf.contrib.layers.fully_connected(logits, self.params['num_targets'], activation_fn=act_fn)
if class_weighted:
crossentropy = tf.losses.sparse_softmax_cross_entropy(labels=y, logits=logits, weights=sample_weights)
else:
crossentropy = tf.losses.sparse_softmax_cross_entropy(labels=y, logits=logits)
loss_mean = tf.reduce_mean(crossentropy)
optimizer = tf.train.AdamOptimizer(learning_rate=lr_rate)
training_op = optimizer.minimize(loss_mean)
prediction = tf.argmax(logits, axis=-1)
is_correct = tf.nn.in_top_k(logits, y, 1) # logits are unscaled, but here we only care the argmax
n_correct = tf.reduce_sum(tf.cast(is_correct, tf.float32))
accuracy = tf.reduce_mean(tf.cast(is_correct, tf.float32))
init = tf.global_variables_initializer()
''' For TensorBoard '''
'''
now = datetime.now().strftime("%Y%m%d%H%M%S")
root_log_dir = 'tf_logs'
logdir = OUTPUT_DIR + root_log_dir + "/run-" + now + "/"
loss_summary = tf.summary.scalar('loss_mean', loss_mean)
acc_summary = tf.summary.scalar('accuracy', accuracy)
file_writer = tf.summary.FileWriter(logdir, tf.get_default_graph())
'''
with tf.Session() as sess:
init.run()
n_total_batches = int(np.ceil(len(self.domains_train) / batch_size))
test_fscore_history = []
for epoch in range(1, n_epochs + 1):
# model training
n_batch = 0
for X_batch_embed, X_batch_mask, domain_actual_lens, X_batch_suf, sample_weights, y_batch in self.next_batch(self.domains_train):
_, acc_batch_train, loss_batch_train, prediction_train = sess.run([training_op, accuracy, loss_mean, prediction],
feed_dict={
'bool_train:0': True,
'embedding:0': X_batch_embed,
'embed_mask:0': X_batch_mask,
'suffix:0': X_batch_suf,
'length:0': domain_actual_lens,
'weight:0': sample_weights,
'target:0': y_batch})
n_batch += 1
if epoch < 2:
# print(prediction_train)
print("Epoch %d - Batch %d/%d: loss = %.4f, accuracy = %.4f" %
(epoch, n_batch, n_total_batches, loss_batch_train, acc_batch_train))
# evaluation on training data
eval_nodes = [n_correct, loss_mean, is_correct, prediction]
print()
print("========== Evaluation at Epoch %d ==========" % epoch)
loss_train, acc_train, _, _ = self.evaluate(self.domains_train, sess, eval_nodes)
print("*** On Training Set:\tloss = %.6f\taccuracy = %.4f" % (loss_train, acc_train))
# evaluation on validation data
loss_val, acc_val, _, _ = self.evaluate(self.domains_val, sess, eval_nodes)
print("*** On Validation Set:\tloss = %.6f\taccuracy = %.4f" % (loss_val, acc_val))
# evaluate on test data
loss_test, acc_test, is_correct_test, pred_test = self.evaluate(self.domains_test, sess, eval_nodes)
print("*** On Test Set:\tloss = %.6f\taccuracy = %.4f" % (loss_test, acc_test))
print()
print("Macro average:")
precisions_macro, recalls_macro, fscores_macro, _ = precision_recall_fscore_support(
[category2index[domain['categories'][1]] for domain in self.domains_test],
pred_test, average='macro')
print("Precision (macro): %.4f, Recall (macro): %.4f, F-score (macro): %.4f" %
(precisions_macro, recalls_macro, fscores_macro))
print()
if not test_fscore_history or fscores_macro > max(test_fscore_history):
# the accuracy of this epoch is the largest
print("Classification Performance on individual classes:")
precisions_none, recalls_none, fscores_none, supports_none = precision_recall_fscore_support(
[category2index[domain['categories'][1]] for domain in self.domains_test],
pred_test, average=None)
print(tabulate(zip((categories[i] for i in range(len(precisions_none))),
precisions_none, recalls_none, fscores_none, supports_none),
headers=['category', 'precision', 'recall', 'f-score', 'support'],
tablefmt='orgtbl'))
# output all incorrect_prediction
with open(os.path.join(OUTPUT_DIR, 'incorrect_predictions.csv'), 'w') as outfile:
csv_writer = csv.writer(outfile)
csv_writer.writerow(('RAW_DOMAIN', 'SEGMENTED_DOMAIN', 'TRUE_CATEGORY', 'PRED_CATEGORY'))
for correct, pred_catIdx, domain in zip(is_correct_test, pred_test, self.domains_test):
if correct:
continue
csv_writer.writerow((domain['raw_domain'],
domain['segmented_domain'],
domain['categories'][1],
categories[pred_catIdx]))
test_fscore_history.append(fscores_macro)
def evaluate(self,
data,
y_map,
tracker_fn='tracker.csv',
channel_probs=False,
predict_english_only=False):
print(channel_probs)
best_epochs = int(re.findall('_e(\d+)_', self.best_fn)[0])
try:
model = load_model(os.path.join(self.modeldir, self.best_fn))
except:
model = load_model(os.path.join(self.modeldir, self.best_fn),
custom_objects={'f1': ut.f1})
y_decoder = {v: k for k, v in y_map.items()}
X_test = data['test']['xarr']
y_test = data['test']['yarr']
y_test_labels = [np.argmax(cat) for cat in y_test]
#predict
#y_pred = model.predict(data.X_test_seq,batch_size=self.batch_size)
if channel_probs == True:
all_lang_preds = []
for arr in X_test:
y_pred = model.predict(arr, batch_size=self.batch_size)
#y_pred_labels=[np.argmax(pred) for pred in y_pred]
all_lang_preds.append(y_pred)
#all_lang_preds.append(y_pred_labels)
y_pred_labels = []
for i, probs in enumerate(zip(*all_lang_preds)):
y_pred_labels.append(
np.argmax(np.average(np.array(probs), axis=0)))
elif predict_english_only == True:
y_pred = model.predict(X_test[0], batch_size=self.batch_size)
y_pred_labels = [np.argmax(pred) for pred in y_pred]
else:
y_pred = model.predict(X_test, batch_size=self.batch_size)
y_pred_labels = [np.argmax(pred) for pred in y_pred]
#evaluate
conf_matrix = confusion_matrix(y_test_labels, y_pred_labels)
acc = accuracy_score(y_test_labels, y_pred_labels)
w_prec, w_rec, w_f1, _ = precision_recall_fscore_support(
y_test_labels, y_pred_labels, average='weighted')
mic_prec, mic_rec, mic_f1, _ = precision_recall_fscore_support(
y_test_labels, y_pred_labels, average='micro')
mac_prec, mac_rec, mac_f1, _ = precision_recall_fscore_support(
y_test_labels, y_pred_labels, average='macro')
results = {
'model_filepath': self.best_fn,
'languages': self.x_cols,
'model_name': self.model_name,
'date': '-',
'time': '-',
'train_time': self.train_time,
'best_epochs': best_epochs,
'accuracy': acc,
'batch_size': self.batch_size,
'total_epochs': self.total_epochs,
'summary': self.summary,
'precision (weighted)': w_prec,
'f1 (weighted)': w_f1,
'precision (micro)': mic_prec,
'recall (micro)': mic_rec,
'f1 (micro)': mic_f1,
'precision (macro)': mac_prec,
'recall (macro)': mac_rec,
'f1 (macro)': mac_f1,
'confusion_matrix': conf_matrix
}
tracker = pd.read_csv(os.path.join(self.modeldir, tracker_fn))
tracker = tracker.append(results, ignore_index=True)
tracker.to_csv(os.path.join(self.modeldir, tracker_fn), index=False)
def train_basic(dirpath_vector, dirpath_output, verbose=True):
logger = utils.get_logger()
x_train = np.genfromtxt(dirpath_vector + '/phylum/train.csv',
delimiter='\n',
dtype=None,
encoding=None)
x_test = np.genfromtxt(dirpath_vector + '/phylum/test.csv',
delimiter='\n',
dtype=None,
encoding=None)
x_val = np.genfromtxt(dirpath_vector + '/phylum/val.csv',
delimiter='\n',
dtype=None,
encoding=None)
arr = []
arr1 = []
arr2 = []
for item in x_train[1:]:
arr.append(ordinal_encoder(string_to_array(item.split(",")[3])))
for item in x_test[1:]:
arr1.append(ordinal_encoder(string_to_array(item.split(",")[3])))
for item in x_val[1:]:
arr2.append(ordinal_encoder(string_to_array(item.split(",")[3])))
maxi = 0
for item in arr:
if len(item) > maxi:
maxi = len(item)
final1 = np.zeros((x_train.shape[0] - 1, maxi))
count = 0
for item in arr:
final1[count][:len(item)] = item
count += 1
maxi1 = 0
for item in arr1:
if len(item) > maxi1:
maxi1 = len(item)
final2 = np.zeros((x_test.shape[0] - 1, maxi1))
count = 0
for item in arr1:
final2[count][:len(item)] = item
count += 1
maxi2 = 0
for item in arr2:
if len(item) > maxi2:
maxi2 = len(item)
final3 = np.zeros((x_val.shape[0] - 1, maxi2))
count = 0
for item in arr2:
final3[count][:len(item)] = item
count += 1
hf = h5py.File(dirpath_vector + '/phylum/ordinal.h5', 'w')
hf.create_dataset('dataset_1', data=final1)
hf.create_dataset('dataset_2', data=final2)
hf.create_dataset('dataset_3', data=final3)
hf.close()
x_train = np.genfromtxt(dirpath_vector + '/class/train.csv',
delimiter='\n',
dtype=None,
encoding=None)
x_test = np.genfromtxt(dirpath_vector + '/class/test.csv',
delimiter='\n',
dtype=None,
encoding=None)
x_val = np.genfromtxt(dirpath_vector + '/class/val.csv',
delimiter='\n',
dtype=None,
encoding=None)
arr = []
arr1 = []
arr2 = []
for item in x_train[1:]:
arr.append(ordinal_encoder(string_to_array(item.split(",")[3])))
for item in x_test[1:]:
arr1.append(ordinal_encoder(string_to_array(item.split(",")[3])))
for item in x_val[1:]:
arr2.append(ordinal_encoder(string_to_array(item.split(",")[3])))
maxi = 0
for item in arr:
if len(item) > maxi:
maxi = len(item)
final1 = np.zeros((x_train.shape[0] - 1, maxi))
count = 0
for item in arr:
final1[count][:len(item)] = item
count += 1
maxi1 = 0
for item in arr1:
if len(item) > maxi1:
maxi1 = len(item)
final2 = np.zeros((x_test.shape[0] - 1, maxi1))
count = 0
for item in arr1:
final2[count][:len(item)] = item
count += 1
maxi2 = 0
for item in arr2:
if len(item) > maxi2:
maxi2 = len(item)
final3 = np.zeros((x_val.shape[0] - 1, maxi2))
count = 0
for item in arr2:
final3[count][:len(item)] = item
count += 1
hf = h5py.File(dirpath_vector + '/class/ordinal.h5', 'w')
hf.create_dataset('dataset_1', data=final1)
hf.create_dataset('dataset_2', data=final2)
hf.create_dataset('dataset_3', data=final3)
hf.close()
hf = h5py.File(dirpath_vector + '/phylum/ordinal.h5', 'r')
n1 = hf.get('dataset_1')
n2 = hf.get('dataset_2')
n3 = hf.get('dataset_3')
X = np.array(n1)
Y = np.array(n2)
V = np.array(n3)
hf.close()
lab = np.genfromtxt(dirpath_vector + '/phylum/train.csv',
delimiter='\n',
dtype=None,
encoding=None)
lab1 = np.genfromtxt(dirpath_vector + '/phylum/test.csv',
delimiter='\n',
dtype=None,
encoding=None)
lab2 = np.genfromtxt(dirpath_vector + '/phylum/val.csv',
delimiter='\n',
dtype=None,
encoding=None)
labels = []
i = 0
for item in lab[1:]:
if item.split(",")[0][0] == "A":
labels.append(0)
elif item.split(",")[0][0] == "F":
labels.append(1)
else:
labels.append(2)
i += 1
labels1 = []
i = 0
for item in lab1[1:]:
if item.split(",")[0][0] == "A":
labels1.append(0)
elif item.split(",")[0][0] == "F":
labels1.append(1)
else:
labels1.append(2)
i += 1
labels2 = []
i = 0
for item in lab2[1:]:
if item.split(",")[0][0] == "A":
labels2.append(0)
elif item.split(",")[0][0] == "F":
labels2.append(1)
else:
labels2.append(2)
i += 1
label = np.array(labels)
label1 = np.array(labels1)
label2 = np.array(labels2)
clf2 = SVC(kernel='rbf')
clf = RandomForestClassifier()
clf2.fit(X, label)
clf.fit(X, label)
preds2 = clf2.predict(X)
preds = clf.predict(X)
scores = clf2.decision_function(X)
scores2 = clf.predict(X)
score = np.amax(scores, axis=1)
fpr, tpr, thresholds = roc_curve(label, score, pos_label=2)
fpr2, tpr2, thresholds2 = roc_curve(label, scores2, pos_label=2)
match2 = 0
for i in range(preds2.shape[0]):
if preds2[i] == label[i]:
match2 += 1
accuracy2 = float(match2) / preds2.shape[0]
p, r, f1, s = precision_recall_fscore_support(label,
preds2,
average='weighted')
match = 0
for i in range(preds.shape[0]):
if preds[i] == label[i]:
match += 1
accuracy = float(match) / preds.shape[0]
p2, r2, f12, s = precision_recall_fscore_support(label,
preds,
average='weighted')
C = confusion_matrix(label, preds2)
logger.info(
'Train Accuracy, precision, recall and F1 Score for SVM model for phylum level is {:.3f}, {:.3f}, {:.3f}, {:.3f}'
.format(accuracy2, p, r, f1))
logger.info(
'Train Accuracy, precision, recall and F1 Score for Random Forest model for phylum level is {:.3f}, {:.3f}, {:.3f}, {:.3f}'
.format(accuracy, p2, r2, f12))
hf = h5py.File(dirpath_vector + '/class/ordinal.h5', 'r')
n1 = hf.get('dataset_1')
n2 = hf.get('dataset_2')
n3 = hf.get('dataset_3')
X = np.array(n1)
Y = np.array(n2)
V = np.array(n3)
hf.close()
lab = np.genfromtxt(dirpath_vector + '/class/train.csv',
delimiter='\n',
dtype=None,
encoding=None)
lab1 = np.genfromtxt(dirpath_vector + '/class/test.csv',
delimiter='\n',
dtype=None,
encoding=None)
lab2 = np.genfromtxt(dirpath_vector + '/class/val.csv',
delimiter='\n',
dtype=None,
encoding=None)
labels = []
i = 0
for item in lab[1:]:
labels.append(int(item.split(",")[2]))
i += 1
labels1 = []
i = 0
for item in lab1[1:]:
labels1.append(int(item.split(",")[2]))
i += 1
labels2 = []
i = 0
for item in lab2[1:]:
labels2.append(int(item.split(",")[2]))
i += 1
label = np.array(labels)
label1 = np.array(labels1)
label2 = np.array(labels2)
clf2 = RandomForestClassifier()
clf = SVC(kernel='rbf')
clf2.fit(X, label)
clf.fit(X, label)
preds2 = clf2.predict(X)
preds = clf.predict(X)
scores = clf2.predict(X)
scores1 = clf.decision_function(X)
score = np.amax(scores1, axis=1)
fpr, tpr, thresholds = roc_curve(label, scores, pos_label=2)
fpr2, tpr2, thresholds2 = roc_curve(label, score, pos_label=2)
match2 = 0
for i in range(preds2.shape[0]):
if preds2[i] == label[i]:
match2 += 1
accuracy2 = float(match2) / preds2.shape[0]
p, r, f1, s = precision_recall_fscore_support(label,
preds2,
average='weighted')
C = confusion_matrix(label, preds2)
match = 0
for i in range(preds.shape[0]):
if preds[i] == label[i]:
match += 1
accuracy = float(match) / preds.shape[0]
p2, r2, f12, s = precision_recall_fscore_support(label,
preds,
average='weighted')
logger.info(
'Train Accuracy, precision, recall and F1 Score for SVM model for phylum level is {:.3f}, {:.3f}, {:.3f}, {:.3f}'
.format(accuracy, p2, r2, f12))
logger.info(
'Train Accuracy, precision, recall and F1 Score for Random Forest model for phylum level is {:.3f}, {:.3f}, {:.3f}, {:.3f}'
.format(accuracy2, p, r, f1))
