def test_pandas_confusion_normalized():
y_true = [2, 0, 2, 2, 0, 1, 1, 2, 2, 0, 1, 2]
y_pred = [0, 0, 2, 1, 0, 2, 1, 0, 2, 0, 2, 2]
cm = ConfusionMatrix(y_true, y_pred)
assert isinstance(cm, pdml.confusion_matrix.LabeledConfusionMatrix)
df = cm.to_dataframe()
df_norm = cm.to_dataframe(normalized=True)
assert (df_norm.sum(axis=1).sum() == len(df))
def test_pandas_confusion_normalized(self):
y_true = [2, 0, 2, 2, 0, 1, 1, 2, 2, 0, 1, 2]
y_pred = [0, 0, 2, 1, 0, 2, 1, 0, 2, 0, 2, 2]
cm = ConfusionMatrix(y_true, y_pred)
assert isinstance(cm, pdml.confusion_matrix.LabeledConfusionMatrix)
df = cm.to_dataframe()
df_norm = cm.to_dataframe(normalized=True)
assert(df_norm.sum(axis=1).sum() == len(df))
def calculate_accuracy(csv_filename):
# Loading csv information into a data frame
data = pd.read_csv(csv_filename)
# assigning actual sentiment data to y_test
y_test = data['Actual_Statement']
# assigning predicted sentiment data to y_pred
y_pred = data['Prediction']
score = accuracy_score(y_test, y_pred)
# calling accuracy_score method to get the accuracy_score
print 'Accuracy Score : ', score
# calling confusion_matrix method from pandas_ml to show the output
confusion_matrix = ConfusionMatrix(y_test, y_pred)
output = confusion_matrix.to_dataframe()
writer = pd.ExcelWriter("azure_text_confusion_matrix_output.xlsx")
output.to_excel(writer, startrow=4, startcol=0)
Acuracy_Score = 'Accuracy Score : ' + str(score)
worksheet = writer.sheets['Sheet1']
worksheet.write(1, 0, Acuracy_Score)
writer.save()
print("Confusion matrix:\n%s" % confusion_matrix)
def test_pandas_confusion_normalized_issue1(self):
# should insure issue 1 is fixed
# see http://stackoverflow.com/questions/19233771/sklearn-plot-confusion-matrix-with-labels/31720054#31720054
y_true = ['business', 'business', 'business', 'business', 'business',
'business', 'business', 'business', 'business', 'business',
'business', 'business', 'business', 'business', 'business',
'business', 'business', 'business', 'business', 'business']
y_pred = ['health', 'business', 'business', 'business', 'business',
'business', 'health', 'health', 'business', 'business', 'business',
'business', 'business', 'business', 'business', 'business',
'health', 'health', 'business', 'health']
cm = ConfusionMatrix(y_true, y_pred)
assert isinstance(cm, pdml.confusion_matrix.BinaryConfusionMatrix)
df = cm.to_dataframe()
df_norm = cm.to_dataframe(normalized=True)
assert(df_norm.sum(axis=1, skipna=False).fillna(1).sum() == len(df))
def test_pandas_confusion_normalized_issue1(self):
# should insure issue 1 is fixed
# see http://stackoverflow.com/questions/19233771/sklearn-plot-confusion-matrix-with-labels/31720054#31720054
y_true = ['business', 'business', 'business', 'business', 'business',
'business', 'business', 'business', 'business', 'business',
'business', 'business', 'business', 'business', 'business',
'business', 'business', 'business', 'business', 'business']
y_pred = ['health', 'business', 'business', 'business', 'business',
'business', 'health', 'health', 'business', 'business', 'business',
'business', 'business', 'business', 'business', 'business',
'health', 'health', 'business', 'health']
cm = ConfusionMatrix(y_true, y_pred)
assert isinstance(cm, pdml.confusion_matrix.BinaryConfusionMatrix)
df = cm.to_dataframe()
df_norm = cm.to_dataframe(normalized=True)
assert(df_norm.sum(axis=1, skipna=False).fillna(1).sum() == len(df))
def on_epoch_end(self, epoch, logs=None):
y_true, y_pred = [], []
for i in range(self.validation_steps):
X_batch, y_true_batch = next(self.validation_data)
y_pred_batch = self.model.predict(X_batch)
y_true.extend(y_true_batch)
y_pred.extend(y_pred_batch)
y_true = np.float32(y_true)
y_pred = np.float32(y_pred)
val_loss = log_loss(y_true, y_pred)
# map integer labels to strings
y_true = list(y_true.argmax(axis=-1))
y_pred = list(y_pred.argmax(axis=-1))
y_true = [self.int2label[y] for y in y_true]
y_pred = [self.int2label[y] for y in y_pred]
confusion = ConfusionMatrix(y_true, y_pred)
accs = self.accuracies(confusion._df_confusion.values)
acc = self.accuracy(confusion._df_confusion.values)
# same for wanted words
y_true = [y if y in self.wanted_words else '_unknown_' for y in y_true]
y_pred = [y if y in self.wanted_words else '_unknown_' for y in y_pred]
wanted_words_confusion = ConfusionMatrix(y_true, y_pred)
wanted_accs = self.accuracies(wanted_words_confusion._df_confusion.values)
acc_line = ('\n[%03d]: val_categorical_accuracy: %.2f, '
'val_mean_categorical_accuracy_wanted: %.2f') % (
epoch, acc, wanted_accs.mean()) # noqa
with open('confusion_matrix.txt', 'a') as f:
f.write('%s\n' % acc_line)
f.write(confusion.to_dataframe().to_string())
with open('wanted_confusion_matrix.txt', 'a') as f:
f.write('%s\n' % acc_line)
f.write(wanted_words_confusion.to_dataframe().to_string())
logs['val_loss'] = val_loss
logs['val_categorical_accuracy'] = acc
logs['val_mean_categorical_accuracy_all'] = accs.mean()
logs['val_mean_categorical_accuracy_wanted'] = wanted_accs.mean()
def on_epoch_end(self, epoch, logs=None):
y_true, y_pred = [], []
for i in range(self.validation_steps):
X_batch, y_true_batch = next(self.validation_data)
y_pred_batch = self.model.predict(X_batch)
y_true.extend(y_true_batch)
y_pred.extend(y_pred_batch)
y_true = np.float32(y_true)
y_pred = np.float32(y_pred)
val_loss = log_loss(y_true, y_pred)
# map integer labels to strings
y_true = list(y_true.argmax(axis=-1))
y_pred = list(y_pred.argmax(axis=-1))
y_true = [self.int2label[y] for y in y_true]
y_pred = [self.int2label[y] for y in y_pred]
confusion = ConfusionMatrix(y_true, y_pred)
accs = self.accuracies(confusion._df_confusion.values)
acc = self.accuracy(confusion._df_confusion.values)
# same for wanted words
y_true = [y if y in self.wanted_words else '_unknown_' for y in y_true]
y_pred = [y if y in self.wanted_words else '_unknown_' for y in y_pred]
wanted_words_confusion = ConfusionMatrix(y_true, y_pred)
wanted_accs = self.accuracies(wanted_words_confusion._df_confusion.values)
acc_line = ('\n[%03d]: val_categorical_accuracy: %.2f, '
'val_mean_categorical_accuracy_wanted: %.2f') % (
epoch, acc, wanted_accs.mean()) # noqa
with open('confusion_matrix.txt', 'a') as f:
f.write('%s\n' % acc_line)
f.write(confusion.to_dataframe().to_string())
with open('wanted_confusion_matrix.txt', 'a') as f:
f.write('%s\n' % acc_line)
f.write(wanted_words_confusion.to_dataframe().to_string())
logs['val_loss'] = val_loss
logs['val_categorical_accuracy'] = acc
logs['val_mean_categorical_accuracy_all'] = accs.mean()
logs['val_mean_categorical_accuracy_wanted'] = wanted_accs.mean()
def gen_conf_mat(y_pred, y_true):
"""Generate confusion matrix with the appropriate naming conventions"""
#ipdb.set_trace()
inv_label_dict = {
0: 'background',
63: 'liver',
126: 'l_kidney',
189: 'r_kidney',
252: 'spleen'
}
#Rename columne for analysis
tmp_conf_mat = ConfusionMatrix(y_true, y_pred)
tmp_conf_mat = tmp_conf_mat.to_dataframe()
filt_df_dict = {
k: v
for k, v in inv_label_dict.items()
if k in tmp_conf_mat.columns.tolist()
}
tmp_conf_mat.rename(filt_df_dict, axis=0, inplace=True)
tmp_conf_mat.rename(filt_df_dict, axis=1, inplace=True)
return tmp_conf_mat
def gen_test_report(clf, y_test, X_test, args, sub_str='_test_report_per_cls'):
#Writing test report to file
y_true, y_pred = y_test, clf.predict(X_test)
#ipdb.set_trace()
#Producing pandas ML confusion matrix and statistical summary
tmp_confusion_matrix = ConfusionMatrix(y_true, y_pred)
#tmp_stat_summary=tmp_confusion_matrix.stats()
tmp_confusion_matrix = tmp_confusion_matrix.to_dataframe()
tmp_confusion_matrix.to_csv(
os.path.join(args.output_model_dir,
args.f_nm_str + '_confusion_matrix'))
#Generation dictionary for analysis
#with open(os.path.join(args.output_model_dir,args.f_nm_str+'_descriptive_stat.pickle')) as fb:
# pickle.dump(tmp_stat_summary,fb)
file_nm_test_report = args.f_nm_str + sub_str
#Generating report on test data for analysis
test_report_raw = classification_report(y_true, y_pred, output_dict=True)
test_report_df = pd.DataFrame(test_report_raw).transpose()
#Writing best model to file directory for models
print(test_report_raw)
test_report_df.to_csv(
os.path.join(args.output_model_dir, file_nm_test_report))
df = pd.read_csv('./data/analysis.csv')
for difficulty in [0,1,2]:
sub_df = df.loc[df['difficulty']==difficulty]
cmat = ConfusionMatrix(sub_df['predicted'],sub_df['actual'])
#cmat.print_stats()
#print dir(cmat)
fig = plt.figure()
ax = fig.add_subplot(111)
sns.heatmap(cmat.to_dataframe(),square=True, annot=True, fmt="d", cmap=plt.cm.bone_r)
plt.xticks(rotation=90)
plt.yticks(rotation=0)
a=[item.get_text() for item in ax.get_yticklabels()]
b=[item.get_text() for item in ax.get_xticklabels()]
conversion = {'sympathomimetic': "Sympathomimetic",
'sedative_hypnotic': "Sedative-Hypnotic",
'cholinergic':"Cholinergic",
'anticholinergic':"Anticholinergic",
'opioid':"Opioid",
"serotonin_syndrome":"Serotonin Syndrome"}
new_ticklabels = [conversion[label] for label in b]
ax.set_yticklabels(new_ticklabels)
import seaborn as sns
import matplotlib.pyplot as plt
from pandas_ml import ConfusionMatrix
df = pd.read_csv('./data/analysis.csv')
cmat = ConfusionMatrix(df['predicted'], df['actual'])
#cmat.print_stats()
#print dir(cmat)
fig = plt.figure()
ax = fig.add_subplot(111)
sns.heatmap(cmat.to_dataframe(),
square=True,
annot=True,
fmt="d",
cmap=plt.cm.bone_r)
plt.xticks(rotation=90)
plt.yticks(rotation=0)
a = [item.get_text() for item in ax.get_yticklabels()]
b = [item.get_text() for item in ax.get_xticklabels()]
conversion = {
'sympathomimetic': "Sympathomimetic",
'sedative_hypnotic': "Sedative-Hypnotic",
'cholinergic': "Cholinergic",
'anticholinergic': "Anticholinergic",
'opioid': "Opioid",
df['actual_class'], r_bins = pd.qcut(df['rome'],
10,
labels=list(range(1, 11)),
retbins=True)
df['predic_class'] = pd.qcut(df['predicted'],
10,
labels=list(range(1, 11)))
# df['predic_class'] = pd.cut(df['predicted'], bins=r_bins, labels=list(range(1, 11)))
cm = ConfusionMatrix(df['actual_class'].to_list(),
df['predic_class'].to_list())
cm.print_stats()
statdict = cm.stats()
cm_stats = statdict['class']
matrix = cm.to_dataframe()
matrix.index.rename('ROME decile', inplace=True)
matrix.columns.rename('ROME$_\mathrm{NN}$ decile', inplace=True)
plt.close()
# sns.heatmap(matrix / (len(predicted)//10) , cmap='Greys', annot=matrix, fmt='d')
ax = sns.heatmap(matrix / (len(predicted) // 10),
annot=matrix,
fmt='d',
cmap='gray_r',
vmin=0.0,
vmax=0.56)
cbar = ax.collections[0].colorbar
cbar.ax.tick_params(labelsize=20)
cbar.set_label("Percentage in decile [1]", fontsize=23)
ax.set_xticklabels(ax.get_xticklabels(), fontsize=20)
class EncoderWithClassifier:
def __init__(self,
encoder_model,
name='',
activation='sigmoid',
loss_function='categorical_crossentropy',
optimizer=SGD(lr=0.01),
use_last_dim_as_classifier_dim=True,
classifier_dim=None):
self.__encoder_model = encoder_model
self.__name = name
self.__activation = activation
self.__loss_function = loss_function
self.__optimizer = optimizer
self.__trained = True
self.__validated = True
self.__use_last_dim_as_classifier_dim = use_last_dim_as_classifier_dim
self.__classifier_dim = classifier_dim
self.__validateDimensions()
self.__generateClassifier()
self.__compile()
def __validateDimensions(self):
if not self.__use_last_dim_as_classifier_dim and self.__classifier_dim <= 0:
raise ValueError(
"The number of neurons in a layer (classifier_dim) must be greater than zero"
)
def __generateClassifier(self):
ae_output = self.__encoder_model.layers[-1].output
dim = None
if not self.__use_last_dim_as_classifier_dim:
dim = self.__classifier_dim
else:
dim = self.__encoder_model.layers[-1].units
self.__classifier_layers = Dense(dim,
activation=self.__activation,
name='classifier')(ae_output)
self.__classifier = Model(inputs=[self.__encoder_model.input],
outputs=[self.__classifier_layers])
def __compile(self):
self.__classifier.compile(loss=self.__loss_function,
optimizer=self.__optimizer,
metrics=['acc'])
self.__summary = self.__classifier.summary()
def __stats(self, path=None):
file_pattern = path + self.__name + '.{0}.{1}'
classifier_predictions_max = np.argmax(self.__classifier_predictions,
axis=1)
Ymax = np.argmax(self.__eval_label, axis=1)
self.__confusion_matrix = ConfusionMatrix(Ymax,
classifier_predictions_max)
self.__status_dump(file_pattern,
self.__confusion_matrix,
html=True,
string=True,
pickle=True,
stats_as_txt=True,
latex=True)
def __status_dump(self,
file_pattern,
confusion_matrix,
html=False,
string=False,
pickle=False,
stats_as_txt=False,
latex=False):
dataframe = self.__confusion_matrix.to_dataframe()
if html:
with open(file_pattern.format('confusion_matrix', 'html'),
'w') as file:
file.write(dataframe.to_html())
if string:
with open(file_pattern.format('confusion_matrix', 'txt'),
'w') as file:
file.write(dataframe.to_string())
if pickle:
dataframe.to_pickle(
file_pattern.format('confusion_matrix', 'pickle'))
if stats_as_txt:
with open(file_pattern.format('stats', '.txt'), 'w') as file:
file.write(str(confusion_matrix.stats()))
if latex:
with open(file_pattern.format('confusion_matrix', '.latex_table'),
'w') as file:
file.write(dataframe.to_latex())
def eval(self, feature=None, label=None):
self.__eval_feature = feature
self.__eval_label = label
self.__classifier_predictions = self.__classifier.predict(feature)
def train(self,
feature=None,
label=None,
validation=None,
epochs=None,
batch_size=None,
shuffle=True,
store_history=True,
early_stopping=None,
save_every=1,
callbacks=None):
h = self.__classifier.fit(x=feature,
y=label,
validation_data=validation,
batch_size=batch_size,
epochs=epochs,
shuffle=shuffle,
callbacks=callbacks)
if store_history:
self.__history = h
def eval_stats(self, reportpath):
self.__stats(path=reportpath)
@property
def classifier(self):
return self.__classifier
@property
def summary(self):
return self.__summary
class Autoencoder:
def __init__(self,
encoder_layers,
name='',
hidden_layer_activation='relu',
output_layer_activation='relu',
loss_function='mse',
optimizer=SGD(lr=0.01),
discard_decoder_model=False):
self.__name = name
self.__encoder_layers_config = encoder_layers
self.__discard_decoder_model = discard_decoder_model
self.__hidden_layers_activation = hidden_layer_activation
self.__output_layer_activation = output_layer_activation
self.__loss_function = loss_function
self.__optimizer = optimizer
self.__trained = False
self.__validated = False
self.__generateModels()
self.__compile()
def __validateEncoderLayers(self):
""" validate the encoder layers configured
raise value errors if __encoder_layers_config was not setted or if list len is 'le' than one
"""
print('self.__encoder_layers_config ', self.__encoder_layers_config)
if not self.__encoder_layers_config:
raise ValueError(
'A list with the numbers of neurons in each layer is required.'
)
if len(self.__encoder_layers_config) <= 1:
raise ValueError(
'To generate an autoencoder you have to provide at least 2 layers (two items of a list).'
)
def __generateEncoder(self, input):
for id, neurons in enumerate(self.__encoder_layers_config[1:]):
if id == 0:
self.__encoder_layers = Dense(
neurons,
activation=self.__hidden_layers_activation,
name='enc{}_{}'.format(id, neurons))(input)
else:
self.__encoder_layers = Dense(
neurons,
activation=self.__hidden_layers_activation,
name='enc{}_{}'.format(id, neurons))(self.__encoder_layers)
self.__encoder_model = Model(inputs=[input],
outputs=[self.__encoder_layers])
def __generateDecoder(self):
reversed_encoder_layers = self.__encoder_layers_config[:-1]
for id, neurons in enumerate(reversed(reversed_encoder_layers)):
if id == 0:
self.__decoder_layers = Dense(
neurons,
activation=self.__hidden_layers_activation,
name='dec{}_{}'.format(id, neurons))(self.__encoder_layers)
else:
decoder_activation = ''
if id == len(self.__encoder_layers_config[:-1]) - 1:
decoder_activation = self.__output_layer_activation
else:
decoder_activation = self.__hidden_layers_activation
self.__decoder_layers = Dense(
neurons,
activation=decoder_activation,
name='dec{}_{}'.format(id, neurons))(self.__decoder_layers)
def __generateModels(self):
self.__validateEncoderLayers()
input = Input(shape=(self.__encoder_layers_config[0], ))
self.__generateEncoder(input)
self.__generateDecoder()
self.__autoencoder = Model(inputs=[input],
outputs=[self.__decoder_layers])
# if not self.__discard_decoder_model:
# decoder_input = self.__encoder_model.layers[-1].output
#
# self.__decoder_model = Model(inputs=[decoder_input], outputs=[self.__autoencoder.layers[-1](decoder_input)])
def __compile(self):
self.__autoencoder.compile(loss=self.__loss_function,
optimizer=self.__optimizer,
metrics=['accuracy'])
self.__summary = self.__autoencoder.summary()
def train_and_eval(self,
feature=None,
feature_validation=None,
epochs=1000,
batch_size=32,
shuffle=True,
store_history=True,
callbacks=None):
validation_data = None
#if not feature_validation == None:
validation_data = (feature_validation, feature_validation)
h = self.__autoencoder.fit(x=feature,
y=feature,
validation_data=validation_data,
shuffle=shuffle,
epochs=epochs,
batch_size=batch_size,
callbacks=callbacks)
if store_history:
self.__history = h
self.__trained = True
self.__validated = True
def __stats(self):
file_pattern = 'reports/' + self.__name + '.{0}.{1}'
classifier_predictions_max = np.argmax(self.__classifier_predictions,
axis=1)
Ymax = np.argmax(self.__eval_label, axis=1)
self.__confusion_matrix = ConfusionMatrix(Ymax,
classifier_predictions_max)
self.__status_dump(file_pattern,
self.__confusion_matrix,
html=True,
string=True,
pickle=True,
stats_as_txt=True,
latex=True)
def __status_dump(self,
file_pattern,
confusion_matrix,
html=False,
string=False,
pickle=False,
stats_as_txt=False,
latex=False):
dataframe = self.__confusion_matrix.to_dataframe()
print('Report!')
print('file_pattern ', file_pattern)
if html:
with open(file_pattern.format('confusion_matrix', '.html'),
'w') as file:
file.write(dataframe.to_html())
if string:
with open(file_pattern.format('confusion_matrix', '.txt'),
'w') as file:
file.write(dataframe.to_string())
if pickle:
dataframe.to_pickle(
file_pattern.format('confusion_matrix', '.pickle'))
if stats_as_txt:
with open(file_pattern.format('stats', '.txt'), 'w') as file:
file.write(str(confusion_matrix.stats()))
if latex:
with open(file_pattern.format('confusion_matrix', '.latex_table'),
'w') as file:
file.write(dataframe.to_latex())
def get_classifier(self,
activation=None,
loss_function=None,
optimizer=None,
use_last_dim_as_classifier_dim=None,
classifier_dim=None):
if self.__trained and self.__validated:
classifier = EncoderWithClassifier(
self.__encoder_model,
name=self.__name + '_classifier',
activation=activation,
loss_function=loss_function,
optimizer=optimizer,
use_last_dim_as_classifier_dim=use_last_dim_as_classifier_dim,
classifier_dim=classifier_dim)
return classifier
else:
logging.info(
"impossible to create a classifier. Autoencoder isn't trained or validated!"
)
return None
def eval(self, feature=None):
self.__eval_feature = feature
self.__classifier_predictions = self.__classifier.predict(feature)
def eval_stats(self):
self.__stats()
@property
def encoder_model(self):
return self.__encoder_model
@property
def decoder_model(self):
return self.__decoder_model
@property
def autoencoder(self):
return self.__autoencoder
@property
def training_history(self):
return self.__history
@property
def summary(self):
return self.__summary
