import logging
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
import os
import deepthought
DATA_PATH = os.path.join(deepthought.DATA_PATH, 'rwanda2013rhythms')
MODEL_PATH = os.path.join(deepthought.OUTPUT_PATH, 'nips2014', 'models', 'h0')
OUTPUT_PATH = os.path.join(deepthought.OUTPUT_PATH, 'nips2014', 'figures',
'h0')
print 'data path : {}'.format(DATA_PATH)
print 'model path : {}'.format(MODEL_PATH)
print 'output path: {}'.format(OUTPUT_PATH)
# test with subject 4
# WARNING: code seems to be broken due to library update!
from deepthought.experiments.nips2014.scripts.generate_plots import load_results
from deepthought.pylearn2ext.util import process_dataset
path4 = os.path.join(MODEL_PATH, '4', 'best')
train, model = load_results(path4)
dataset = train.algorithm.monitoring_dataset['test']
y_real, y_pred, output = process_dataset(model, dataset)
# subject 4 analysis
from deepthought.experiments.nips2014.scripts.generate_plots import multi_level_accuracy_analysis
multi_level_accuracy_analysis(y_real, y_pred)
from deepthought.pylearn2ext.util import aggregate_classification
t_real, t_pred, t_predf, t_predp = aggregate_classification(
dataset.trial_partitions, y_real, y_pred, output)
multi_level_accuracy_analysis(t_real, t_pred)
def on_monitor(self, model, dataset, algorithm):
# print 'self.dataset: {}\t {} '.format(self.dataset.name, self.dataset);
# print self.dataset.X[0,0:5];
y_real, y_pred, output = process_dataset(model,
self.dataset,
data_specs=self.data_specs,
output_fn=self.output_fn)
if self.header is not None:
print self.header
# Compute confusion matrix
# print classification_report(y_real, y_pred);
conf_matrix = confusion_matrix(y_real, y_pred)
# if self.dataset.name == 'test':
# print conf_matrix;
# log values in monitoring channels
channels = model.monitor.channels
if hasattr(self.dataset, 'get_class_labels'):
class_labels = self.dataset.get_class_labels()
else:
class_labels = ['0', '1']
# FIXME: more flexible fallback required
# p, r, f1, s = precision_recall_fscore_support(y_real, y_pred, average=None);
p, r, f1 = precision_recall_fscore_support(y_real,
y_pred,
average=None)[0:3]
mean_f1 = np.mean(f1)
misclass = (y_real != y_pred).mean()
report = [['frames', mean_f1, misclass]]
channels[self.channel_prefix + '_f1_mean'].val_record[-1] = mean_f1
if self.class_prf1_channels:
for i, class_label in enumerate(class_labels):
channels[self.channel_prefix + '_precision_' +
str(class_label)].val_record[-1] = p[i]
channels[self.channel_prefix + '_recall_' +
str(class_label)].val_record[-1] = r[i]
channels[self.channel_prefix + '_f1_' +
str(class_label)].val_record[-1] = f1[i]
if self.confusion_channels:
# add channels for confusion matrix
for i, c1 in enumerate(class_labels):
for j, c2 in enumerate(class_labels):
channels[self.channel_prefix + '_confusion_' + c1 +
'_as_' + c2].val_record[-1] = conf_matrix[i][j]
if self.dataset.name == 'test':
print confusion_matrix(y_real, y_pred)
if hasattr(self.dataset, 'sequence_partitions'):
# print 'sequence-aggregated performance';
s_real, s_pred, s_predf, s_predp = aggregate_classification(
self.dataset.sequence_partitions, y_real, y_pred, output)
# NOTE: uses weighted version for printout
# both, weighted and un-weighted are logged in the monitor for plotting
# p, r, f1, s = precision_recall_fscore_support(s_real, s_pred, average=None);
p, r, f1 = precision_recall_fscore_support(s_real,
s_pred,
average=None)[0:3]
s_mean_f1 = np.mean(f1)
# p, r, f1, s = precision_recall_fscore_support(s_real, s_predf, average=None);
p, r, f1 = precision_recall_fscore_support(s_real,
s_predf,
average=None)[0:3]
ws_mean_f1 = np.mean(f1)
# p, r, f1, s = precision_recall_fscore_support(s_real, s_predp, average=None);
p, r, f1 = precision_recall_fscore_support(s_real,
s_predp,
average=None)[0:3]
ps_mean_f1 = np.mean(f1)
# print classification_report(s_real, s_predf);
# print confusion_matrix(s_real, s_predf);
s_misclass = (s_real != s_pred).mean()
ws_misclass = (s_real != s_predf).mean()
ps_misclass = (s_real != s_predp).mean()
report.append(['sequences', s_mean_f1, s_misclass])
report.append(['w. sequences', ws_mean_f1, ws_misclass])
report.append(['p. sequences', ps_mean_f1, ps_misclass])
# print 'seq misclass {:.4f}'.format(s_misclass);
# print 'weighted seq misclass {:.4f}'.format(ws_misclass);
channels[self.channel_prefix +
'_seq_misclass_rate'].val_record[-1] = s_misclass
channels[self.channel_prefix +
'_wseq_misclass_rate'].val_record[-1] = ws_misclass
channels[self.channel_prefix +
'_pseq_misclass_rate'].val_record[-1] = ps_misclass
channels[self.channel_prefix +
'_seq_mean_f1'].val_record[-1] = s_mean_f1
channels[self.channel_prefix +
'_wseq_mean_f1'].val_record[-1] = ws_mean_f1
channels[self.channel_prefix +
'_pseq_mean_f1'].val_record[-1] = ps_mean_f1
if hasattr(self.dataset, 'trial_partitions'):
# print 'trial-aggregated performance';
t_real, t_pred, t_predf, t_predp = aggregate_classification(
self.dataset.trial_partitions, y_real, y_pred, output)
# NOTE: uses un-weighted version
# both, weighted and un-weighted are logged in the monitor for plotting
# p, r, f1, s = precision_recall_fscore_support(t_real, t_pred, average=None);
p, r, f1 = precision_recall_fscore_support(t_real,
t_pred,
average=None)[0:3]
t_mean_f1 = np.mean(f1)
# p, r, f1, s = precision_recall_fscore_support(t_real, t_predf, average=None);
p, r, f1 = precision_recall_fscore_support(t_real,
t_predf,
average=None)[0:3]
wt_mean_f1 = np.mean(f1)
# p, r, f1, s = precision_recall_fscore_support(t_real, t_predp, average=None);
p, r, f1 = precision_recall_fscore_support(t_real,
t_predp,
average=None)[0:3]
pt_mean_f1 = np.mean(f1)
# print classification_report(t_real, t_pred);
# if self.dataset.name == 'test':
# print confusion_matrix(t_real, t_predp);
t_misclass = (t_real != t_pred).mean()
wt_misclass = (t_real != t_predf).mean()
pt_misclass = (t_real != t_predp).mean()
report.append(['trials', t_mean_f1, t_misclass])
report.append(['w. trials', wt_mean_f1, wt_misclass])
report.append(['p. trials', pt_mean_f1, pt_misclass])
# print 'trial misclass {:.4f}'.format(t_misclass);
# print 'weighted trial misclass {:.4f}'.format(wt_misclass);
channels[self.channel_prefix +
'_trial_misclass_rate'].val_record[-1] = t_misclass
channels[self.channel_prefix +
'_wtrial_misclass_rate'].val_record[-1] = wt_misclass
channels[self.channel_prefix +
'_ptrial_misclass_rate'].val_record[-1] = pt_misclass
channels[self.channel_prefix +
'_trial_mean_f1'].val_record[-1] = t_mean_f1
channels[self.channel_prefix +
'_wtrial_mean_f1'].val_record[-1] = wt_mean_f1
channels[self.channel_prefix +
'_ptrial_mean_f1'].val_record[-1] = pt_mean_f1
for label, f1, misclass in report:
print '{:>15}: f1 = {:.3f} mc = {:.3f}'.format(
label, f1, misclass)
def extract_cube(experiment_path, best_epoch, config):
# load best model
model_file = os.path.join(experiment_path, 'epochs',
'epoch{}.pkl'.format(best_epoch))
print 'loading ' + model_file
model = serial.load(model_file)
# print model;
# additional dataset params
config.start_sample = 11200
config.stop_sample = 12800
config.name = 'test'
# load dataset
dataset, dataset_yaml = load_yaml_file(
os.path.join(os.path.dirname(__file__), '..', 'run',
'dataset_template.yaml'),
params=config,
)
with log_timing(log, 'processing dataset'):
y_real, y_pred, output = process_dataset(model, dataset)
print classification_report(y_real, y_pred)
print confusion_matrix(y_real, y_pred)
misclass = np.not_equal(y_real, y_pred).astype(int)
print misclass.mean()
print '----- sequence aggregration -----'
s_real, s_pred, s_predf, s_predp = aggregate_classification(
dataset.sequence_partitions, y_real, y_pred, output)
print classification_report(s_real, s_predf)
print confusion_matrix(s_real, s_predf)
print(s_real != s_predf).mean()
print '----- channel aggregration -----'
t_real, t_pred, t_predf, t_predp = aggregate_classification(
dataset.trial_partitions, y_real, y_pred, output)
print classification_report(t_real, t_predf)
print confusion_matrix(t_real, t_predf)
print(t_real != t_predf).mean()
cube = DataCube()
cube.add(dataset.metadata, misclass)
for cat, entry in cube.store.items():
print cat
for key, values in cube.store[cat].items():
print '{:>30} : {:.3f}'.format(key, np.mean(values))
print np.mean(cube.get_entries())
header = ' | '
for c in xrange(18):
header += ' {:2} '.format(c)
header += ' avg '
print header
for r in xrange(48):
line = '{:>3} | '.format(r)
for c in xrange(18):
line += ' ' + cube.get_entries_mean_str(channels=[c], stimuli=[r])
line += ' ' + cube.get_entries_mean_str(stimuli=[r])
# average over all channels
print line
print
line = '{:>3} | '.format('avg')
for c in xrange(18):
line += ' ' + cube.get_entries_mean_str(channels=[c])
# average over all stimuli
line += ' ' + cube.get_entries_mean_str()
# average over all stimuli and channels
print line
return cube
def extract_cube(experiment_path, best_epoch, config):
# load best model
model_file = os.path.join(experiment_path, "epochs", "epoch{}.pkl".format(best_epoch))
print "loading " + model_file
model = serial.load(model_file)
# print model;
# additional dataset params
config.start_sample = 11200
config.stop_sample = 12800
config.name = "test"
# load dataset
dataset, dataset_yaml = load_yaml_file(
os.path.join(os.path.dirname(__file__), "..", "run", "dataset_template.yaml"), params=config
)
with log_timing(log, "processing dataset"):
y_real, y_pred, output = process_dataset(model, dataset)
print classification_report(y_real, y_pred)
print confusion_matrix(y_real, y_pred)
misclass = np.not_equal(y_real, y_pred).astype(int)
print misclass.mean()
print "----- sequence aggregration -----"
s_real, s_pred, s_predf, s_predp = aggregate_classification(dataset.sequence_partitions, y_real, y_pred, output)
print classification_report(s_real, s_predf)
print confusion_matrix(s_real, s_predf)
print (s_real != s_predf).mean()
print "----- channel aggregration -----"
t_real, t_pred, t_predf, t_predp = aggregate_classification(dataset.trial_partitions, y_real, y_pred, output)
print classification_report(t_real, t_predf)
print confusion_matrix(t_real, t_predf)
print (t_real != t_predf).mean()
cube = DataCube()
cube.add(dataset.metadata, misclass)
for cat, entry in cube.store.items():
print cat
for key, values in cube.store[cat].items():
print "{:>30} : {:.3f}".format(key, np.mean(values))
print np.mean(cube.get_entries())
header = " | "
for c in xrange(18):
header += " {:2} ".format(c)
header += " avg "
print header
for r in xrange(48):
line = "{:>3} | ".format(r)
for c in xrange(18):
line += " " + cube.get_entries_mean_str(channels=[c], stimuli=[r])
line += " " + cube.get_entries_mean_str(stimuli=[r])
# average over all channels
print line
print
line = "{:>3} | ".format("avg")
for c in xrange(18):
line += " " + cube.get_entries_mean_str(channels=[c])
# average over all stimuli
line += " " + cube.get_entries_mean_str()
# average over all stimuli and channels
print line
return cube
logger.setLevel(logging.DEBUG)
import os;
import deepthought;
DATA_PATH = os.path.join(deepthought.DATA_PATH, 'rwanda2013rhythms');
MODEL_PATH = os.path.join(deepthought.OUTPUT_PATH, 'nips2014', 'models', 'h0');
OUTPUT_PATH = os.path.join(deepthought.OUTPUT_PATH, 'nips2014', 'figures', 'h0');
print 'data path : {}'.format(DATA_PATH);
print 'model path : {}'.format(MODEL_PATH);
print 'output path: {}'.format(OUTPUT_PATH);
# test with subject 4
# WARNING: code seems to be broken due to library update!
from deepthought.experiments.nips2014.scripts.generate_plots import load_results;
from deepthought.pylearn2ext.util import process_dataset;
path4 = os.path.join(MODEL_PATH, '4', 'best');
train, model = load_results(path4);
dataset = train.algorithm.monitoring_dataset['test'];
y_real, y_pred, output = process_dataset(model, dataset);
# subject 4 analysis
from deepthought.experiments.nips2014.scripts.generate_plots import multi_level_accuracy_analysis;
multi_level_accuracy_analysis(y_real, y_pred);
from deepthought.pylearn2ext.util import aggregate_classification
t_real, t_pred, t_predf, t_predp = aggregate_classification(dataset.trial_partitions, y_real, y_pred, output);
multi_level_accuracy_analysis(t_real, t_pred);
def on_monitor(self, model, dataset, algorithm):
# print 'self.dataset: {}\t {} '.format(self.dataset.name, self.dataset);
# print self.dataset.X[0,0:5];
y_real, y_pred, output = process_dataset(model,
self.dataset,
data_specs=self.data_specs,
output_fn=self.output_fn)
if self.header is not None:
print self.header;
# Compute confusion matrix
# print classification_report(y_real, y_pred);
conf_matrix = confusion_matrix(y_real, y_pred);
# if self.dataset.name == 'test':
# print conf_matrix;
# log values in monitoring channels
channels = model.monitor.channels;
if hasattr(self.dataset, 'get_class_labels'):
class_labels = self.dataset.get_class_labels();
else:
class_labels = ['0', '1']; # FIXME: more flexible fallback required
# p, r, f1, s = precision_recall_fscore_support(y_real, y_pred, average=None);
p, r, f1 = precision_recall_fscore_support(y_real, y_pred, average=None)[0:3];
mean_f1 = np.mean(f1);
misclass = (y_real != y_pred).mean();
report = [['frames', mean_f1, misclass]];
channels[self.channel_prefix+'_f1_mean'].val_record[-1] = mean_f1;
if self.class_prf1_channels:
for i, class_label in enumerate(class_labels):
channels[self.channel_prefix+'_precision_'+str(class_label)].val_record[-1] = p[i];
channels[self.channel_prefix+'_recall_'+str(class_label)].val_record[-1] = r[i];
channels[self.channel_prefix+'_f1_'+str(class_label)].val_record[-1] = f1[i];
if self.confusion_channels:
# add channels for confusion matrix
for i, c1 in enumerate(class_labels):
for j, c2 in enumerate(class_labels):
channels[self.channel_prefix+'_confusion_'+c1+'_as_'+c2].val_record[-1] = conf_matrix[i][j];
if self.dataset.name == 'test':
print confusion_matrix(y_real, y_pred);
if hasattr(self.dataset, 'sequence_partitions'):
# print 'sequence-aggregated performance';
s_real, s_pred, s_predf, s_predp = aggregate_classification(
self.dataset.sequence_partitions,
y_real, y_pred, output);
# NOTE: uses weighted version for printout
# both, weighted and un-weighted are logged in the monitor for plotting
# p, r, f1, s = precision_recall_fscore_support(s_real, s_pred, average=None);
p, r, f1 = precision_recall_fscore_support(s_real, s_pred, average=None)[0:3];
s_mean_f1 = np.mean(f1);
# p, r, f1, s = precision_recall_fscore_support(s_real, s_predf, average=None);
p, r, f1 = precision_recall_fscore_support(s_real, s_predf, average=None)[0:3];
ws_mean_f1 = np.mean(f1);
# p, r, f1, s = precision_recall_fscore_support(s_real, s_predp, average=None);
p, r, f1 = precision_recall_fscore_support(s_real, s_predp, average=None)[0:3];
ps_mean_f1 = np.mean(f1);
# print classification_report(s_real, s_predf);
# print confusion_matrix(s_real, s_predf);
s_misclass = (s_real != s_pred).mean();
ws_misclass = (s_real != s_predf).mean();
ps_misclass = (s_real != s_predp).mean();
report.append(['sequences', s_mean_f1, s_misclass]);
report.append(['w. sequences', ws_mean_f1, ws_misclass]);
report.append(['p. sequences', ps_mean_f1, ps_misclass]);
# print 'seq misclass {:.4f}'.format(s_misclass);
# print 'weighted seq misclass {:.4f}'.format(ws_misclass);
channels[self.channel_prefix+'_seq_misclass_rate'].val_record[-1] = s_misclass;
channels[self.channel_prefix+'_wseq_misclass_rate'].val_record[-1] = ws_misclass;
channels[self.channel_prefix+'_pseq_misclass_rate'].val_record[-1] = ps_misclass;
channels[self.channel_prefix+'_seq_mean_f1'].val_record[-1] = s_mean_f1;
channels[self.channel_prefix+'_wseq_mean_f1'].val_record[-1] = ws_mean_f1;
channels[self.channel_prefix+'_pseq_mean_f1'].val_record[-1] = ps_mean_f1;
if hasattr(self.dataset, 'trial_partitions'):
# print 'trial-aggregated performance';
t_real, t_pred, t_predf, t_predp = aggregate_classification(
self.dataset.trial_partitions,
y_real, y_pred, output);
# NOTE: uses un-weighted version
# both, weighted and un-weighted are logged in the monitor for plotting
# p, r, f1, s = precision_recall_fscore_support(t_real, t_pred, average=None);
p, r, f1 = precision_recall_fscore_support(t_real, t_pred, average=None)[0:3];
t_mean_f1 = np.mean(f1);
# p, r, f1, s = precision_recall_fscore_support(t_real, t_predf, average=None);
p, r, f1 = precision_recall_fscore_support(t_real, t_predf, average=None)[0:3];
wt_mean_f1 = np.mean(f1);
# p, r, f1, s = precision_recall_fscore_support(t_real, t_predp, average=None);
p, r, f1 = precision_recall_fscore_support(t_real, t_predp, average=None)[0:3];
pt_mean_f1 = np.mean(f1);
# print classification_report(t_real, t_pred);
# if self.dataset.name == 'test':
# print confusion_matrix(t_real, t_predp);
t_misclass = (t_real != t_pred).mean();
wt_misclass = (t_real != t_predf).mean();
pt_misclass = (t_real != t_predp).mean();
report.append(['trials', t_mean_f1, t_misclass]);
report.append(['w. trials', wt_mean_f1, wt_misclass]);
report.append(['p. trials', pt_mean_f1, pt_misclass]);
# print 'trial misclass {:.4f}'.format(t_misclass);
# print 'weighted trial misclass {:.4f}'.format(wt_misclass);
channels[self.channel_prefix+'_trial_misclass_rate'].val_record[-1] = t_misclass;
channels[self.channel_prefix+'_wtrial_misclass_rate'].val_record[-1] = wt_misclass;
channels[self.channel_prefix+'_ptrial_misclass_rate'].val_record[-1] = pt_misclass;
channels[self.channel_prefix+'_trial_mean_f1'].val_record[-1] = t_mean_f1;
channels[self.channel_prefix+'_wtrial_mean_f1'].val_record[-1] = wt_mean_f1;
channels[self.channel_prefix+'_ptrial_mean_f1'].val_record[-1] = pt_mean_f1;
for label, f1, misclass in report:
print '{:>15}: f1 = {:.3f} mc = {:.3f}'.format(label, f1, misclass);