Exemple #1
0
import logging
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)

import os
import deepthought
DATA_PATH = os.path.join(deepthought.DATA_PATH, 'rwanda2013rhythms')
MODEL_PATH = os.path.join(deepthought.OUTPUT_PATH, 'nips2014', 'models', 'h0')
OUTPUT_PATH = os.path.join(deepthought.OUTPUT_PATH, 'nips2014', 'figures',
                           'h0')
print 'data path  : {}'.format(DATA_PATH)
print 'model path : {}'.format(MODEL_PATH)
print 'output path: {}'.format(OUTPUT_PATH)

# test with subject 4
# WARNING: code seems to be broken due to library update!
from deepthought.experiments.nips2014.scripts.generate_plots import load_results
from deepthought.pylearn2ext.util import process_dataset
path4 = os.path.join(MODEL_PATH, '4', 'best')
train, model = load_results(path4)
dataset = train.algorithm.monitoring_dataset['test']
y_real, y_pred, output = process_dataset(model, dataset)

# subject 4 analysis
from deepthought.experiments.nips2014.scripts.generate_plots import multi_level_accuracy_analysis
multi_level_accuracy_analysis(y_real, y_pred)

from deepthought.pylearn2ext.util import aggregate_classification
t_real, t_pred, t_predf, t_predp = aggregate_classification(
    dataset.trial_partitions, y_real, y_pred, output)
multi_level_accuracy_analysis(t_real, t_pred)
    def on_monitor(self, model, dataset, algorithm):

        #         print 'self.dataset: {}\t {} '.format(self.dataset.name, self.dataset);

        #         print self.dataset.X[0,0:5];

        y_real, y_pred, output = process_dataset(model,
                                                 self.dataset,
                                                 data_specs=self.data_specs,
                                                 output_fn=self.output_fn)

        if self.header is not None:
            print self.header

        # Compute confusion matrix
#         print classification_report(y_real, y_pred);
        conf_matrix = confusion_matrix(y_real, y_pred)

        #         if self.dataset.name == 'test':
        #             print conf_matrix;

        # log values in monitoring channels
        channels = model.monitor.channels

        if hasattr(self.dataset, 'get_class_labels'):
            class_labels = self.dataset.get_class_labels()
        else:
            class_labels = ['0', '1']
            # FIXME: more flexible fallback required

#         p, r, f1, s = precision_recall_fscore_support(y_real, y_pred, average=None);
        p, r, f1 = precision_recall_fscore_support(y_real,
                                                   y_pred,
                                                   average=None)[0:3]

        mean_f1 = np.mean(f1)
        misclass = (y_real != y_pred).mean()
        report = [['frames', mean_f1, misclass]]

        channels[self.channel_prefix + '_f1_mean'].val_record[-1] = mean_f1

        if self.class_prf1_channels:
            for i, class_label in enumerate(class_labels):
                channels[self.channel_prefix + '_precision_' +
                         str(class_label)].val_record[-1] = p[i]
                channels[self.channel_prefix + '_recall_' +
                         str(class_label)].val_record[-1] = r[i]
                channels[self.channel_prefix + '_f1_' +
                         str(class_label)].val_record[-1] = f1[i]

        if self.confusion_channels:
            # add channels for confusion matrix
            for i, c1 in enumerate(class_labels):
                for j, c2 in enumerate(class_labels):
                    channels[self.channel_prefix + '_confusion_' + c1 +
                             '_as_' + c2].val_record[-1] = conf_matrix[i][j]

        if self.dataset.name == 'test':
            print confusion_matrix(y_real, y_pred)

        if hasattr(self.dataset, 'sequence_partitions'):
            #             print 'sequence-aggregated performance';

            s_real, s_pred, s_predf, s_predp = aggregate_classification(
                self.dataset.sequence_partitions, y_real, y_pred, output)
            # NOTE: uses weighted version for printout
            # both, weighted and un-weighted are logged in the monitor for plotting

            #             p, r, f1, s = precision_recall_fscore_support(s_real, s_pred, average=None);
            p, r, f1 = precision_recall_fscore_support(s_real,
                                                       s_pred,
                                                       average=None)[0:3]
            s_mean_f1 = np.mean(f1)

            #             p, r, f1, s = precision_recall_fscore_support(s_real, s_predf, average=None);
            p, r, f1 = precision_recall_fscore_support(s_real,
                                                       s_predf,
                                                       average=None)[0:3]
            ws_mean_f1 = np.mean(f1)

            #             p, r, f1, s = precision_recall_fscore_support(s_real, s_predp, average=None);
            p, r, f1 = precision_recall_fscore_support(s_real,
                                                       s_predp,
                                                       average=None)[0:3]
            ps_mean_f1 = np.mean(f1)

            #             print classification_report(s_real, s_predf);
            #             print confusion_matrix(s_real, s_predf);

            s_misclass = (s_real != s_pred).mean()
            ws_misclass = (s_real != s_predf).mean()
            ps_misclass = (s_real != s_predp).mean()

            report.append(['sequences', s_mean_f1, s_misclass])
            report.append(['w. sequences', ws_mean_f1, ws_misclass])
            report.append(['p. sequences', ps_mean_f1, ps_misclass])

            #             print 'seq misclass {:.4f}'.format(s_misclass);
            #             print 'weighted seq misclass {:.4f}'.format(ws_misclass);

            channels[self.channel_prefix +
                     '_seq_misclass_rate'].val_record[-1] = s_misclass
            channels[self.channel_prefix +
                     '_wseq_misclass_rate'].val_record[-1] = ws_misclass
            channels[self.channel_prefix +
                     '_pseq_misclass_rate'].val_record[-1] = ps_misclass

            channels[self.channel_prefix +
                     '_seq_mean_f1'].val_record[-1] = s_mean_f1
            channels[self.channel_prefix +
                     '_wseq_mean_f1'].val_record[-1] = ws_mean_f1
            channels[self.channel_prefix +
                     '_pseq_mean_f1'].val_record[-1] = ps_mean_f1

        if hasattr(self.dataset, 'trial_partitions'):
            #             print 'trial-aggregated performance';

            t_real, t_pred, t_predf, t_predp = aggregate_classification(
                self.dataset.trial_partitions, y_real, y_pred, output)
            # NOTE: uses un-weighted version
            # both, weighted and un-weighted are logged in the monitor for plotting

            #             p, r, f1, s = precision_recall_fscore_support(t_real, t_pred, average=None);
            p, r, f1 = precision_recall_fscore_support(t_real,
                                                       t_pred,
                                                       average=None)[0:3]
            t_mean_f1 = np.mean(f1)

            #             p, r, f1, s = precision_recall_fscore_support(t_real, t_predf, average=None);
            p, r, f1 = precision_recall_fscore_support(t_real,
                                                       t_predf,
                                                       average=None)[0:3]
            wt_mean_f1 = np.mean(f1)

            #             p, r, f1, s = precision_recall_fscore_support(t_real, t_predp, average=None);
            p, r, f1 = precision_recall_fscore_support(t_real,
                                                       t_predp,
                                                       average=None)[0:3]
            pt_mean_f1 = np.mean(f1)

            #             print classification_report(t_real, t_pred);

            #             if self.dataset.name == 'test':
            #                 print confusion_matrix(t_real, t_predp);

            t_misclass = (t_real != t_pred).mean()
            wt_misclass = (t_real != t_predf).mean()
            pt_misclass = (t_real != t_predp).mean()

            report.append(['trials', t_mean_f1, t_misclass])
            report.append(['w. trials', wt_mean_f1, wt_misclass])
            report.append(['p. trials', pt_mean_f1, pt_misclass])

            #             print 'trial misclass {:.4f}'.format(t_misclass);
            #             print 'weighted trial misclass {:.4f}'.format(wt_misclass);

            channels[self.channel_prefix +
                     '_trial_misclass_rate'].val_record[-1] = t_misclass
            channels[self.channel_prefix +
                     '_wtrial_misclass_rate'].val_record[-1] = wt_misclass
            channels[self.channel_prefix +
                     '_ptrial_misclass_rate'].val_record[-1] = pt_misclass

            channels[self.channel_prefix +
                     '_trial_mean_f1'].val_record[-1] = t_mean_f1
            channels[self.channel_prefix +
                     '_wtrial_mean_f1'].val_record[-1] = wt_mean_f1
            channels[self.channel_prefix +
                     '_ptrial_mean_f1'].val_record[-1] = pt_mean_f1

        for label, f1, misclass in report:
            print '{:>15}:  f1 = {:.3f}  mc = {:.3f}'.format(
                label, f1, misclass)
def extract_cube(experiment_path, best_epoch, config):

    # load best model
    model_file = os.path.join(experiment_path, 'epochs',
                              'epoch{}.pkl'.format(best_epoch))
    print 'loading ' + model_file
    model = serial.load(model_file)

    #     print model;

    # additional dataset params
    config.start_sample = 11200
    config.stop_sample = 12800
    config.name = 'test'

    # load dataset
    dataset, dataset_yaml = load_yaml_file(
        os.path.join(os.path.dirname(__file__), '..', 'run',
                     'dataset_template.yaml'),
        params=config,
    )

    with log_timing(log, 'processing dataset'):
        y_real, y_pred, output = process_dataset(model, dataset)

    print classification_report(y_real, y_pred)
    print confusion_matrix(y_real, y_pred)
    misclass = np.not_equal(y_real, y_pred).astype(int)
    print misclass.mean()

    print '----- sequence aggregration -----'
    s_real, s_pred, s_predf, s_predp = aggregate_classification(
        dataset.sequence_partitions, y_real, y_pred, output)

    print classification_report(s_real, s_predf)
    print confusion_matrix(s_real, s_predf)
    print(s_real != s_predf).mean()

    print '----- channel aggregration -----'
    t_real, t_pred, t_predf, t_predp = aggregate_classification(
        dataset.trial_partitions, y_real, y_pred, output)

    print classification_report(t_real, t_predf)
    print confusion_matrix(t_real, t_predf)
    print(t_real != t_predf).mean()

    cube = DataCube()
    cube.add(dataset.metadata, misclass)

    for cat, entry in cube.store.items():
        print cat
        for key, values in cube.store[cat].items():
            print '{:>30} : {:.3f}'.format(key, np.mean(values))

    print np.mean(cube.get_entries())

    header = '    | '
    for c in xrange(18):
        header += '  {:2}  '.format(c)
    header += '    avg  '
    print header

    for r in xrange(48):
        line = '{:>3} | '.format(r)
        for c in xrange(18):
            line += ' ' + cube.get_entries_mean_str(channels=[c], stimuli=[r])
        line += '   ' + cube.get_entries_mean_str(stimuli=[r])
        # average over all channels
        print line

    print
    line = '{:>3} | '.format('avg')
    for c in xrange(18):
        line += ' ' + cube.get_entries_mean_str(channels=[c])
        # average over all stimuli
    line += '   ' + cube.get_entries_mean_str()
    # average over all stimuli and channels
    print line

    return cube
def extract_cube(experiment_path, best_epoch, config):

    # load best model
    model_file = os.path.join(experiment_path, "epochs", "epoch{}.pkl".format(best_epoch))
    print "loading " + model_file
    model = serial.load(model_file)

    #     print model;

    # additional dataset params
    config.start_sample = 11200
    config.stop_sample = 12800
    config.name = "test"

    # load dataset
    dataset, dataset_yaml = load_yaml_file(
        os.path.join(os.path.dirname(__file__), "..", "run", "dataset_template.yaml"), params=config
    )

    with log_timing(log, "processing dataset"):
        y_real, y_pred, output = process_dataset(model, dataset)

    print classification_report(y_real, y_pred)
    print confusion_matrix(y_real, y_pred)
    misclass = np.not_equal(y_real, y_pred).astype(int)
    print misclass.mean()

    print "----- sequence aggregration -----"
    s_real, s_pred, s_predf, s_predp = aggregate_classification(dataset.sequence_partitions, y_real, y_pred, output)

    print classification_report(s_real, s_predf)
    print confusion_matrix(s_real, s_predf)
    print (s_real != s_predf).mean()

    print "----- channel aggregration -----"
    t_real, t_pred, t_predf, t_predp = aggregate_classification(dataset.trial_partitions, y_real, y_pred, output)

    print classification_report(t_real, t_predf)
    print confusion_matrix(t_real, t_predf)
    print (t_real != t_predf).mean()

    cube = DataCube()
    cube.add(dataset.metadata, misclass)

    for cat, entry in cube.store.items():
        print cat
        for key, values in cube.store[cat].items():
            print "{:>30} : {:.3f}".format(key, np.mean(values))

    print np.mean(cube.get_entries())

    header = "    | "
    for c in xrange(18):
        header += "  {:2}  ".format(c)
    header += "    avg  "
    print header

    for r in xrange(48):
        line = "{:>3} | ".format(r)
        for c in xrange(18):
            line += " " + cube.get_entries_mean_str(channels=[c], stimuli=[r])
        line += "   " + cube.get_entries_mean_str(stimuli=[r])
        # average over all channels
        print line

    print
    line = "{:>3} | ".format("avg")
    for c in xrange(18):
        line += " " + cube.get_entries_mean_str(channels=[c])
        # average over all stimuli
    line += "   " + cube.get_entries_mean_str()
    # average over all stimuli and channels
    print line

    return cube
Exemple #5
0
logger.setLevel(logging.DEBUG)

import os;
import deepthought;
DATA_PATH = os.path.join(deepthought.DATA_PATH, 'rwanda2013rhythms');
MODEL_PATH = os.path.join(deepthought.OUTPUT_PATH, 'nips2014', 'models', 'h0');
OUTPUT_PATH = os.path.join(deepthought.OUTPUT_PATH, 'nips2014', 'figures', 'h0');
print 'data path  : {}'.format(DATA_PATH);
print 'model path : {}'.format(MODEL_PATH);
print 'output path: {}'.format(OUTPUT_PATH);



# test with subject 4
# WARNING: code seems to be broken due to library update!
from deepthought.experiments.nips2014.scripts.generate_plots import load_results;
from deepthought.pylearn2ext.util import process_dataset;
path4 = os.path.join(MODEL_PATH, '4', 'best');
train, model = load_results(path4);
dataset = train.algorithm.monitoring_dataset['test'];
y_real, y_pred, output = process_dataset(model, dataset);


# subject 4 analysis
from deepthought.experiments.nips2014.scripts.generate_plots import multi_level_accuracy_analysis;
multi_level_accuracy_analysis(y_real, y_pred);


from deepthought.pylearn2ext.util import aggregate_classification
t_real, t_pred, t_predf, t_predp = aggregate_classification(dataset.trial_partitions, y_real, y_pred, output);  
multi_level_accuracy_analysis(t_real, t_pred);
    def on_monitor(self, model, dataset, algorithm):
        
#         print 'self.dataset: {}\t {} '.format(self.dataset.name, self.dataset);
        
#         print self.dataset.X[0,0:5];
                
        y_real, y_pred, output = process_dataset(model, 
                                                 self.dataset, 
                                                 data_specs=self.data_specs, 
                                                 output_fn=self.output_fn)
        
        if self.header is not None:
            print self.header;                            

        # Compute confusion matrix
#         print classification_report(y_real, y_pred);
        conf_matrix = confusion_matrix(y_real, y_pred);
        
#         if self.dataset.name == 'test':
#             print conf_matrix;
                
        # log values in monitoring channels
        channels = model.monitor.channels;
        
        
        if hasattr(self.dataset, 'get_class_labels'): 
            class_labels = self.dataset.get_class_labels();
        else:
            class_labels = ['0', '1']; # FIXME: more flexible fallback required
        
#         p, r, f1, s = precision_recall_fscore_support(y_real, y_pred, average=None);
        p, r, f1 = precision_recall_fscore_support(y_real, y_pred, average=None)[0:3];
        
        mean_f1 = np.mean(f1);
        misclass = (y_real != y_pred).mean();
        report = [['frames', mean_f1, misclass]];
        
        channels[self.channel_prefix+'_f1_mean'].val_record[-1] = mean_f1;
        
        if self.class_prf1_channels:
            for i, class_label in enumerate(class_labels):
                channels[self.channel_prefix+'_precision_'+str(class_label)].val_record[-1] = p[i];
                channels[self.channel_prefix+'_recall_'+str(class_label)].val_record[-1] = r[i];
                channels[self.channel_prefix+'_f1_'+str(class_label)].val_record[-1] = f1[i];
        
        if self.confusion_channels:
            # add channels for confusion matrix
            for i, c1 in enumerate(class_labels):
                for j, c2 in enumerate(class_labels):
                    channels[self.channel_prefix+'_confusion_'+c1+'_as_'+c2].val_record[-1] = conf_matrix[i][j];
                    
        if self.dataset.name == 'test':
            print confusion_matrix(y_real, y_pred);

        if hasattr(self.dataset, 'sequence_partitions'):
#             print 'sequence-aggregated performance';
            
            s_real, s_pred, s_predf, s_predp = aggregate_classification(
                                                                          self.dataset.sequence_partitions,
                                                                          y_real, y_pred, output);            
            # NOTE: uses weighted version for printout
            # both, weighted and un-weighted are logged in the monitor for plotting
            
#             p, r, f1, s = precision_recall_fscore_support(s_real, s_pred, average=None);
            p, r, f1 = precision_recall_fscore_support(s_real, s_pred, average=None)[0:3];
            s_mean_f1 = np.mean(f1);
            
#             p, r, f1, s = precision_recall_fscore_support(s_real, s_predf, average=None);
            p, r, f1 = precision_recall_fscore_support(s_real, s_predf, average=None)[0:3];
            ws_mean_f1 = np.mean(f1);
            
#             p, r, f1, s = precision_recall_fscore_support(s_real, s_predp, average=None);
            p, r, f1 = precision_recall_fscore_support(s_real, s_predp, average=None)[0:3];
            ps_mean_f1 = np.mean(f1);
            
#             print classification_report(s_real, s_predf);
#             print confusion_matrix(s_real, s_predf);
            
            s_misclass = (s_real != s_pred).mean();
            ws_misclass = (s_real != s_predf).mean();
            ps_misclass = (s_real != s_predp).mean();
            
            report.append(['sequences', s_mean_f1, s_misclass]);
            report.append(['w. sequences', ws_mean_f1, ws_misclass]);
            report.append(['p. sequences', ps_mean_f1, ps_misclass]);
            
#             print 'seq misclass {:.4f}'.format(s_misclass);
#             print 'weighted seq misclass {:.4f}'.format(ws_misclass);
                        
            channels[self.channel_prefix+'_seq_misclass_rate'].val_record[-1] = s_misclass;
            channels[self.channel_prefix+'_wseq_misclass_rate'].val_record[-1] = ws_misclass;
            channels[self.channel_prefix+'_pseq_misclass_rate'].val_record[-1] = ps_misclass;
            
            channels[self.channel_prefix+'_seq_mean_f1'].val_record[-1] = s_mean_f1;
            channels[self.channel_prefix+'_wseq_mean_f1'].val_record[-1] = ws_mean_f1;
            channels[self.channel_prefix+'_pseq_mean_f1'].val_record[-1] = ps_mean_f1;
        
        if hasattr(self.dataset, 'trial_partitions'):
#             print 'trial-aggregated performance';
                        
            t_real, t_pred, t_predf, t_predp = aggregate_classification(
                                                                          self.dataset.trial_partitions,
                                                                          y_real, y_pred, output);            
            # NOTE: uses un-weighted version
            # both, weighted and un-weighted are logged in the monitor for plotting
            
#             p, r, f1, s = precision_recall_fscore_support(t_real, t_pred, average=None);
            p, r, f1 = precision_recall_fscore_support(t_real, t_pred, average=None)[0:3];
            t_mean_f1 = np.mean(f1);
            
#             p, r, f1, s = precision_recall_fscore_support(t_real, t_predf, average=None);
            p, r, f1 = precision_recall_fscore_support(t_real, t_predf, average=None)[0:3];
            wt_mean_f1 = np.mean(f1);
            
#             p, r, f1, s = precision_recall_fscore_support(t_real, t_predp, average=None);
            p, r, f1 = precision_recall_fscore_support(t_real, t_predp, average=None)[0:3];
            pt_mean_f1 = np.mean(f1);
            
#             print classification_report(t_real, t_pred);

#             if self.dataset.name == 'test':
#                 print confusion_matrix(t_real, t_predp);
            
            t_misclass = (t_real != t_pred).mean();
            wt_misclass = (t_real != t_predf).mean();
            pt_misclass = (t_real != t_predp).mean();
            
            report.append(['trials', t_mean_f1, t_misclass]);
            report.append(['w. trials', wt_mean_f1, wt_misclass]);
            report.append(['p. trials', pt_mean_f1, pt_misclass]);

#             print 'trial misclass {:.4f}'.format(t_misclass);
#             print 'weighted trial misclass {:.4f}'.format(wt_misclass);
            
            channels[self.channel_prefix+'_trial_misclass_rate'].val_record[-1] = t_misclass;
            channels[self.channel_prefix+'_wtrial_misclass_rate'].val_record[-1] = wt_misclass;
            channels[self.channel_prefix+'_ptrial_misclass_rate'].val_record[-1] = pt_misclass;
            
            channels[self.channel_prefix+'_trial_mean_f1'].val_record[-1] = t_mean_f1;
            channels[self.channel_prefix+'_wtrial_mean_f1'].val_record[-1] = wt_mean_f1;
            channels[self.channel_prefix+'_ptrial_mean_f1'].val_record[-1] = pt_mean_f1;
        
        for label, f1, misclass in report:
            print '{:>15}:  f1 = {:.3f}  mc = {:.3f}'.format(label, f1, misclass);