def extract_output(config, best_epoch): # load best model model_file = os.path.join(config.experiment_root, 'epochs', 'epoch{}.pkl'.format(best_epoch)) print 'loading ' + model_file model = serial.load(model_file) # print model; # additional dataset params config.start_sample = 11200 config.stop_sample = 12800 config.name = 'test' # load dataset dataset, dataset_yaml = load_yaml_file( os.path.join(os.path.dirname(__file__), '..', 'run', 'dataset_template.yaml'), params=config, ) with log_timing(log, 'processing dataset'): y_real, y_pred, output = process_dataset(model, dataset) return y_real, y_pred, output
def extract_output(experiment_root): train, model = load_results(experiment_root); # get the datasets with their names from the monitor for key, dataset in train.algorithm.monitoring_dataset.items(): # process each dataset with log_timing(log, 'processing dataset \'{}\''.format(key)): y_real, y_pred, output = process_dataset(model, dataset) save(os.path.join(experiment_root, 'cache', key+'_output.pklz'), (y_real, y_pred, output));
def extract_output(experiment_root): train, model = load_results(experiment_root) # get the datasets with their names from the monitor for key, dataset in train.algorithm.monitoring_dataset.items(): # process each dataset with log_timing(log, 'processing dataset \'{}\''.format(key)): y_real, y_pred, output = process_dataset(model, dataset) save(os.path.join(experiment_root, 'cache', key + '_output.pklz'), (y_real, y_pred, output))
def extract_output(config, best_epoch): # load best model model_file = os.path.join(config.experiment_root, "epochs", "epoch{}.pkl".format(best_epoch)) print "loading " + model_file model = serial.load(model_file) # print model; # additional dataset params config.start_sample = 11200 config.stop_sample = 12800 config.name = "test" # load dataset dataset, dataset_yaml = load_yaml_file( os.path.join(os.path.dirname(__file__), "..", "run", "dataset_template.yaml"), params=config ) with log_timing(log, "processing dataset"): y_real, y_pred, output = process_dataset(model, dataset) return y_real, y_pred, output
import logging logger = logging.getLogger() logger.setLevel(logging.DEBUG) import os import deepthought DATA_PATH = os.path.join(deepthought.DATA_PATH, 'rwanda2013rhythms') MODEL_PATH = os.path.join(deepthought.OUTPUT_PATH, 'nips2014', 'models', 'h0') OUTPUT_PATH = os.path.join(deepthought.OUTPUT_PATH, 'nips2014', 'figures', 'h0') print 'data path : {}'.format(DATA_PATH) print 'model path : {}'.format(MODEL_PATH) print 'output path: {}'.format(OUTPUT_PATH) # test with subject 4 # WARNING: code seems to be broken due to library update! from deepthought.experiments.nips2014.scripts.generate_plots import load_results from deepthought.pylearn2ext.util import process_dataset path4 = os.path.join(MODEL_PATH, '4', 'best') train, model = load_results(path4) dataset = train.algorithm.monitoring_dataset['test'] y_real, y_pred, output = process_dataset(model, dataset) # subject 4 analysis from deepthought.experiments.nips2014.scripts.generate_plots import multi_level_accuracy_analysis multi_level_accuracy_analysis(y_real, y_pred) from deepthought.pylearn2ext.util import aggregate_classification t_real, t_pred, t_predf, t_predp = aggregate_classification( dataset.trial_partitions, y_real, y_pred, output) multi_level_accuracy_analysis(t_real, t_pred)
def on_monitor(self, model, dataset, algorithm): # print 'self.dataset: {}\t {} '.format(self.dataset.name, self.dataset); # print self.dataset.X[0,0:5]; y_real, y_pred, output = process_dataset(model, self.dataset, data_specs=self.data_specs, output_fn=self.output_fn) if self.header is not None: print self.header # Compute confusion matrix # print classification_report(y_real, y_pred); conf_matrix = confusion_matrix(y_real, y_pred) # if self.dataset.name == 'test': # print conf_matrix; # log values in monitoring channels channels = model.monitor.channels if hasattr(self.dataset, 'get_class_labels'): class_labels = self.dataset.get_class_labels() else: class_labels = ['0', '1'] # FIXME: more flexible fallback required # p, r, f1, s = precision_recall_fscore_support(y_real, y_pred, average=None); p, r, f1 = precision_recall_fscore_support(y_real, y_pred, average=None)[0:3] mean_f1 = np.mean(f1) misclass = (y_real != y_pred).mean() report = [['frames', mean_f1, misclass]] channels[self.channel_prefix + '_f1_mean'].val_record[-1] = mean_f1 if self.class_prf1_channels: for i, class_label in enumerate(class_labels): channels[self.channel_prefix + '_precision_' + str(class_label)].val_record[-1] = p[i] channels[self.channel_prefix + '_recall_' + str(class_label)].val_record[-1] = r[i] channels[self.channel_prefix + '_f1_' + str(class_label)].val_record[-1] = f1[i] if self.confusion_channels: # add channels for confusion matrix for i, c1 in enumerate(class_labels): for j, c2 in enumerate(class_labels): channels[self.channel_prefix + '_confusion_' + c1 + '_as_' + c2].val_record[-1] = conf_matrix[i][j] if self.dataset.name == 'test': print confusion_matrix(y_real, y_pred) if hasattr(self.dataset, 'sequence_partitions'): # print 'sequence-aggregated performance'; s_real, s_pred, s_predf, s_predp = aggregate_classification( self.dataset.sequence_partitions, y_real, y_pred, output) # NOTE: uses weighted version for printout # both, weighted and un-weighted are logged in the monitor for plotting # p, r, f1, s = precision_recall_fscore_support(s_real, s_pred, average=None); p, r, f1 = precision_recall_fscore_support(s_real, s_pred, average=None)[0:3] s_mean_f1 = np.mean(f1) # p, r, f1, s = precision_recall_fscore_support(s_real, s_predf, average=None); p, r, f1 = precision_recall_fscore_support(s_real, s_predf, average=None)[0:3] ws_mean_f1 = np.mean(f1) # p, r, f1, s = precision_recall_fscore_support(s_real, s_predp, average=None); p, r, f1 = precision_recall_fscore_support(s_real, s_predp, average=None)[0:3] ps_mean_f1 = np.mean(f1) # print classification_report(s_real, s_predf); # print confusion_matrix(s_real, s_predf); s_misclass = (s_real != s_pred).mean() ws_misclass = (s_real != s_predf).mean() ps_misclass = (s_real != s_predp).mean() report.append(['sequences', s_mean_f1, s_misclass]) report.append(['w. sequences', ws_mean_f1, ws_misclass]) report.append(['p. sequences', ps_mean_f1, ps_misclass]) # print 'seq misclass {:.4f}'.format(s_misclass); # print 'weighted seq misclass {:.4f}'.format(ws_misclass); channels[self.channel_prefix + '_seq_misclass_rate'].val_record[-1] = s_misclass channels[self.channel_prefix + '_wseq_misclass_rate'].val_record[-1] = ws_misclass channels[self.channel_prefix + '_pseq_misclass_rate'].val_record[-1] = ps_misclass channels[self.channel_prefix + '_seq_mean_f1'].val_record[-1] = s_mean_f1 channels[self.channel_prefix + '_wseq_mean_f1'].val_record[-1] = ws_mean_f1 channels[self.channel_prefix + '_pseq_mean_f1'].val_record[-1] = ps_mean_f1 if hasattr(self.dataset, 'trial_partitions'): # print 'trial-aggregated performance'; t_real, t_pred, t_predf, t_predp = aggregate_classification( self.dataset.trial_partitions, y_real, y_pred, output) # NOTE: uses un-weighted version # both, weighted and un-weighted are logged in the monitor for plotting # p, r, f1, s = precision_recall_fscore_support(t_real, t_pred, average=None); p, r, f1 = precision_recall_fscore_support(t_real, t_pred, average=None)[0:3] t_mean_f1 = np.mean(f1) # p, r, f1, s = precision_recall_fscore_support(t_real, t_predf, average=None); p, r, f1 = precision_recall_fscore_support(t_real, t_predf, average=None)[0:3] wt_mean_f1 = np.mean(f1) # p, r, f1, s = precision_recall_fscore_support(t_real, t_predp, average=None); p, r, f1 = precision_recall_fscore_support(t_real, t_predp, average=None)[0:3] pt_mean_f1 = np.mean(f1) # print classification_report(t_real, t_pred); # if self.dataset.name == 'test': # print confusion_matrix(t_real, t_predp); t_misclass = (t_real != t_pred).mean() wt_misclass = (t_real != t_predf).mean() pt_misclass = (t_real != t_predp).mean() report.append(['trials', t_mean_f1, t_misclass]) report.append(['w. trials', wt_mean_f1, wt_misclass]) report.append(['p. trials', pt_mean_f1, pt_misclass]) # print 'trial misclass {:.4f}'.format(t_misclass); # print 'weighted trial misclass {:.4f}'.format(wt_misclass); channels[self.channel_prefix + '_trial_misclass_rate'].val_record[-1] = t_misclass channels[self.channel_prefix + '_wtrial_misclass_rate'].val_record[-1] = wt_misclass channels[self.channel_prefix + '_ptrial_misclass_rate'].val_record[-1] = pt_misclass channels[self.channel_prefix + '_trial_mean_f1'].val_record[-1] = t_mean_f1 channels[self.channel_prefix + '_wtrial_mean_f1'].val_record[-1] = wt_mean_f1 channels[self.channel_prefix + '_ptrial_mean_f1'].val_record[-1] = pt_mean_f1 for label, f1, misclass in report: print '{:>15}: f1 = {:.3f} mc = {:.3f}'.format( label, f1, misclass)
train = load_yaml(train_yaml)[0]; return train, model; if __name__ == '__main__': init_logging(pylearn2_loglevel=logging.INFO); parser = argparse.ArgumentParser(prog='generate_plots', description='generates plots ;-)'); # global options parser.add_argument('path', help='root path of the experiment'); args = parser.parse_args(); experiment_root = args.path; # experiment_root = '/Users/sstober/git/deepbeat/deepbeat/spearmint/h0_input47/20041_h0_pattern_width_[47]_h0_patterns_[30]_h0_pool_size_[1]_learning_rate_[0.01]' # path = '/Users/sstober/git/deepbeat/deepbeat/spearmint/best/h0_1bar_nophase_49bins' train, model = load_results(experiment_root); # get the datasets with their names from the monitor for key, dataset in train.algorithm.monitoring_dataset.items(): # process each dataset with log_timing(log, 'processing dataset \'{}\''.format(key)): y_real, y_pred, output = process_dataset(model, dataset) generate_plots(y_real, y_pred, output, key, experiment_root);
def extract_cube(experiment_path, best_epoch, config): # load best model model_file = os.path.join(experiment_path, 'epochs', 'epoch{}.pkl'.format(best_epoch)) print 'loading ' + model_file model = serial.load(model_file) # print model; # additional dataset params config.start_sample = 11200 config.stop_sample = 12800 config.name = 'test' # load dataset dataset, dataset_yaml = load_yaml_file( os.path.join(os.path.dirname(__file__), '..', 'run', 'dataset_template.yaml'), params=config, ) with log_timing(log, 'processing dataset'): y_real, y_pred, output = process_dataset(model, dataset) print classification_report(y_real, y_pred) print confusion_matrix(y_real, y_pred) misclass = np.not_equal(y_real, y_pred).astype(int) print misclass.mean() print '----- sequence aggregration -----' s_real, s_pred, s_predf, s_predp = aggregate_classification( dataset.sequence_partitions, y_real, y_pred, output) print classification_report(s_real, s_predf) print confusion_matrix(s_real, s_predf) print(s_real != s_predf).mean() print '----- channel aggregration -----' t_real, t_pred, t_predf, t_predp = aggregate_classification( dataset.trial_partitions, y_real, y_pred, output) print classification_report(t_real, t_predf) print confusion_matrix(t_real, t_predf) print(t_real != t_predf).mean() cube = DataCube() cube.add(dataset.metadata, misclass) for cat, entry in cube.store.items(): print cat for key, values in cube.store[cat].items(): print '{:>30} : {:.3f}'.format(key, np.mean(values)) print np.mean(cube.get_entries()) header = ' | ' for c in xrange(18): header += ' {:2} '.format(c) header += ' avg ' print header for r in xrange(48): line = '{:>3} | '.format(r) for c in xrange(18): line += ' ' + cube.get_entries_mean_str(channels=[c], stimuli=[r]) line += ' ' + cube.get_entries_mean_str(stimuli=[r]) # average over all channels print line print line = '{:>3} | '.format('avg') for c in xrange(18): line += ' ' + cube.get_entries_mean_str(channels=[c]) # average over all stimuli line += ' ' + cube.get_entries_mean_str() # average over all stimuli and channels print line return cube
def extract_cube(experiment_path, best_epoch, config): # load best model model_file = os.path.join(experiment_path, "epochs", "epoch{}.pkl".format(best_epoch)) print "loading " + model_file model = serial.load(model_file) # print model; # additional dataset params config.start_sample = 11200 config.stop_sample = 12800 config.name = "test" # load dataset dataset, dataset_yaml = load_yaml_file( os.path.join(os.path.dirname(__file__), "..", "run", "dataset_template.yaml"), params=config ) with log_timing(log, "processing dataset"): y_real, y_pred, output = process_dataset(model, dataset) print classification_report(y_real, y_pred) print confusion_matrix(y_real, y_pred) misclass = np.not_equal(y_real, y_pred).astype(int) print misclass.mean() print "----- sequence aggregration -----" s_real, s_pred, s_predf, s_predp = aggregate_classification(dataset.sequence_partitions, y_real, y_pred, output) print classification_report(s_real, s_predf) print confusion_matrix(s_real, s_predf) print (s_real != s_predf).mean() print "----- channel aggregration -----" t_real, t_pred, t_predf, t_predp = aggregate_classification(dataset.trial_partitions, y_real, y_pred, output) print classification_report(t_real, t_predf) print confusion_matrix(t_real, t_predf) print (t_real != t_predf).mean() cube = DataCube() cube.add(dataset.metadata, misclass) for cat, entry in cube.store.items(): print cat for key, values in cube.store[cat].items(): print "{:>30} : {:.3f}".format(key, np.mean(values)) print np.mean(cube.get_entries()) header = " | " for c in xrange(18): header += " {:2} ".format(c) header += " avg " print header for r in xrange(48): line = "{:>3} | ".format(r) for c in xrange(18): line += " " + cube.get_entries_mean_str(channels=[c], stimuli=[r]) line += " " + cube.get_entries_mean_str(stimuli=[r]) # average over all channels print line print line = "{:>3} | ".format("avg") for c in xrange(18): line += " " + cube.get_entries_mean_str(channels=[c]) # average over all stimuli line += " " + cube.get_entries_mean_str() # average over all stimuli and channels print line return cube
def on_monitor(self, model, dataset, algorithm): # print 'self.dataset: {}\t {} '.format(self.dataset.name, self.dataset); # print self.dataset.X[0,0:5]; y_real, y_pred, output = process_dataset(model, self.dataset, data_specs=self.data_specs, output_fn=self.output_fn) if self.header is not None: print self.header; # Compute confusion matrix # print classification_report(y_real, y_pred); conf_matrix = confusion_matrix(y_real, y_pred); # if self.dataset.name == 'test': # print conf_matrix; # log values in monitoring channels channels = model.monitor.channels; if hasattr(self.dataset, 'get_class_labels'): class_labels = self.dataset.get_class_labels(); else: class_labels = ['0', '1']; # FIXME: more flexible fallback required # p, r, f1, s = precision_recall_fscore_support(y_real, y_pred, average=None); p, r, f1 = precision_recall_fscore_support(y_real, y_pred, average=None)[0:3]; mean_f1 = np.mean(f1); misclass = (y_real != y_pred).mean(); report = [['frames', mean_f1, misclass]]; channels[self.channel_prefix+'_f1_mean'].val_record[-1] = mean_f1; if self.class_prf1_channels: for i, class_label in enumerate(class_labels): channels[self.channel_prefix+'_precision_'+str(class_label)].val_record[-1] = p[i]; channels[self.channel_prefix+'_recall_'+str(class_label)].val_record[-1] = r[i]; channels[self.channel_prefix+'_f1_'+str(class_label)].val_record[-1] = f1[i]; if self.confusion_channels: # add channels for confusion matrix for i, c1 in enumerate(class_labels): for j, c2 in enumerate(class_labels): channels[self.channel_prefix+'_confusion_'+c1+'_as_'+c2].val_record[-1] = conf_matrix[i][j]; if self.dataset.name == 'test': print confusion_matrix(y_real, y_pred); if hasattr(self.dataset, 'sequence_partitions'): # print 'sequence-aggregated performance'; s_real, s_pred, s_predf, s_predp = aggregate_classification( self.dataset.sequence_partitions, y_real, y_pred, output); # NOTE: uses weighted version for printout # both, weighted and un-weighted are logged in the monitor for plotting # p, r, f1, s = precision_recall_fscore_support(s_real, s_pred, average=None); p, r, f1 = precision_recall_fscore_support(s_real, s_pred, average=None)[0:3]; s_mean_f1 = np.mean(f1); # p, r, f1, s = precision_recall_fscore_support(s_real, s_predf, average=None); p, r, f1 = precision_recall_fscore_support(s_real, s_predf, average=None)[0:3]; ws_mean_f1 = np.mean(f1); # p, r, f1, s = precision_recall_fscore_support(s_real, s_predp, average=None); p, r, f1 = precision_recall_fscore_support(s_real, s_predp, average=None)[0:3]; ps_mean_f1 = np.mean(f1); # print classification_report(s_real, s_predf); # print confusion_matrix(s_real, s_predf); s_misclass = (s_real != s_pred).mean(); ws_misclass = (s_real != s_predf).mean(); ps_misclass = (s_real != s_predp).mean(); report.append(['sequences', s_mean_f1, s_misclass]); report.append(['w. sequences', ws_mean_f1, ws_misclass]); report.append(['p. sequences', ps_mean_f1, ps_misclass]); # print 'seq misclass {:.4f}'.format(s_misclass); # print 'weighted seq misclass {:.4f}'.format(ws_misclass); channels[self.channel_prefix+'_seq_misclass_rate'].val_record[-1] = s_misclass; channels[self.channel_prefix+'_wseq_misclass_rate'].val_record[-1] = ws_misclass; channels[self.channel_prefix+'_pseq_misclass_rate'].val_record[-1] = ps_misclass; channels[self.channel_prefix+'_seq_mean_f1'].val_record[-1] = s_mean_f1; channels[self.channel_prefix+'_wseq_mean_f1'].val_record[-1] = ws_mean_f1; channels[self.channel_prefix+'_pseq_mean_f1'].val_record[-1] = ps_mean_f1; if hasattr(self.dataset, 'trial_partitions'): # print 'trial-aggregated performance'; t_real, t_pred, t_predf, t_predp = aggregate_classification( self.dataset.trial_partitions, y_real, y_pred, output); # NOTE: uses un-weighted version # both, weighted and un-weighted are logged in the monitor for plotting # p, r, f1, s = precision_recall_fscore_support(t_real, t_pred, average=None); p, r, f1 = precision_recall_fscore_support(t_real, t_pred, average=None)[0:3]; t_mean_f1 = np.mean(f1); # p, r, f1, s = precision_recall_fscore_support(t_real, t_predf, average=None); p, r, f1 = precision_recall_fscore_support(t_real, t_predf, average=None)[0:3]; wt_mean_f1 = np.mean(f1); # p, r, f1, s = precision_recall_fscore_support(t_real, t_predp, average=None); p, r, f1 = precision_recall_fscore_support(t_real, t_predp, average=None)[0:3]; pt_mean_f1 = np.mean(f1); # print classification_report(t_real, t_pred); # if self.dataset.name == 'test': # print confusion_matrix(t_real, t_predp); t_misclass = (t_real != t_pred).mean(); wt_misclass = (t_real != t_predf).mean(); pt_misclass = (t_real != t_predp).mean(); report.append(['trials', t_mean_f1, t_misclass]); report.append(['w. trials', wt_mean_f1, wt_misclass]); report.append(['p. trials', pt_mean_f1, pt_misclass]); # print 'trial misclass {:.4f}'.format(t_misclass); # print 'weighted trial misclass {:.4f}'.format(wt_misclass); channels[self.channel_prefix+'_trial_misclass_rate'].val_record[-1] = t_misclass; channels[self.channel_prefix+'_wtrial_misclass_rate'].val_record[-1] = wt_misclass; channels[self.channel_prefix+'_ptrial_misclass_rate'].val_record[-1] = pt_misclass; channels[self.channel_prefix+'_trial_mean_f1'].val_record[-1] = t_mean_f1; channels[self.channel_prefix+'_wtrial_mean_f1'].val_record[-1] = wt_mean_f1; channels[self.channel_prefix+'_ptrial_mean_f1'].val_record[-1] = pt_mean_f1; for label, f1, misclass in report: print '{:>15}: f1 = {:.3f} mc = {:.3f}'.format(label, f1, misclass);