def evaluate(experiment): datasets = collect_results(experiment) evaluations = [ ('Accuracy', lambda x: accuracy(x['confusion'][0])), ('Precision', lambda x: precision(x['confusion'][0])), ('Recall', lambda x: recall(x['confusion'][0])), ('F1', lambda x: f1_score(x['confusion'][0])), ('Time', lambda x: x['time'][0]), ('ClassParams', lambda x: x['params'][0])] # summary = {} # for name, eval_func in evaluations: # summary[name] = get_mean_and_error(datasets, eval_func) summary = [(name, get_mean_and_error(datasets, eval_func)) for name, eval_func in evaluations] suite = PyExperimentSuite() params = suite.get_params(experiment) row = [(x,params.get(x,'n/a')) for x in ['dataset','classifier','vectors','testset']] for name, (value, error) in summary: row.append( (name,value) ) row.append( (name + " error", error) ) #print row return row
def evaluate_dimensions(path): suite = PyExperimentSuite() rows = [] for experiment in os.listdir(path): try: joined = os.path.join(path,experiment) if os.path.isdir(joined): params = suite.get_params(joined) if params['classifier'] != 'cone': continue datasets = collect_results(joined) dimensions = params['dimensions'] values = {d:[] for d in dimensions} for data in datasets: for dimension_row in data['info'][0]: d = dimension_row[0]['dimensions'] values[d] += dimension_row[2] print values means = [(d, get_mean_and_error(values[d], lambda x:x)) for d in dimensions] rows = [[('dimensions', x[0]), ('accuracy', x[1][0]), ('error', x[1][1])] for x in means] write_summary(rows, 'analysis/' + params['dataset'] + '_dims.csv') except MissingDataException: continue
def loadExperiment(self, experiment): suite = Suite() suite.parse_opt() suite.parse_cfg() experiment_dir = experiment.split('/')[1] params = suite.items_to_params(suite.cfgparser.items(experiment_dir)) self.params = params predictions = suite.get_history(experiment, 0, 'predictions') truth = suite.get_history(experiment, 0, 'truth') self.iteration = suite.get_history(experiment, 0, 'iteration') self.train = suite.get_history(experiment, 0, 'train') self.truth = np.array(truth, dtype=np.float) if params['output_encoding'] == 'likelihood': from nupic.encoders.scalar import ScalarEncoder as NupicScalarEncoder self.outputEncoder = NupicScalarEncoder(w=1, minval=0, maxval=40000, n=22, forced=True) predictions_np = np.zeros((len(predictions), self.outputEncoder.n)) for i in xrange(len(predictions)): if predictions[i] is not None: predictions_np[i, :] = np.array(predictions[i]) self.predictions = predictions_np else: self.predictions = np.array(predictions, dtype=np.float)
def evaluate_dimensions(path): suite = PyExperimentSuite() rows = [] for experiment in os.listdir(path): try: joined = os.path.join(path, experiment) if os.path.isdir(joined): params = suite.get_params(joined) if params['classifier'] != 'cone': continue datasets = collect_results(joined) dimensions = params['dimensions'] values = {d: [] for d in dimensions} for data in datasets: for dimension_row in data['info'][0]: d = dimension_row[0]['dimensions'] values[d] += dimension_row[2] print values means = [(d, get_mean_and_error(values[d], lambda x: x)) for d in dimensions] rows = [[('dimensions', x[0]), ('accuracy', x[1][0]), ('error', x[1][1])] for x in means] write_summary(rows, 'analysis/' + params['dataset'] + '_dims.csv') except MissingDataException: print "Warning: ignoring due to missing data" continue
def evaluate(experiment): datasets = collect_results(experiment) evaluations = [('Accuracy', lambda x: accuracy(x['confusion'][0])), ('Precision', lambda x: precision(x['confusion'][0])), ('Recall', lambda x: recall(x['confusion'][0])), ('F1', lambda x: f1_score(x['confusion'][0])), ('Time', lambda x: x['time'][0]), ('ClassParams', lambda x: x['params'][0])] # summary = {} # for name, eval_func in evaluations: # summary[name] = get_mean_and_error(datasets, eval_func) summary = [(name, get_mean_and_error(datasets, eval_func)) for name, eval_func in evaluations] suite = PyExperimentSuite() params = suite.get_params(experiment) row = [(x, params.get(x, 'n/a')) for x in ['dataset', 'classifier', 'vectors', 'testset']] for name, (value, error) in summary: row.append((name, value)) row.append((name + " error", error)) #print row return row
def collect_results(experiment): suite = PyExperimentSuite() params = suite.get_params(experiment) reps = params['repetitions'] datasets = [] for rep in range(reps): results = suite.get_history(experiment, rep, 'all') if len(results) == 0: raise MissingDataException datasets.append(results) return datasets
def evaluate_all(path): suite = PyExperimentSuite() rows = [] for experiment in os.listdir(path): try: joined = os.path.join(path,experiment) if os.path.isdir(joined): params = suite.get_params(joined) row = evaluate(joined) rows.append(row) except MissingDataException: continue return rows
def evaluate_all(path): suite = PyExperimentSuite() rows = [] for experiment in os.listdir(path): try: joined = os.path.join(path,experiment) if os.path.isdir(joined): print joined params = suite.get_params(joined) row = evaluate(joined) rows.append(row) else: #print "Warning: not directory",joined continue except MissingDataException: print "Warning: MissingDataException" continue return rows
def evaluate_all(path): suite = PyExperimentSuite() rows = [] for experiment in os.listdir(path): try: joined = os.path.join(path, experiment) if os.path.isdir(joined): print joined params = suite.get_params(joined) row = evaluate(joined) rows.append(row) else: #print "Warning: not directory",joined continue except MissingDataException: print "Warning: MissingDataException" continue return rows
if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('experiments', metavar='/path/to/experiment /path/...', nargs='+', type=str) parser.add_argument('-w', '--window', type=int, default=100) parser.add_argument('-n', '--num', type=int, default=None) parser.add_argument('-t', '--training-hide', type=int, nargs='+') parser.add_argument('-g', '--graph-labels', type=str, nargs='+') parser.add_argument('-s', '--size-of-line', type=float, nargs='+') parser.add_argument('-l', '--legend-position', type=int, default=4) parser.add_argument('-f', '--full', action='store_true') parser.add_argument('-o', '--output', type=str, default=None) suite = PyExperimentSuite() args = parser.parse_args() from pylab import rcParams rcParams.update({'figure.autolayout': True}) rcParams.update({'figure.facecolor': 'white'}) rcParams.update({'ytick.labelsize': 8}) rcParams.update({'figure.figsize': (12, 6)}) rcParams.update({'pdf.fonttype': 42}) experiments = args.experiments for i, experiment in enumerate(experiments): iteration = suite.get_history(experiment, 0, 'iteration') predictions = suite.get_history(experiment, 0, 'predictions')
# Constants values used across all experiments STRIDE = 1 PADDING = 0 KERNEL_SIZE = 5 def computeMaxPool(input_width): """ Compute CNN max pool width """ wout = math.floor((input_width + 2 * PADDING - KERNEL_SIZE) / STRIDE + 1) return int(math.floor(wout / 2.0)) if __name__ == '__main__': suite = PyExperimentSuite() suite.parse_opt() suite.parse_cfg() experiments = suite.options.experiments or suite.cfgparser.sections() paramsTable = [[ "Network", "L1 F", "L1 Sparsity", "L2 F", "L2 Sparsity", "L3 N", "L3 Sparsity", "Wt Sparsity" ]] for name in experiments: # Iterate over experiments, skipping over errors. try: exps = suite.get_exps(suite.get_exp(name)[0]) except: print("Couldn't parse experiment:", name)
def __init__(self): PyExperimentSuite.__init__(self) self.dataset = None