def evaluate(experiment): datasets = collect_results(experiment) evaluations = [ ('Accuracy', lambda x: accuracy(x['confusion'][0])), ('Precision', lambda x: precision(x['confusion'][0])), ('Recall', lambda x: recall(x['confusion'][0])), ('F1', lambda x: f1_score(x['confusion'][0])), ('Time', lambda x: x['time'][0]), ('ClassParams', lambda x: x['params'][0])] # summary = {} # for name, eval_func in evaluations: # summary[name] = get_mean_and_error(datasets, eval_func) summary = [(name, get_mean_and_error(datasets, eval_func)) for name, eval_func in evaluations] suite = PyExperimentSuite() params = suite.get_params(experiment) row = [(x,params.get(x,'n/a')) for x in ['dataset','classifier','vectors','testset']] for name, (value, error) in summary: row.append( (name,value) ) row.append( (name + " error", error) ) #print row return row
def evaluate_dimensions(path): suite = PyExperimentSuite() rows = [] for experiment in os.listdir(path): try: joined = os.path.join(path,experiment) if os.path.isdir(joined): params = suite.get_params(joined) if params['classifier'] != 'cone': continue datasets = collect_results(joined) dimensions = params['dimensions'] values = {d:[] for d in dimensions} for data in datasets: for dimension_row in data['info'][0]: d = dimension_row[0]['dimensions'] values[d] += dimension_row[2] print values means = [(d, get_mean_and_error(values[d], lambda x:x)) for d in dimensions] rows = [[('dimensions', x[0]), ('accuracy', x[1][0]), ('error', x[1][1])] for x in means] write_summary(rows, 'analysis/' + params['dataset'] + '_dims.csv') except MissingDataException: continue
def evaluate_dimensions(path): suite = PyExperimentSuite() rows = [] for experiment in os.listdir(path): try: joined = os.path.join(path, experiment) if os.path.isdir(joined): params = suite.get_params(joined) if params['classifier'] != 'cone': continue datasets = collect_results(joined) dimensions = params['dimensions'] values = {d: [] for d in dimensions} for data in datasets: for dimension_row in data['info'][0]: d = dimension_row[0]['dimensions'] values[d] += dimension_row[2] print values means = [(d, get_mean_and_error(values[d], lambda x: x)) for d in dimensions] rows = [[('dimensions', x[0]), ('accuracy', x[1][0]), ('error', x[1][1])] for x in means] write_summary(rows, 'analysis/' + params['dataset'] + '_dims.csv') except MissingDataException: print "Warning: ignoring due to missing data" continue
def evaluate(experiment): datasets = collect_results(experiment) evaluations = [('Accuracy', lambda x: accuracy(x['confusion'][0])), ('Precision', lambda x: precision(x['confusion'][0])), ('Recall', lambda x: recall(x['confusion'][0])), ('F1', lambda x: f1_score(x['confusion'][0])), ('Time', lambda x: x['time'][0]), ('ClassParams', lambda x: x['params'][0])] # summary = {} # for name, eval_func in evaluations: # summary[name] = get_mean_and_error(datasets, eval_func) summary = [(name, get_mean_and_error(datasets, eval_func)) for name, eval_func in evaluations] suite = PyExperimentSuite() params = suite.get_params(experiment) row = [(x, params.get(x, 'n/a')) for x in ['dataset', 'classifier', 'vectors', 'testset']] for name, (value, error) in summary: row.append((name, value)) row.append((name + " error", error)) #print row return row
def collect_results(experiment): suite = PyExperimentSuite() params = suite.get_params(experiment) reps = params['repetitions'] datasets = [] for rep in range(reps): results = suite.get_history(experiment, rep, 'all') if len(results) == 0: raise MissingDataException datasets.append(results) return datasets
def collect_results(experiment): suite = PyExperimentSuite() params = suite.get_params(experiment) reps = params['repetitions'] datasets = [] for rep in range(reps): results = suite.get_history(experiment, rep, 'all') if len(results) == 0: raise MissingDataException datasets.append(results) return datasets
def evaluate_all(path): suite = PyExperimentSuite() rows = [] for experiment in os.listdir(path): try: joined = os.path.join(path,experiment) if os.path.isdir(joined): params = suite.get_params(joined) row = evaluate(joined) rows.append(row) except MissingDataException: continue return rows
def evaluate_all(path): suite = PyExperimentSuite() rows = [] for experiment in os.listdir(path): try: joined = os.path.join(path,experiment) if os.path.isdir(joined): print joined params = suite.get_params(joined) row = evaluate(joined) rows.append(row) else: #print "Warning: not directory",joined continue except MissingDataException: print "Warning: MissingDataException" continue return rows
def evaluate_all(path): suite = PyExperimentSuite() rows = [] for experiment in os.listdir(path): try: joined = os.path.join(path, experiment) if os.path.isdir(joined): print joined params = suite.get_params(joined) row = evaluate(joined) rows.append(row) else: #print "Warning: not directory",joined continue except MissingDataException: print "Warning: MissingDataException" continue return rows
"Network", "L1 F", "L1 Sparsity", "L2 F", "L2 Sparsity", "L3 N", "L3 Sparsity", "Wt Sparsity" ]] for name in experiments: # Iterate over experiments, skipping over errors. try: exps = suite.get_exps(suite.get_exp(name)[0]) except: print("Couldn't parse experiment:", name) continue for exp in exps: if not os.path.exists(exp): continue params = suite.get_params(exp=exp) l3_n = params["n"] l3_k = params["k"] l3_sp = "{0:.1f}%".format(100 * float(l3_k) / l3_n) wt_sp = "{0}%".format(100 * float(params["weight_sparsity"])) c1_k = params["c1_k"] if isinstance(c1_k, basestring): c1_k = map(int, c1_k.split("_")) l1_k = c1_k[0] l2_k = c1_k[1] else: l1_k = int(c1_k) l2_k = None