def run_network_replacerule(model_dir, rule, replace_rule, rule_strength): """Run the network but with replaced rule input weights. Args: model_dir: model directory rule: the rule to test on replace_rule: a list of rule input units to use rule_strength: the relative strength of each replace rule unit """ model = Model(model_dir) hp = model.hp with tf.Session() as sess: model.restore() # Get performance batch_size_test = 1000 n_rep = 20 batch_size_test_rep = int(batch_size_test / n_rep) perf_rep = list() for i_rep in range(n_rep): trial = generate_trials(rule, hp, 'random', batch_size=batch_size_test_rep, replace_rule=replace_rule, rule_strength=rule_strength) feed_dict = tools.gen_feed_dict(model, trial, hp) y_hat_test = sess.run(model.y_hat, feed_dict=feed_dict) perf_rep.append(np.mean(get_perf(y_hat_test, trial.y_loc))) return np.mean(perf_rep), rule_strength
def do_eval_test(sess, model, rule): """Do evaluation. Args: sess: tensorflow session model: Model class instance rule_train: string or list of strings, the rules being trained """ hp = model.hp trial = generate_trials(rule, hp, 'test') feed_dict = tools.gen_feed_dict(model, trial, hp) c_lsq, c_reg, y_hat_test = sess.run( [model.cost_lsq, model.cost_reg, model.y_hat], feed_dict=feed_dict) # Cost is first summed over time, # and averaged across batch and units # We did the averaging over time through c_mask perf_test = np.mean(get_perf(y_hat_test, trial.y_loc)) sys.stdout.flush() return c_lsq, c_reg, perf_test
def lesions(self): labels = self.labels from network import get_perf from task import generate_trials # The first will be the intact network lesion_units_list = [None] for il, l in enumerate(self.unique_labels): ind_l = np.where(labels == l)[0] # In original indices lesion_units_list += [self.ind_active[ind_l]] perfs_store_list = list() perfs_changes = list() cost_store_list = list() cost_changes = list() for i, lesion_units in enumerate(lesion_units_list): model = Model(self.model_dir) hp = model.hp with tf.Session() as sess: model.restore() model.lesion_units(sess, lesion_units) perfs_store = list() cost_store = list() for rule in self.rules: n_rep = 16 batch_size_test = 256 batch_size_test_rep = int(batch_size_test / n_rep) clsq_tmp = list() perf_tmp = list() for i_rep in range(n_rep): trial = generate_trials(rule, hp, 'random', batch_size=batch_size_test_rep) feed_dict = tools.gen_feed_dict(model, trial, hp) y_hat_test, c_lsq = sess.run( [model.y_hat, model.cost_lsq], feed_dict=feed_dict) # Cost is first summed over time, and averaged across batch and units # We did the averaging over time through c_mask # IMPORTANT CHANGES: take overall mean perf_test = np.mean(get_perf(y_hat_test, trial.y_loc)) clsq_tmp.append(c_lsq) perf_tmp.append(perf_test) perfs_store.append(np.mean(perf_tmp)) cost_store.append(np.mean(clsq_tmp)) perfs_store = np.array(perfs_store) cost_store = np.array(cost_store) perfs_store_list.append(perfs_store) cost_store_list.append(cost_store) if i > 0: perfs_changes.append(perfs_store - perfs_store_list[0]) cost_changes.append(cost_store - cost_store_list[0]) perfs_changes = np.array(perfs_changes) cost_changes = np.array(cost_changes) return perfs_changes, cost_changes
def do_eval(sess, model, log, rule_train): """Do evaluation. Args: sess: tensorflow session model: Model class instance log: dictionary that stores the log rule_train: string or list of strings, the rules being trained """ hp = model.hp if not hasattr(rule_train, '__iter__'): rule_name_print = rule_train else: rule_name_print = ' & '.join(rule_train) print('Trial {:7d}'.format(log['trials'][-1]) + ' | Time {:0.2f} s'.format(log['times'][-1]) + ' | Now training ' + rule_name_print) for rule_test in hp['rules']: n_rep = 16 batch_size_test_rep = int(hp['batch_size_test'] / n_rep) clsq_tmp = list() creg_tmp = list() perf_tmp = list() for i_rep in range(n_rep): trial = generate_trials(rule_test, hp, 'random', batch_size=batch_size_test_rep) feed_dict = tools.gen_feed_dict(model, trial, hp) c_lsq, c_reg, y_hat_test = sess.run( [model.cost_lsq, model.cost_reg, model.y_hat], feed_dict=feed_dict) # Cost is first summed over time, # and averaged across batch and units # We did the averaging over time through c_mask perf_test = np.mean(get_perf(y_hat_test, trial.y_loc)) clsq_tmp.append(c_lsq) creg_tmp.append(c_reg) perf_tmp.append(perf_test) log['cost_' + rule_test].append(np.mean(clsq_tmp, dtype=np.float64)) log['creg_' + rule_test].append(np.mean(creg_tmp, dtype=np.float64)) log['perf_' + rule_test].append(np.mean(perf_tmp, dtype=np.float64)) print('{:15s}'.format(rule_test) + '| cost {:0.6f}'.format(np.mean(clsq_tmp)) + '| c_reg {:0.6f}'.format(np.mean(creg_tmp)) + ' | perf {:0.2f}'.format(np.mean(perf_tmp))) sys.stdout.flush() # TODO: This needs to be fixed since now rules are strings if hasattr(rule_train, '__iter__'): rule_tmp = rule_train else: rule_tmp = [rule_train] perf_tests_mean = np.mean([log['perf_' + r][-1] for r in rule_tmp]) log['perf_avg'].append(perf_tests_mean) perf_tests_min = np.min([log['perf_' + r][-1] for r in rule_tmp]) log['perf_min'].append(perf_tests_min) # Saving the model model.save() tools.save_log(log) return log