def merge_sim_files(filter, input_folder): from os import listdir from os.path import join sims_dirs = [ f for f in listdir(input_folder) if isdir(join(input_folder, f)) ] total_sims = 0 all_train = [] for d in sims_dirs: DLogger.logger().debug(d) full_path = input_folder + d + '/' conf = pd.read_csv(full_path + 'config.csv') if filter(conf): train = pd.read_csv(full_path + 'train.csv') train['group'] = conf['group'][0] train['prop0'] = conf['prop0'][0] train['prop1'] = conf['prop1'][0] train['id'] = 'ID_' + conf['group'][0] + str(conf['N'][0]) train['sim_ID'] = d all_train.append(train) total_sims += 1 DLogger.logger().debug("total sims: " + str(total_sims)) # concatinating all training inputs all_train = pd.concat(all_train, ignore_index=True) # finding wheher the best action was taken according to the planned probabilities best_action = 1 * (all_train['prop1'] > all_train['prop0']) all_train['best_action'] = 1 * (best_action == all_train['choices']) all_train['key'] = all_train['choices'] return all_train
def get_instance_without_pser(n_actions, alpha, gamma): options = {'persv': False} DLogger.logger().debug("options: " + str(Helper.dicstr(options))) return QL(n_actions, [ logit(tf.constant(alpha, dtype=Const.FLOAT)), tf.log(tf.constant(gamma, dtype=Const.FLOAT)) ], options)
def GQL_classify_subjects(): tf.reset_default_graph() data = DataReader.read_BD() ids = data['id'].unique().tolist() dftr = pd.DataFrame({'id': ids, 'train': 'train'}) train, test = DataProcess.train_test_between_subject( data, dftr, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]) model_iter = 'model-final' df = pd.DataFrame(columns=('model', 'id', 'loss')) config = tf.ConfigProto(device_count={'GPU': 0}) subj_paths = finding_CV(Paths.rest_path + 'archive/beh/gql-ml-pred-diag/') worker = GQL.get_instance(2, 2, {}) worker.set_params(OptML.get_variables(worker.get_params())) for k, tr in train.iteritems(): for g, p in subj_paths[k].iteritems(): DLogger.logger().debug('subject ' + k + ' group ' + g + ' path ' + p) model_path = p + model_iter + '/' with tf.Session(config=config) as sess: load_model(sess, model_path) total_loss = 0 for v in tr: ell, _, _ = worker.simulate(sess, v['reward'], v['action']) total_loss += -ell df.loc[len(df)] = [g, k, total_loss] df.to_csv(Paths.local_path + 'BD/gql_diag.csv')
def simulate_model(input_folder, output_folder, data, n_cells): dftr = pd.DataFrame({'id': data['id'].unique().tolist(), 'train': 'train'}) train, _ = DataProcess.train_test_between_subject( data, dftr, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]) tf.reset_default_graph() worker = LSTMBeh(2, 0, n_cells) DLogger.logger().debug('started simulations') Simulator.simulate_worker(worker, input_folder, train, output_folder)
def evaluate_CV(worker_gen, base_input_folder, base_output_folder, cells, n_actions, n_states, data, folds, model_iters, trials): df = pd.DataFrame() for group in folds.keys(): for n_cells in cells: for fold in folds[group]: for model_iter in model_iters: input_folder = base_input_folder + str( n_cells) + 'cells/' + group + '/' + fold + '/' output_folder = base_output_folder + str( n_cells ) + 'cells/' + group + '/' + fold + '/' + model_iter + '/' tr_tst = pd.read_csv(input_folder + 'train_test.csv') if not ('id' in tr_tst): DLogger.logger().debug( 'id not found in test train file. Using ID instead.' ) tr_tst['id'] = tr_tst['ID'] tst_ids = tr_tst.loc[tr_tst.train == 'test']['id'] dftr = pd.DataFrame({'id': tst_ids, 'train': 'train'}) train, _ = DataProcess.train_test_between_subject( data, dftr, trials) tf.reset_default_graph() worker = worker_gen(n_actions, n_states, n_cells) DLogger.logger().debug(input_folder + model_iter) Simulator.simulate_worker( worker, input_folder + model_iter + '/', train, output_folder) train = pd.read_csv(output_folder + 'train.csv') policies = pd.read_csv(output_folder + 'policies-.csv') acc, nlp, total_nlp = Assessor.evaluate_fit_multi( policies, train) df = df.append( pd.DataFrame({ 'acc': [acc], 'nlp': [nlp], 'total nlp': [total_nlp], 'group': group, 'cell': n_cells, 'fold': fold, 'model_iter': model_iter })) df.to_csv(base_output_folder + 'accu.csv')
def dPolicy_dReward(self, policy_trials): """ Calculates the gradient of policies wrt to rewards :param policy_trials: the array/list containing index of the action for which the gradinet of the cell output is calculated. :return: an array of size len(trial_list) x nTrials for example output [t1, t2] is the gradient of policy at trial t1 wrt to reward received at trial t2 """ DLogger.logger().debug("adding gradients for each cell-action pair...") grads = [] for a in policy_trials: grads.append(tf.gradients(self.policy[0, a, 0], self.prev_rewards)[0][0, :, 0]) grads = tf.convert_to_tensor(grads) DLogger.logger().debug("finished adding gradients.") return grads
def dPolicy_dH(self, policy_trials): """ Calculates the gradient of policies at policy_trials wrt to rnn_outs (for all trials) :param policy_trials: the array/list containing index of the action for which the gradinet of the cell output is calculated. :return: an array of size nCells x len(policy_trials) x nTrials """ DLogger.logger().debug("adding gradients for each cell-action pair...") grads = [] for a in policy_trials: grads.append(tf.gradients(self.policy[0, a, 0], self.state_in_pret)[0][0]) grads = tf.convert_to_tensor(grads) grads = tf.transpose(grads, [2, 0, 1]) DLogger.logger().debug("finished adding gradients.") return grads
def finding_CV(base_input_folder): from os import listdir from os.path import join group_address = { 'Healthy': base_input_folder + 'Healthy/fold0/', 'Bipolar': base_input_folder + 'Bipolar/fold0/', 'Depression': base_input_folder + 'Depression/fold0/' } subj_address = {} for group in ['Healthy', 'Depression', 'Bipolar']: input_folder = base_input_folder + group + '/' sims_dirs = [ f for f in listdir(input_folder) if isdir(join(input_folder, f)) ] for d in sims_dirs: DLogger.logger().debug(d) full_path = input_folder + d + '/' tr_tst = pd.read_csv(full_path + 'train_test.csv') if not ('id' in tr_tst): tr_tst['id'] = tr_tst['ID'] tst_ids = tr_tst.loc[tr_tst.train == 'test']['id'] if len(tst_ids) > 1: raise Exception('tests should contain one data-point.') if len(tst_ids) > 0: _id = tst_ids.iloc[0] subj_address[_id] = {} for group2 in ['Healthy', 'Depression', 'Bipolar']: if group == group2: subj_address[_id][group] = full_path else: subj_address[_id][group2] = group_address[group2] return subj_address
def __init__(self, a_size, s_size, n_cells): DLogger.logger().debug("model created with ncells: " + str(n_cells)) DLogger.logger().debug("number of actions: " + str(a_size)) DLogger.logger().debug("number of states: " + str(s_size)) # placeholders self.prev_rewards = tf.placeholder(shape=[None, None, 1], dtype=Const.FLOAT) # DIM: nBatches x (nChoices + 1) x 1 self.prev_actions = tf.placeholder(shape=[None, None], dtype=tf.int32) # DIM: nBatches x (nChoices + 1) self.timestep = tf.placeholder(shape=[None, None], dtype=tf.int32) # DIM: nBatches x nChoices self.prev_actions_onehot = tf.one_hot(self.prev_actions, a_size, dtype=Const.FLOAT, axis=-1) # DIM: nBatches x (nChoices + 1) x nActionTypes self.actions = tf.placeholder(shape=[None, None], dtype=tf.int32) # DIM: nBatches x nChoices self.actions_onehot = tf.one_hot(self.actions, a_size, dtype=Const.FLOAT, axis=-1) # DIM: nBatches x nChoices x nActionTypes self.n_batches = tf.shape(self.prev_rewards)[0] self.n_cells = n_cells if s_size != 0: self.prev_states = tf.placeholder(shape=[None, None, s_size], dtype=Const.FLOAT) # self.prev_states_onehot = tf.one_hot(self.prev_states, s_size, dtype=Const.FLOAT) rnn_in = tf.concat(values=[self.prev_rewards, self.prev_actions_onehot, self.prev_states], axis=2) # DIM: nBatches x (nChoices + 1 ) x ( 1 + nActionTypes + nStateTypes) else: rnn_in = tf.concat(values=[self.prev_rewards, self.prev_actions_onehot], axis=2) # DIM: nBatches x (nChoices + 1 ) x ( 1 + nActionTypes) self.state_init, \ self.state_in, \ self.rnn_out, \ self.state_out, \ self.state_track = self.rnn_cells(n_cells, rnn_in) # Output layers for policy and value estimations self.policy = slim.fully_connected(self.rnn_out, a_size, activation_fn=tf.nn.softmax, weights_initializer=normalized_columns_initializer(0.01), biases_initializer=None, scope='softmax') # DIM: nBatches x (nChoices + 1) x (nActionTypes) # self.value = slim.fully_connected(self.rnn_out, 1, # activation_fn=None, # weights_initializer=normalized_columns_initializer(1.0), # biases_initializer=None, # sope='softmax-v') self.beh_loss = self._get_beh_loss()
def RNN_classify_subjects(): data = DataReader.read_BD() ids = data['id'].unique().tolist() dftr = pd.DataFrame({'id': ids, 'train': 'train'}) train, test = DataProcess.train_test_between_subject( data, dftr, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]) n_cells = {'Healthy': 10, 'Depression': 10, 'Bipolar': 20} model_iter = 'model-final' df = pd.DataFrame(columns=('model', 'id', 'loss')) config = tf.ConfigProto(device_count={'GPU': 0}) subj_paths = finding_CV(Paths.rest_path + 'archive/beh/rnn-pred-diag/') for k, tr in train.iteritems(): for g, p in subj_paths[k].iteritems(): tf.reset_default_graph() worker = LSTMBeh(2, 0, n_cells[g]) saver = tf.train.Saver(max_to_keep=5) DLogger.logger().debug('subject ' + k + ' group ' + g + ' path ' + p) model_path = p + model_iter + '/' ckpt = tf.train.get_checkpoint_state(model_path) tf.train.import_meta_graph(model_path + 'model.cptk.meta') with tf.Session(config=config) as sess: saver.restore(sess, ckpt.model_checkpoint_path) total_loss = 0 for v in tr: policies, c_track, h_track, loss = worker.simulate( sess, v['reward'], v['action'], v['state']) total_loss += loss df.loc[len(df)] = [g, k, total_loss] df.to_csv(Paths.local_path + 'BD/rnn_diag.csv')
def extract_run_rew(data): ids = data['id'].unique().tolist() total_data = [] for id in ids: DLogger.logger().debug('processing subject ' + id) id_data = data.loc[data.id == id] sim_ids = id_data['sim_ID'].unique().tolist() for sim_id in sim_ids: sim_data = id_data.loc[id_data.sim_ID == sim_id] next_key = sim_data['key'][1:].tolist() + [None] sim_data = sim_data.assign(next_key=next_key) sim_data = sim_data.assign( same=sim_data['key'] == sim_data['next_key']) sim_data.ix[sim_data.index[-1], 'same'] = 0 last_key = None total_keys = 0 total_rews = 0 sim_data = sim_data.assign(prev_key=None) sim_data = sim_data.assign(prev_rewards=None) for i in range(sim_data.shape[0]): if last_key == sim_data.iloc[i]['key']: sim_data.ix[sim_data.index[i], 'prev_key'] = total_keys sim_data.ix[sim_data.index[i], 'prev_rewards'] = total_rews total_keys += 1 total_rews += sim_data.iloc[i]['reward'] else: total_keys = 1 total_rews = sim_data.iloc[i]['reward'] sim_data.ix[sim_data.index[i], 'prev_key'] = 0 sim_data.ix[sim_data.index[i], 'prev_rewards'] = 0 last_key = sim_data.iloc[i]['key'] total_data.append(sim_data) all_trials = pd.concat(total_data) return all_trials
def run_BD(i): data = DataReader.read_BD() ncells = configs[i]['cells'] group = configs[i]['g'] input_path = Paths.rest_path + 'archive/beh/rnn-opt-rand-init/' + 'run_' + \ str(configs[i]['s']) + '/' + str(ncells) + 'cells/' + group + '/model-final/' output_path = Paths.local_path + 'BD/rnn-opt-rand-init-evals/' + 'run_' + \ str(configs[i]['s']) + '/' + str(ncells) + 'cells/' + group + '/' gdata = data.loc[data.diag == group] ids = gdata['id'].unique().tolist() dftr = pd.DataFrame({'id': ids, 'train': 'train'}) tdftr = pd.DataFrame({'id': ids, 'train': 'test'}) train, test = DataProcess.train_test_between_subject( gdata, pd.concat((dftr, tdftr)), [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]) test = DataProcess.merge_data(test) tf.reset_default_graph() worker = LSTMBeh(2, 0, ncells) saver = tf.train.Saver(max_to_keep=None) with tf.Session() as sess: DLogger.logger().debug("loading model from: " + str(input_path)) ckpt = tf.train.get_checkpoint_state(input_path) tf.train.import_meta_graph(input_path + 'model.cptk.meta') saver.restore(sess, ckpt.model_checkpoint_path) for k, tr in test.iteritems(): for v in tr: _, _, _, ell = worker.simulate(sess, v['reward'], v['action'], v['state']) DLogger.logger().debug("input path: " + input_path) DLogger.logger().debug("output path: " + output_path) DLogger.logger().debug("total nlp: {} ".format(str(ell))) return pd.DataFrame({ 'total nlp': [ell], 'group': group, 'cell': ncells, 'fold': None, 'model_iter': 'model-final', 's': configs[i]['s'] })
def onpol_sim(worker, subj_list, output_prefix, model_path): params = OptML.get_variables(worker.get_params()) worker.set_params(params) config = tf.ConfigProto(device_count={'GPU': 0}) with tf.Session(config=config) as sess: DLogger.logger().debug("loading mode....") load_model(sess, model_path) DLogger.logger().debug("finished loading mode.") for s in range(len(subj_list)): DLogger.logger().debug("parameters {}".format(sess.run(params))) for c in subj_list[s]: DLogger.logger().debug("subject {} trial {}".format( c['id'], c['block'])) choices = c['choices'] output_path = output_prefix + 'sim_' + id_generator( size=7) + '/' if not os.path.exists(output_path): os.makedirs(output_path) c2 = c.copy() c2['option'] = {} c2['N'] = c['id'] pdc = pd.DataFrame(c2, index=[0]) pdc.to_csv(output_path + 'config.csv', index=False) _, _ = Simulator.simulate_env(sess, worker, output_path, choices, bandit_evn(c['prop0'], c['prop1'], init_state=None, init_action=-1, init_reward=0), greedy=False)
def simulate_ml(): _, _, by_group = get_BD_confs() for g in by_group.keys(): subj_list = by_group[g] tf.reset_default_graph() cur_group = 'gql10d-ml-opt/' + g model_path = Paths.rest_path + 'archive/beh/' + cur_group + '/model-final/' output_prefix = Paths.local_path + 'BD/sims/gql10d-ml/' worker = GQL.get_instance(2, 10, {}) DLogger.logger().debug("model path: " + model_path) DLogger.logger().debug("output prefix" + output_prefix) onpol_sim(worker, subj_list, output_prefix, model_path) for g in by_group.keys(): subj_list = by_group[g] tf.reset_default_graph() cur_group = 'qlp-ml-opt/' + g model_path = Paths.rest_path + 'archive/beh/' + cur_group + '/model-final/' output_prefix = Paths.local_path + 'BD/sims/qlp-ml/' worker = QL.get_instance_with_pser(2, 0.1, 0.1, 0.1) DLogger.logger().debug("model path: " + model_path) DLogger.logger().debug("output prefix" + output_prefix) onpol_sim(worker, subj_list, output_prefix, model_path) for g in by_group.keys(): subj_list = by_group[g] tf.reset_default_graph() cur_group = 'ql-ml-opt/' + g model_path = Paths.rest_path + 'archive/beh/' + cur_group + '/model-final/' output_prefix = Paths.local_path + 'BD/sims/ql-ml/' worker = QL.get_instance_without_pser(2, 0.1, 0.1) DLogger.logger().debug("model path: " + model_path) DLogger.logger().debug("output prefix" + output_prefix) onpol_sim(worker, subj_list, output_prefix, model_path) for g in by_group.keys(): subj_list = by_group[g] tf.reset_default_graph() cur_group = 'gql-ml-opt/' + g model_path = Paths.rest_path + 'archive/beh/' + cur_group + '/model-final/' output_prefix = Paths.local_path + 'BD/sims/gql-ml/' worker = GQL.get_instance(2, 2, {}) DLogger.logger().debug("model path: " + model_path) DLogger.logger().debug("output prefix" + output_prefix) onpol_sim(worker, subj_list, output_prefix, model_path)
def evaluate_CV(base_input_folder, base_output_folder, test_and_save, data, folds, model_iters, trials, random_tie=True): df = pd.DataFrame() saver = tf.train.Saver(max_to_keep=5) config = tf.ConfigProto(device_count={'GPU': 0}) with tf.Session(config=config) as sess: for group in sorted(folds.keys()): for fold in folds[group]: for model_iter in model_iters: input_folder = base_input_folder + '/' + group + '/' + fold + '/' if base_output_folder is not None: output_folder = base_output_folder + group + '/' + fold + '/' + model_iter + '/' else: output_folder = None DLogger.logger().debug( "input folder: {}".format(input_folder)) DLogger.logger().debug( "output folder: {}".format(output_folder)) tr_tst = pd.read_csv(input_folder + 'train_test.csv') if 'ID' in tr_tst: DLogger.logger().debug( "id column was not found. Replaced id column with ID." ) tr_tst['id'] = tr_tst['ID'] tst_ids = tr_tst.loc[tr_tst.train == 'test']['id'] dftr = pd.DataFrame({'id': tst_ids, 'train': 'train'}) train, _ = DataProcess.train_test_between_subject( data, dftr, trials) model_path = input_folder + model_iter + '/' ckpt = tf.train.get_checkpoint_state(model_path) # tf.train.import_meta_graph(model_path + 'model.cptk.meta') saver.restore(sess, ckpt.model_checkpoint_path) policies = test_and_save(sess, train, output_folder) if output_folder is not None: Export.export_train(train, output_folder, 'train.csv') train_merged = Export.merge_train(train) #add a dummy column at the beginning train_merged.insert(loc=0, column='tmp', value='') policies_merged = Export.merge_policies(policies) #add a dummy column at the beginning policies_merged.insert(loc=0, column='tmp', value='') # train_merged = pd.read_csv(output_folder + 'train.csv') # policies_merged = pd.read_csv(output_folder + 'policies.csv') acc, nlp, total_nlp = Assessor.evaluate_fit_multi( policies_merged, train_merged, pol_in_log=True, random_tie=random_tie) df = df.append( pd.DataFrame({ 'acc': [acc], 'nlp': [nlp], 'total nlp': [total_nlp], 'group': group, 'option': Helper.dicstr({}), 'fold': fold, 'model_iter': model_iter })) if base_output_folder is not None: df.to_csv(base_output_folder + 'accu.csv') return df
def __init__(self, a_size, s_size, n_cells): GRUBase.__init__(self, a_size, s_size, n_cells) DLogger.logger().debug("model created with ncells: " + str(n_cells)) DLogger.logger().debug("number of actions: " + str(a_size)) DLogger.logger().debug("number of states: " + str(s_size))