def create_dataset(self, sim_pos): """ Creates a GradientData object by executing all behavioral choices at simulated positions once per second :param sim_pos: Previously created simulation trajectory :return: GradientData object with all necessary training in- and outputs """ if sim_pos.shape[1] != 3: raise ValueError( "sim_pos has to be nx3 array with xpos, ypos and heading at each timepoint" ) history = GlobalDefs.frame_rate * GlobalDefs.hist_seconds start = history + 1 # start data creation with enough history present btypes = ["C", "S", "P", "L", "R"] # create vector that selects one position every second on average sel = np.random.rand(sim_pos.shape[0]) < (1 / GlobalDefs.frame_rate) sel[:start + 1] = False # initialize model inputs and outputs inputs = np.zeros((sel.sum(), 3, history), np.float32) outputs = np.zeros((sel.sum(), 5), np.float32) # loop over each position, simulating PRED_WINDOW into future to obtain real finish temperature curr_sel = 0 for step in range(start, sim_pos.shape[0]): if not sel[step]: continue # obtain inputs at given step inputs[curr_sel, 0, :] = self.temperature( sim_pos[step - history + 1:step + 1, 0], sim_pos[step - history + 1:step + 1, 1]) spd = np.sqrt( np.sum( np.diff(sim_pos[step - history:step + 1, 0:2], axis=0)**2, 1)) inputs[curr_sel, 1, :] = spd inputs[curr_sel, 2, :] = np.diff(sim_pos[step - history:step + 1, 2], axis=0) # select each possible behavior in turn starting from this step and simulate # the behavior into the future and obtain final temperature as output by # approximating PRED_WINDOW steps into the future moving in a straight line for i, b in enumerate(btypes): fpos = self.sim_forward(self.alen, sim_pos[step, :], b)[-1, :] fpos[0] += np.cos(fpos[2]) * PRED_WINDOW * self.mu_disp fpos[1] += np.sin(fpos[2]) * PRED_WINDOW * self.mu_disp outputs[curr_sel, i] = self.temperature(fpos[0], fpos[1]) curr_sel += 1 assert not np.any(np.sum(inputs, (1, 2)) == 0) assert not np.any(np.sum(outputs, 1) == 0) return GradientData(inputs, outputs, PRED_WINDOW)
for elem in range(TESTSIZE): rank_real = np.unique(ytest[elem, :], return_inverse=True)[1] rank_pred = np.unique(pred_test[elem, :], return_inverse=True)[1] sum_rank_diffs += np.sum(np.abs(rank_real - rank_pred)) print("TEST") print('step %d, test loss %g, rank loss %g' % (global_count, cur_l, sum_rank_diffs / TESTSIZE)) print("TEST") test_losses.append(cur_l) test_rank_errors.append(sum_rank_diffs / TESTSIZE) net_model.train(xbatch, ybatch, keep=0.5) if __name__ == "__main__": if TRAIN_BOUT_FREQ == 1: trainingData_1 = GradientData.load("gd_training_data.hdf5") trainingData_2 = GradientData.load("gd_training_data_rev.hdf5") testData = GradientData.load("gd_test_data_radial.hdf5") elif TRAIN_BOUT_FREQ == 0.5: trainingData_1 = GradientData.load("gd_05Hz_training_data.hdf5") trainingData_2 = GradientData.load("gd_05Hz_training_data_rev.hdf5") testData = GradientData.load("gd_05Hz_test_data_radial.hdf5") elif TRAIN_BOUT_FREQ == 2: trainingData_1 = GradientData.load("gd_2Hz_training_data.hdf5") trainingData_2 = GradientData.load("gd_2Hz_training_data_rev.hdf5") testData = GradientData.load("gd_2Hz_test_data_radial.hdf5") else: raise Exception( "No training data has been generated for the requested bout frequency" ) # enforce same scaling on testData as on trainingData
if global_step % EVAL_TEST_EVERY == 0: test() data_batch = train_list[td_ix[1]].training_batch(BATCHSIZE) train_func(data_batch[0], data_batch[1], removal=droplist) global_step += 1 sf = m.save_state(chk_file, global_step, True) print("Retrained model saved in file {0}.".format(sf)) error_file = h5py.File(save_path + "/losses.hdf5", "x") error_file.create_dataset("test_rank_errors", data=np.array(test_errors)) error_file.create_dataset("test_eval", data=np.array(test_steps)) error_file.close() if __name__ == '__main__': # load training and test data tD_1 = GradientData.load("ce_gd_training_data.hdf5") tD_2 = GradientData.load("ce_gd_training_data_rev.hdf5") tD_2.copy_normalization(tD_1) train_list = [tD_1, tD_2] testData = GradientData.load("ce_gd_test_data_radial.hdf5") # enforce same scaling on testData as on trainingData testData.copy_normalization(tD_1) ana = a.Analyzer(MoTypes(True), tD_1.standards, None, "ce_activity_store.hdf5") # load cell unit ids and cluster ids dfile = h5py.File("stimFile.hdf5", 'r') tsin = np.array(dfile['sine_L_H_temp']) x = np.arange(tsin.size) # stored at 20 Hz ! xinterp = np.linspace(0, tsin.size,
n_iter = 500000 n_gen = 50 TPREFERRED = 26 # Indicates the baseline bout frequency TRAIN_BOUT_FREQ = 1 def mpath(path): return base_path + path[:-1] # need to remove trailing slash if __name__ == "__main__": # load training data for scaling - dependent on bout frequency if TRAIN_BOUT_FREQ == 1: std = GradientData.load_standards("gd_training_data.hdf5") elif TRAIN_BOUT_FREQ == 0.5: std = GradientData.load_standards("gd_05Hz_training_data.hdf5") elif TRAIN_BOUT_FREQ == 2: std = GradientData.load_standards("gd_2Hz_training_data.hdf5") else: raise Exception( "No training data has been generated for the requested bout frequency" ) # evolve each 512 network unless it has been done before for p in paths_512: model_path = mpath(p) savedir = model_path + '/evolve/' if os.path.exists(savedir): print(
if response == "y": n_steps = int(input("Number of steps to perform?")) gradsim = CircGradientTrainer(100, 22, 37) gradsim.p_move *= TRAIN_BOUT_FREQ # Adjust bout frequency during training data navigation print("Running radial simulation, inside-out") pos = gradsim.run_simulation(n_steps) pl.figure() pl.plot(pos[:, 0], pos[:, 1]) pl.xlabel("X position [mm]") pl.ylabel("Y position [mm]") sns.despine() print("Generating gradient data") grad_data = gradsim.create_dataset(pos) all_in = grad_data.model_in_raw all_out = grad_data.model_out_raw print("Running radial simulation, outside-in") gradsim = CircGradientTrainer(100, 37, 22) gradsim.p_move *= TRAIN_BOUT_FREQ pos = gradsim.run_simulation(n_steps) pl.figure() pl.plot(pos[:, 0], pos[:, 1]) pl.xlabel("X position [mm]") pl.ylabel("Y position [mm]") sns.despine() print("Generating gradient data") grad_data = gradsim.create_dataset(pos) all_in = np.r_[all_in, grad_data.model_in_raw] all_out = np.r_[all_out, grad_data.model_out_raw] grad_data = GradientData(all_in, all_out, grad_data.pred_window) print("Done")
from core import GradientData, ModelData, ZfGpNetworkModel from trainingData import CircGradientTrainer from global_defs import GlobalDefs if __name__ == "__main__": if sys.platform == "darwin" and "Tk" not in mpl.get_backend(): print( "On OSX tkinter likely does not work properly if matplotlib uses a backend that is not TkAgg!" ) print( "If using ipython activate TkAgg backend with '%matplotlib tk' and retry." ) sys.exit(1) # load training data to obtain temperature scaling try: std = GradientData.load_standards("gd_training_data.hdf5") except IOError: print("No standards found attempting to load full training data") train_data = GradientData.load("gd_training_data.hdf5") std = train_data.standards # load and interpolate temperature stimulus dfile = h5py.File("stimFile.hdf5", 'r') tsin = np.array(dfile['sine_L_H_temp']) x = np.arange(tsin.size) # stored at 20 Hz ! xinterp = np.linspace(0, tsin.size, tsin.size * GlobalDefs.frame_rate // 20) temp = np.interp(xinterp, x, tsin) dfile.close() print("Select model directory") root = tk.Tk() root.update()
sum_rank_diffs = 0.0 for elem in range(TESTSIZE): rank_real = np.unique(ytest[elem, :], return_inverse=True)[1] rank_pred = np.unique(pred_test[elem, :], return_inverse=True)[1] sum_rank_diffs += np.sum(np.abs(rank_real - rank_pred)) print("TEST") print('step %d, test loss %g, rank loss %g' % (global_count, cur_l, sum_rank_diffs / TESTSIZE)) print("TEST") test_losses.append(cur_l) test_rank_errors.append(sum_rank_diffs / TESTSIZE) net_model.train(xbatch, ybatch) if __name__ == "__main__": trainingData = GradientData.load("photo_training_data.hdf5") testData = GradientData.load("photo_test_data.hdf5") # enforce same scaling on testData as on trainingData testData.copy_normalization(trainingData) epoch_size = trainingData.data_size // BATCHSIZE train_losses = [] rank_errors = [] test_losses = [] test_rank_errors = [] global_count = 0 total_steps = N_EPOCHS * epoch_size with ZfGpNetworkModel() as Model: Model.setup(N_CONV, N_UNITS, N_BRANCH, N_MIXED) # save naive model including full graph save_path = Model.save_state(chk_file, 0)
mo_type = "" while mo_type != "c" and mo_type != "z": mo_type = input("Please select either (z)ebrafish or (c) elegans simulation [z/c]:") mo_type = mo_type.lower() n_steps = 2000000 TPREFERRED = 25 root = tk.Tk() root.update() root.withdraw() print("Select model directory") model_dir = filedialog.askdirectory(title="Select directory with model checkpoints", initialdir="./model_data/") root.update() mdata = ModelData(model_dir) # load training data for scaling if mo_type == "z": std = GradientData.load_standards("gd_training_data.hdf5") else: std = GradientData.load_standards("ce_gd_training_data.hdf5") sim_type = "" while sim_type != "l" and sim_type != "r": sim_type = input("Please select either (l)inear or (r)adial simulation [l/r]:") if mo_type == "z": mot = MoTypes(False) else: mot = MoTypes(True) gpn_naive = mot.network_model() gpn_naive.load(mdata.ModelDefinition, mdata.FirstCheckpoint) gpn_trained = mot.network_model() gpn_trained.load(mdata.ModelDefinition, mdata.LastCheckpoint) if sim_type == "l": sim_type = "x" # so we call run_simulation correctly later