with open(data_path + "/tracker_input.pickle", "rb") as f: tracker_input = pickle.load(f) Generator_vec = TrainedGenerator(path_to_meta=meta_path_vec, path_to_config=config_path_vec, gpu_options=gpu_options) time_it(data=tracker_input, generator=Generator_vec, batch_size=batch_size, mode=prefix + "Vector") elif time_id == 1: model_path_im = "../../Results/B2Dmunu/{}/".format(model_im) meta_path_im = model_path_im + "TFGraphs/" config_path_im = model_path_im + "config.json" with open(data_path + "/Trained/PiplusLowerP_CWGANGP8_out_1.pickle", "rb") as f: cgan_images = np.clip(padding_zeros(pickle.load(f), top=4, bottom=4), a_min=0, a_max=6120).reshape([-1, 64, 64, 1]) / 6120 Generator_im = TrainedIm2Im(path_to_meta=meta_path_im, path_to_config=config_path_im, gpu_options=gpu_options) time_it(data=cgan_images, generator=Generator_im, batch_size=batch_size, mode=prefix + "Image") elif time_id == 2: model_path_direct = "../../Results/ServerTemp/B2DmunuTracker/1Good/BiCycleGANTracker13/" meta_path_direct = model_path_direct + "TFGraphs/" config_path_direct = model_path_direct + "config.json" with open(data_path + "/tracker_images.pickle", "rb") as f: tracker_images = padding_zeros(
path_loading = "../Data/B2Dmunu/LargeSample" path_results = "../Results/B2Dmunu" else: path_loading = "../Data/B2Dmunu/Debug" path_results = "../Results/Test/B2Dmunu" ############################################################################################################ # Data loading ############################################################################################################ with open("{}/tracker_images.pickle".format(path_loading), "rb") as f: train_x = pickle.load(f) with open("{}/calo_images.pickle".format(path_loading), "rb") as f: train_y = pickle.load(f) image_shape = [64, 64, 1] train_x = padding_zeros(train_x, **padding).reshape([-1, *image_shape])[:50000] train_y = padding_zeros(train_y, **padding).reshape([-1, *image_shape])[:50000] energy_scaler = np.max(train_y) train_x /= energy_scaler train_y /= energy_scaler nr_test = int(min(0.1 * len(train_x), 100)) test_x = train_x[-nr_test:] train_x = train_x[:-nr_test] test_y = train_y[-nr_test:] train_y = train_y[:-nr_test] nr_train = len(train_x)
# x_test = x_test.reshape((len(x_test), np.prod(x_test.shape[1:]))) # y_train = enc.fit_transform(y_train.reshape(-1, 1)) # y_test = enc.transform(y_test.reshape(-1, 1)) # gen_architecture = [ # [tf.layers.dense, {"units": 256, "activation": tf.nn.relu}], # [tf.layers.dense, {"units": 512, "activation": tf.nn.relu}], # ] # critic_architecture = [ # [tf.layers.dense, {"units": 512, "activation": tf.nn.relu}], # [tf.layers.dense, {"units": 256, "activation": tf.nn.relu}], # ] # inpt_dim = 784 # image_shape=[28, 28, 1] ########### Image input x_train = padding_zeros(x_train, top=2, bottom=2, left=2, right=2) x_train = np.reshape(x_train, newshape=(-1, 32, 32, 1)) x_test = padding_zeros(x_test, top=2, bottom=2, left=2, right=2) x_test = np.reshape(x_test, newshape=(-1, 32, 32, 1)) x_train_log = [x_train[y_train.tolist().index(i)] for i in range(10)] x_train_log = np.reshape(x_train_log, newshape=(-1, 32, 32, 1)) y_train = enc.fit_transform(y_train.reshape(-1, 1)) y_test = enc.transform(y_test.reshape(-1, 1)) gen_architecture = [ [tf.layers.dense, { "units": 4 * 4 * 512, "activation": tf.nn.relu }], [reshape_layer, {
# axs[-1].set_xlabel(r"Tracker $E_T$") # axs[-1].set_ylabel(r"HCAL $E_T$") # plt.savefig(savefolder+"/cGAN_energy.png", dpi=300) ##################################################################################################### # Generate example images ##################################################################################################### nr_samples = 100 calo_images = data["Calo"][:nr_samples]*energy_scaler/1000 inputs = data["Tracker"][keep_cols].values[:nr_samples] inputs = [[track] for track in inputs] generated_images = Generator.generate_batches(inputs, batch_size=100)*energy_scaler/1000 plt_idxs = [] calo_images = padding_zeros(calo_images, top=2, bottom=2) tracker_images = np.flip(padding_zeros(tracker_images, top=2, bottom=2)[:nr_samples] / 1000, axis=1) def set_title_for(ax, im): im = im.reshape([1, 56, 64]) energy, max_energy = get_energies(im), get_max_energy(im) ax.set_title("Energy: %.2f GeV\nMaximum Energy: %.2f GeV" % (energy, max_energy)) idxs = [1, 2, 7, 12, 17, 18] for row, idx in enumerate(idxs): print(inputs[idx]) max_e = np.max([np.max(calo_images[idx]), np.max(generated_images[idx])]) fig, axs = plt.subplots(nrows=1, ncols=3, figsize=(10, 7)) fig.subplots_adjust(left=0.05, bottom=0.02, right=0.999, top=0.999, wspace=0.3, hspace=0.4) axs[0].imshow(calo_images[idx], vmin=0, vmax=max_e) # set_title_for(ax=axs[0], im=calo_images[idx])
##################################################################################################### # Data loadng ##################################################################################################### simulation = "B2Dmunu" path_loading = "../../Data/{}/LargeSample".format(simulation) nr_sim = 1 image_shape = [56, 64] padding = {"top": 2, "bottom": 2, "left":0, "right":0} if simulation == "Piplus": data, scaler = init.load_processed_data(path_loading, mode="train") calo_images = data["Calo"]*scaler["Calo"] / 1000 calo_images = padding_zeros(calo_images, **padding) with open(path_loading+"/tracker_images.pickle".format(nr_sim), "rb") as f: tracker_images = pickle.load(f)[data["Idx"]] tracker_events = pd.read_csv(path_loading+"/tracker_events.csv").loc[data["Idx"]] tracker_real_ET = tracker_events["real_ET"].values exclude = ["phi", "theta", "region"] columns = tracker_events.drop(exclude, axis=1).columns transformed_events = scaler["Tracker"].transform(tracker_events.drop(exclude, axis=1)) transformed_events = pd.DataFrame(data=transformed_events, columns=columns, index=tracker_events.index) tracker_events = pd.concat([transformed_events, tracker_events[exclude]], axis=1) elif simulation == "B2Dmunu": with open("../../Data/Piplus/LargeSample/ProcessedScaler.pickle", "rb") as f: scaler = pickle.load(f) with open(path_loading+"/calo_images.pickle", "rb") as f:
"../../Data/B2Dmunu/TestingPurpose/Trained/PiplusLowerP_CWGANGP8_out_1.pickle", "rb") as f: gan_data = pickle.load(f) with open("../../Data/B2Dmunu/TestingPurpose/tracker_images.pickle", "rb") as f: tracker_images = pickle.load(f) with open("../../Data/B2Dmunu/TestingPurpose/tracker_events.pickle", "rb") as f: tracker_events = pickle.load(f) tracker_real_ET = tracker_events["real_ET"].apply( sum).to_numpy()[-nr_test_hist:] with open("../../Data/Piplus/LargeSample/ProcessedScaler.pickle", "rb") as f: scaler = pickle.load(f) calo_scaler = scaler["Calo"] mc_data_images_m = padding_zeros(mc_data[-nr_test_hist:], top=6, bottom=6).reshape(-1, 64, 64) / calo_scaler gan_data_m = np.clip(padding_zeros(gan_data[-nr_test_hist:], top=4, bottom=4), a_min=0, a_max=calo_scaler) / calo_scaler tracker_images_m = padding_zeros( tracker_images[-nr_test_hist:], top=6, bottom=6).reshape( [-1, *image_shape]) / calo_scaler ##################################################################################################### # Model loading ##################################################################################################### for model_idx, model_path in enumerate(model_paths): print("Working on {} / {}: {}...".format(model_idx + 1, len(model_paths), model_path)) if os.path.exists(model_path + "/Evaluation.pdf"):
os.mkdir(saving_path) ##################################################################################################### # Data loading ##################################################################################################### Generator = TrainedGenerator(path_to_meta=meta_path, path_to_config=config_path) with open(config_path, "r") as f: config = json.load(f) padding = config["padding"] keep_cols = config["keep_cols"] data, scaler = init.load_processed_data(data_path=data_path, mode="test", return_scaler=True) calo_images = padding_zeros(data["Calo"], **padding) tracker_events = data["Tracker"] tracker_events_list = [] for _, row in tracker_events.iterrows(): tracker_events_list.append([row[keep_cols].tolist()]) tracker_events_list = np.array(tracker_events_list) image_shape = calo_images.shape[1:] ##################################################################################################### # Check for correctness ##################################################################################################### # data_path = "../../Data/B2Dmunu/Debug" # with open(data_path+"/calo_images.pickle", "rb") as f: # calo_images = pickle.load(f) # calo_images = padding_zeros(calo_images, **padding) # image_shape = calo_images.shape[1:]
else: data_path = "/home/tneuer/Backup/Algorithmen/0TestData/image_to_image/fashion_mnist" with open("{}/train_images.pickle".format(data_path), "rb") as f: x_train_orig = (pickle.load(f)[0].reshape(-1, 28, 28, 1) / 255)[:20000] x_test_orig = x_train_orig[-100:] x_train_orig = x_train_orig[:-100] with open("{}/train_images_rotated.pickle".format(data_path), "rb") as f: y_train_orig = (pickle.load(f).reshape(-1, 28, 28, 1) / 255)[:20000] y_test_orig = y_train_orig[-100:] y_train_orig = y_train_orig[:-100] ########### Architecture x_train_orig = padding_zeros(x_train_orig, top=2, bottom=2, left=2, right=2) x_test_orig = padding_zeros(x_test_orig, top=2, bottom=2, left=2, right=2) y_train_orig = padding_zeros(y_train_orig, top=2, bottom=2, left=2, right=2) y_test_orig = padding_zeros(y_test_orig, top=2, bottom=2, left=2, right=2) inpt_dim = x_train_orig[0].shape opt_dim = y_train_orig[0].shape param_dict = { "adv_steps": [1], "architecture": ["keraslike"],
if __name__ == '__main__': ########### Image input import pickle if "lhcb_data2" in os.getcwd(): data_path = "../../Data/fashion_mnist" else: data_path = "/home/tneuer/Backup/Algorithmen/0TestData/image_to_image/fashion_mnist" with open("{}/train_images.pickle".format(data_path), "rb") as f: x_train = pickle.load(f)[0].reshape(-1, 28, 28, 1) / 255 with open("{}/train_images_rotated.pickle".format(data_path), "rb") as f: y_train = pickle.load(f).reshape(-1, 28, 28, 1) / 255 ########### Architecture x_train = padding_zeros(x_train, top=2, bottom=2, left=2, right=2) y_train = padding_zeros(y_train, top=2, bottom=2, left=2, right=2) gen_xy_architecture = [ [conv2d_logged, {"filters": 32, "kernel_size": 5, "strides": 1, "padding": "same", "activation": tf.nn.leaky_relu}], [tf.layers.max_pooling2d, {"pool_size": 2, "strides": 2}], # [unet_original, {"depth": 2, "activation": tf.nn.leaky_relu}], # [inception_block, {"filters": 32}], [conv2d_transpose_logged, {"filters": 32, "kernel_size": 2, "strides": 2, "activation": tf.nn.leaky_relu}], [conv2d_logged, {"filters": 1, "kernel_size": 2, "strides": 1, "padding":"same", "activation": tf.nn.sigmoid}] ] disc_xy_architecture = [ # [tf.layers.conv2d, {"filters": 32, "kernel_size": 2, "strides": 2, "activation": tf.nn.leaky_relu}], # [residual_block, {"filters": 32, "kernel_size": 2, "activation": tf.nn.leaky_relu, "skip_layers": 3}], # [tf.layers.conv2d, {"filters": 64, "kernel_size": 2, "strides": 2, "activation": tf.nn.leaky_relu}], # [residual_block, {"filters": 128, "kernel_size": 2, "activation": tf.nn.leaky_relu, "skip_layers": 3}], # [inception_block, {"filters": 32}],
os.mkdir(path_saving + "/Evaluation/Cells") os.mkdir(path_saving + "/Evaluation/CenterOfMassX") os.mkdir(path_saving + "/Evaluation/CenterOfMassY") os.mkdir(path_saving + "/Evaluation/Energy") os.mkdir(path_saving + "/Evaluation/MaxEnergy") os.mkdir(path_saving + "/Evaluation/StdEnergy") os.mkdir(path_saving + "/ModelSave") ############################################################################################################ # Data loading ############################################################################################################ with open( "{}/Trained/PiplusLowerP_CWGANGP8_out_1.pickle".format( path_loading), "rb") as f: train_x = pickle.load(f) train_x = padding_zeros(train_x, **padding) with open("{}/calo_images.pickle".format(path_loading), "rb") as f: train_y = pickle.load(f) train_y = padding_zeros(train_y, top=6, bottom=6, left=0, right=0).reshape([-1, 64, 64, 1]) energy_scaler = np.max(train_y) train_x = np.clip(train_x, a_min=0, a_max=energy_scaler) train_x /= energy_scaler train_y /= energy_scaler nr_test = int(min(0.1 * len(train_x), 500)) test_x = train_x[-nr_test:] train_x = train_x[:-nr_test] test_y = train_y[-nr_test:]
nr_samples = 20000 compare_to = "STP" ##################################################################################################### # Data loading ##################################################################################################### with open(config_path_vec, "r") as f: config = json.load(f) padding = config["padding"] keep_cols = config["keep_cols"] energy_scaler = 6120 with open(data_path + "/calo_images.pickle", "rb") as f: calo_images = padding_zeros(pickle.load(f)[:nr_samples], top=6, bottom=6) / 1000 with open(data_path + "/tracker_input.pickle", "rb") as f: tracker_input = pickle.load(f)[:nr_samples] with open(data_path + "/tracker_events.pickle", "rb") as f: tracker_events = pickle.load(f)[:nr_samples] tracker_real_ET = tracker_events["real_ET"].apply(sum).to_numpy() / 1000 with open(data_path + "/Trained/PiplusLowerP_CWGANGP8_out_1.pickle", "rb") as f: cgan_images = pickle.load(f)[:nr_samples] with open(data_path + "/tracker_images.pickle", "rb") as f: tracker_images = padding_zeros( pickle.load(f)[:nr_samples], top=6, bottom=6) / energy_scaler assert tracker_input.shape[0] == calo_images.shape[0] == tracker_events.shape[ 0], ("Shape mismatch: {}, {}, {}.".format(tracker_input.shape, calo_images.shape,
############################################################################################################ # Data loading ############################################################################################################ if not os.path.exists(path_results): os.mkdir(path_results) path_saving = init.initialize_folder(algorithm=algorithm, base_folder=path_results) data, scaler = init.load_processed_data(path_loading, return_scaler=True) train_calo = data["train"]["Calo"][:nr_train] train_tracker = data["train"]["Tracker"][:nr_train] test_calo = data["test"]["Calo"] test_tracker = data["test"]["Tracker"] train_calo = padding_zeros(train_calo, **padding).reshape([-1, *image_shape]) test_calo = padding_zeros(test_calo, **padding).reshape([-1, *image_shape]) test_calo = test_calo[:nr_test] logging_calo = test_calo[:15] ##### Rescale and check that identical def invert_standardize_data(data, scaler, exclude=None): import pandas as pd standardized_data = data.drop(exclude, axis=1, inplace=False) colnames = standardized_data.columns.values standardized_data = pd.DataFrame(data=scaler.inverse_transform(standardized_data), columns=colnames, index=data.index) data = pd.concat([standardized_data, data[exclude]], axis=1, sort=False) return data train_tracker["real_ET"] = invert_standardize_data(data=train_tracker, scaler=scaler["Tracker"], exclude=["theta", "phi", "region"])["real_ET"] train_tracker["real_ET"] /= scaler["Calo"]