def _test(): from dataset import get_data_iter from keras4hep.projects.qgjets.utils import get_dataset_paths path = get_dataset_paths(min_pt=100)["training"] data_iter = get_data_iter(path) batch = data_iter.next() x_shape = data_iter.get_shape("x", batch_shape=False) model = build_model(x_shape) logits = model.predict_on_batch(batch.x) print("logits: {}".format(logits.shape))
def _test(): from dataset import get_data_iter from keras4hep.projects.qgjets.utils import get_dataset_paths path = get_dataset_paths(min_pt=100)["training"] seq_maxlen = { "x": (40, "float32"), } data_iter = get_data_iter(path, seq_maxlen=seq_maxlen) batch = data_iter.next() x_shape = data_iter.get_shape("x", batch_shape=False) model = build_classifier(x_shape, name="RNN") model.summary()
def _test(): from dataset import get_data_iter from keras4hep.projects.qgjets.utils import get_dataset_paths path = get_dataset_paths(min_pt=100)["training"] data_iter = get_data_iter(path, seq_maxlen={"x_kin": 32, "x_pid": 32}) batch = data_iter.next() x_kin_shape = data_iter.get_shape("x_kin", batch_shape=False) x_pid_shape = data_iter.get_shape("x_pid", batch_shape=False) model = build_model(x_kin_shape, x_pid_shape, name="RNN") model.summary() y_score = model.predict_on_batch([batch.x_kin, batch.x_pid, batch.x_len]) print("y_score: {}".format(y_score.shape))
def _test(): from keras4hep.projects.qgjets.utils import get_dataset_paths from dataset import get_data_iter paths = get_dataset_paths(min_pt=100) data_iter = get_data_iter(paths["training"]) batch = data_iter.next() for key, value in batch.iteritems(): print(key, value.shape) print(data_iter._dataset[:1]["x_img"].shape[1:]) print(data_iter.get_shape("x_img", False)) print(data_iter.get_shape("x_img", True))
def _test(): from keras4hep.projects.qgjets.utils import get_dataset_paths from dataset import get_data_iter paths = get_dataset_paths(min_pt=100) data_iter = get_data_iter(paths["training"]) batch = data_iter.next() x_shape = data_iter.get_shape("x", batch_shape=False) model = build_model(x_shape, kernel_size=7, name="VanillaConvNet") logits = model.predict_on_batch(batch.x) print("logits: {}".format(logits.shape)) model.summary()
def _test(): from dataset import get_data_iter from keras4hep.projects.qgjets.utils import get_dataset_paths path = get_dataset_paths(min_pt=100)["training"] seq_maxlen = { "x": (40, "float32"), } data_iter = get_data_iter(path, seq_maxlen=seq_maxlen) batch = data_iter.next() x_shape = data_iter.get_shape("x", batch_shape=False) model = build_model(x_shape, name="RNN") model.summary() y_score = model.predict_on_batch([batch.x]) print("y_score: {}".format(y_score.shape))
def main(): parser = argparse.ArgumentParser() parser.add_argument("--name", default="rnn{}".format(datetime.now().strftime("%y%m%d%H%M%S"))) parser.add_argument("--directory", default="./logs") # GPU parser.add_argument("--num_gpus", default=len(get_available_gpus()), type=int) parser.add_argument("--multi-gpu", default=False, action='store_true', dest='multi_gpu') # Hyperparameters parser.add_argument("--epoch", dest="epochs", default=200, type=int) parser.add_argument("--batch_size", default=128, type=int) parser.add_argument("--valid_batch_size", default=1024, type=int) # Optimizer parser.add_argument("--optimizer", default="Adam", type=str) parser.add_argument("--lr", default=0.001, type=float) parser.add_argument("--clipnorm", default=-1, type=float, help="if it is greater than 0, then graidient clipping is activated") parser.add_argument("--clipvalue", default=-1, type=float) parser.add_argument("--use-class-weight", dest="use_class_weight", default=False, action="store_true") # Frequencies parser.add_argument("--valid_freq", type=int, default=32) parser.add_argument("--save_freq", type=int, default=32) parser.add_argument("-v", "--verbose", action="store_true") # Project parameters parser.add_argument("--min-pt", dest="min_pt", default=100, type=int) # Model Archtecture parser.add_argument("--act", dest="activation", default="elu", type=str) parser.add_argument("--rnn", default="gru", type=str) args = parser.parse_args() ################### # ################### log_dir = Directory(path=os.path.join(args.directory, args.name)) log_dir.mkdir("script") log_dir.mkdir("checkpoint") log_dir.mkdir("learning_curve") log_dir.mkdir("roc_curve") log_dir.mkdir("model_response") config = Config(log_dir.path, "w") config.append(args) config["hostname"] = os.environ["HOSTNAME"] config["log_dir"] = log_dir.path config.save() scripts = [ "./dataset.py", "./model.py", "./train.py", ] for each in scripts: shutil.copy2(each, log_dir.script.path) open(log_dir.script.concat("__init__.py"), 'w').close() ######################################## # Load training and validation datasets ######################################## dset = get_dataset_paths(config.min_pt) config.append(dset) config["seq_maxlen"] = { "x_kin": 30, "x_pid": 30 } train_iter = get_data_iter( path=dset["training"], batch_size=config.batch_size, seq_maxlen=config.seq_maxlen, fit_generator_mode=True) valid_iter = get_data_iter( path=dset["validation"], batch_size=config.valid_batch_size, seq_maxlen=config.seq_maxlen, fit_generator_mode=True) test_iter = get_data_iter( path=dset["test"], batch_size=config.valid_batch_size, seq_maxlen=config.seq_maxlen, fit_generator_mode=False) if config.use_class_weight: class_weight = get_class_weight(train_iter) config["class_weight"] = list(class_weight) else: class_weight = None ################################# # Build & Compile a model. ################################# x_kin_shape = train_iter.get_shape("x_kin", batch_shape=False) x_pid_shape = train_iter.get_shape("x_pid", batch_shape=False) model = build_model( x_kin_shape, x_pid_shape, rnn=config.rnn, activation=config.activation, name=config.name) config["model"] = model.get_config() if config.multi_gpu: model = multi_gpu_model(model, gpus=config.num_gpus) if config.hostname == "cms05.sscc.uos.ac.kr": model_plot_path = log_dir.concat("model.png") plot_model(model, to_file=model_plot_path, show_shapes=True) else: model.summary() loss = 'categorical_crossentropy' # TODO capsulisation optimizer_kwargs = {} if config.clipnorm > 0: optimizer_kwargs["clipnorm"] = config.clipnorm if config.clipvalue > 0: optimizer_kwargs["clipvalue"] = config.clipvalue optimizer = getattr(optimizers, config.optimizer)(lr=config.lr, **optimizer_kwargs) metric_list = ["accuracy" , roc_auc] model.compile( loss=loss, optimizer=optimizer, metrics=metric_list) config["loss"] = loss config["optimizer_config"] = optimizer.get_config() ########################################################################### # Callbacks ########################################################################### ckpt_format_str = "weights_epoch-{epoch:02d}_loss-{val_loss:.4f}_acc-{val_acc:.4f}_auc-{val_roc_auc:.4f}.hdf5" ckpt_path = log_dir.checkpoint.concat(ckpt_format_str) csv_log_path = log_dir.concat("log_file.csv") learning_curve = LearningCurve(directory=log_dir.learning_curve.path) learning_curve.book(x="step", y="roc_auc", best="max") learning_curve.book(x="step", y="acc", best="max") learning_curve.book(x="step", y="loss", best="min") callback_list = [ callbacks.ModelCheckpoint(filepath=ckpt_path), callbacks.ReduceLROnPlateau(verbose=1), callbacks.CSVLogger(csv_log_path), learning_curve, ] ############################################################################ # Training ############################################################################ model.fit_generator( train_iter, steps_per_epoch=len(train_iter), epochs=config.epochs, validation_data=valid_iter, validation_steps=len(valid_iter), callbacks=callback_list, shuffle=True, class_weight=class_weight) del model print("Training is over! :D") ########################################### # Evaluation ############################################ train_iter.fit_generator_mode = False train_iter.cycle = False good_ckpt = find_good_checkpoint( log_dir.checkpoint.path, which={"max": ["auc", "acc"], "min": ["loss"]}) all_ckpt = set(log_dir.checkpoint.get_entries()) # no local optima useless_ckpt = all_ckpt.difference(good_ckpt) for each in useless_ckpt: os.remove(each) for idx, each in enumerate(good_ckpt, 1): print("[{}/{}] {}".format(idx, len(good_ckpt), each)) K.clear_session() evaluate(checkpoint_path=each, train_iter=train_iter, test_iter=test_iter, log_dir=log_dir) config.save()
def main(): parser = argparse.ArgumentParser() parser.add_argument("--logdir", dest="log_dir", type=str, default="./logs/untitled-{}".format( datetime.now().strftime("%y%m%d-%H%M%S"))) parser.add_argument("--num_gpus", default=len(get_available_gpus()), type=int) parser.add_argument("--multi-gpu", default=False, action='store_true', dest='multi_gpu') # Hyperparameters parser.add_argument("--epoch", dest="num_epochs", default=100, type=int) parser.add_argument("--batch_size", default=128, type=int) parser.add_argument("--valid_batch_size", default=1024, type=int) parser.add_argument("--lr", default=0.001, type=float) parser.add_argument( "--clipnorm", default=-1, type=float, help="if it is greater than 0, then graidient clipping is activated") parser.add_argument("--clipvalue", default=-1, type=float) parser.add_argument("--use-class-weight", dest="use_class_weight", default=False, action="store_true") # Frequencies parser.add_argument("--valid_freq", type=int, default=32) parser.add_argument("--save_freq", type=int, default=32) parser.add_argument("-v", "--verbose", action="store_true") # Project parameters parser.add_argument("--min-pt", dest="min_pt", default=100, type=int) args = parser.parse_args() ################### # ################### log_dir = Directory(path=args.log_dir) log_dir.mkdir("script") log_dir.mkdir("model_checkpoint") log_dir.mkdir("learning_curve") log_dir.mkdir("roc_curve") log_dir.mkdir("model_response") backup_scripts(log_dir.script.path) config = Config(log_dir.path, "w") config.append(args) config["hostname"] = os.environ["HOSTNAME"] ######################################## # Load training and validation datasets ######################################## dset = get_dataset_paths(args.min_pt) config.append(dset) config["seq_maxlen"] = {"x": 30} train_iter = get_data_iter(path=dset["training"], batch_size=args.batch_size, seq_maxlen=config.seq_maxlen, fit_generator_mode=True) valid_iter = get_data_iter(path=dset["validation"], batch_size=args.valid_batch_size, seq_maxlen=config.seq_maxlen, fit_generator_mode=True) test_iter = get_data_iter(path=dset["test"], batch_size=args.valid_batch_size, seq_maxlen=config.seq_maxlen, fit_generator_mode=False) if args.use_class_weight: class_weight = get_class_weight(train_iter) config["class_weight"] = list(class_weight) else: class_weight = None ################################# # Build & Compile a model. ################################# x_shape = train_iter.get_shape("x", batch_shape=False) model = build_a_model(x_shape=x_shape) config["model"] = model.get_config() if args.multi_gpu: model = multi_gpu_model(_model, gpus=args.num_gpus) model_plot_path = log_dir.concat("model.png") plot_model(model, to_file=model_plot_path, show_shapes=True) loss = 'categorical_crossentropy' # TODO capsulisation optimizer_kwargs = {} if args.clipnorm > 0: optimzer_kwargs["clipnorm"] = args.clipnorm if args.clipvalue > 0: optimzer_kwargs["clipvalue"] = args.clipvalue optimizer = optimizers.Adam(lr=args.lr, **optimizer_kwargs) metric_list = ["accuracy", roc_auc] model.compile(loss=loss, optimizer=optimizer, metrics=metric_list) config["loss"] = loss config["optimizer"] = "Adam" config["optimizer_config"] = optimizer.get_config() ########################################################################### # Callbacks ########################################################################### ckpt_format_str = "weights_epoch-{epoch:02d}_loss-{val_loss:.4f}_acc-{val_acc:.4f}_auc-{val_roc_auc:.4f}.hdf5" ckpt_path = log_dir.model_checkpoint.concat(ckpt_format_str) csv_log_path = log_dir.concat("log_file.csv") learning_curve = LearningCurve(directory=log_dir.learning_curve.path) learning_curve.book(x="step", y="roc_auc", best="max") learning_curve.book(x="step", y="acc", best="max") learning_curve.book(x="step", y="loss", best="min") callback_list = [ callbacks.ModelCheckpoint(filepath=ckpt_path), callbacks.EarlyStopping(monitor="val_loss", patience=5), callbacks.ReduceLROnPlateau(), callbacks.CSVLogger(csv_log_path), learning_curve, ] ############################################################################ # Training ############################################################################ model.fit_generator(train_iter, steps_per_epoch=len(train_iter), epochs=50, validation_data=valid_iter, validation_steps=len(valid_iter), callbacks=callback_list, shuffle=True, class_weight=class_weight) print("Training is over! :D") del model ########################################### # Evaluation ############################################ train_iter.fit_generator_mode = False train_iter.cycle = False good_ckpt = find_good_checkpoint(log_dir.model_checkpoint.path, which={ "max": ["auc", "acc"], "min": ["loss"] }) for idx, each in enumerate(good_ckpt, 1): print("[{}/{}] {}".format(idx, len(good_ckpt), each)) K.clear_session() evaluate(custom_objects={"roc_auc": roc_auc}, checkpoint_path=each, train_iter=train_iter, test_iter=test_iter, log_dir=log_dir) config.save()
def train(): parser = argparse.ArgumentParser() parser.add_argument("--datasets_dir", default="/store/slowmoyang/QGJets/data/root_100_200/2-Refined/", type=str) parser.add_argument("--log_dir", default="./logs/{name}", type=str) parser.add_argument("--num_gpus", default=len(get_available_gpus()), type=int) parser.add_argument("--multi-gpu", default=False, action='store_true', dest='multi_gpu') # Hyperparameters parser.add_argument("--num_epochs", default=50, type=int) parser.add_argument("--batch_size", default=128, type=int) parser.add_argument("--valid_batch_size", default=1024, type=int) parser.add_argument("--lr", default=0.001, type=float) # Frequencies parser.add_argument("--valid_freq", type=int, default=32) parser.add_argument("--save_freq", type=int, default=32) parser.add_argument("-v", "--verbose", action="store_true") # Project parameters args = parser.parse_args() ######################################################### # Log directory ####################################################### if '{name}' in args.log_dir: args.log_dir = args.log_dir.format( name="Untitled_{}".format( datetime.today().strftime("%Y-%m-%d_%H-%M-%S"))) log_dir = get_log_dir(path=args.log_dir, creation=True) # Config config = Config(dpath=log_dir.path, mode="WRITE") config.update(args) dataset_paths = get_dataset_paths(config.datasets_dir) for key, value in dataset_paths.iteritems(): print("{}: {}".format(key, value)) config.update(dataset_paths) ######################################## # Load training and validation datasets ######################################## config["seq_maxlen"] = {"x_kinematics": 40, "x_pid": 40} train_iter = get_data_iter( path=config.dijet_training_set, seq_maxlen=config.seq_maxlen, batch_size=config.batch_size) valid_dijet_iter = get_data_iter( path=config.dijet_validation_set, seq_maxlen=config.seq_maxlen, batch_size=config.valid_batch_size, cyclic=True) valid_zjet_iter = get_data_iter( path=config.zjet_validation_set, seq_maxlen=config.seq_maxlen, batch_size=config.valid_batch_size, cyclic=True) steps_per_epoch = len(train_iter) total_step = config.num_epochs * steps_per_epoch if config.verbose: print("# of steps per one epoch: {}".format(steps_per_epoch)) print("Total step: {}".format(total_step)) ################################# # Build & Compile a model. ################################# x_kinematics_shape = train_iter.get_shape("x_kinematics", batch_shape=False) x_pid_shape = train_iter.get_shape("x_pid", batch_shape=False) _model = build_a_model(x_kinematics_shape, x_pid_shape) if config.multi_gpu: model = multi_gpu_model(_model, gpus=config.num_gpus) else: model = _model # TODO config should have these information. loss = 'binary_crossentropy' optimizer = optimizers.Adam(lr=config.lr) metric_list = ['accuracy'] model.compile( loss=loss, optimizer=optimizer, metrics=metric_list) if config.verbose: model.summary() ####################################### # ########################################### meter = Meter( name_list=["step", "lr", "train_loss", "dijet_loss", "zjet_loss", "train_acc", "dijet_acc", "zjet_acc"], dpath=log_dir.validation.path) ####################################### # Training with validation ####################################### start_message = "TRAINING START" print("$" * (len(start_message) + 4)) print("$ {} $".format(start_message)) print("$" * (len(start_message) + 4)) step = 0 for epoch in range(config.num_epochs): print("Epoch [{epoch}/{num_epochs}]".format(epoch=(epoch+1), num_epochs=config.num_epochs)) for train_batch in train_iter: ######################################### # Validation ################################################ if step % config.valid_freq == 0 or step % config.save_freq == 0: valid_dj_batch = valid_dijet_iter.next() valid_zj_batch = valid_zjet_iter.next() train_loss, train_acc = model.test_on_batch( x=[train_batch.x_kinematics, train_batch.x_pid], y=train_batch.y) dijet_loss, dijet_acc = model.test_on_batch( x=[valid_dj_batch.x_kinematics, valid_dj_batch.x_pid], y=valid_dj_batch.y) zjet_loss, zjet_acc = model.test_on_batch( x=[valid_zj_batch.x_kinematics, valid_zj_batch.x_pid], y=valid_zj_batch.y) print("Step [{step}/{total_step}]".format(step=step, total_step=total_step)) print(" Training:\n\tLoss {:.3f} | Acc. {:.3f}".format(train_loss, train_acc)) print(" Validation on Dijet\n\tLoss {:.3f} | Acc. {:.3f}".format(dijet_loss, dijet_acc)) print(" Validation on Z+jet\n\tLoss {:.3f} | Acc. {:.3f}".format(zjet_loss,zjet_acc)) # print(" LR: {:.5f}".format(K.get_value(model.optimizer.lr))) meter.append({ "step": step, "lr": K.get_value(model.optimizer.lr), "train_loss": train_loss, "dijet_loss": dijet_loss, "zjet_loss": zjet_loss, "train_acc": train_acc, "dijet_acc": dijet_acc, "zjet_acc": zjet_acc}) # Save model if (step != 0) and (step % config.save_freq == 0): filepath = os.path.join( log_dir.saved_models.path, "model_step-{step:06d}_loss-{loss:.3f}_acc-{acc:.3f}.h5".format( step=step, loss=dijet_loss, acc=dijet_acc)) _model.save(filepath) # Train on batch step += 1 model.train_on_batch( x=[train_batch.x_kinematics, train_batch.x_pid], y=train_batch.y) # new_lr = np.power(step, -0.5) #K.set_value(_model.optimizer.lr, new_lr) ############################### # On Epoch End ########################### ############################# # #############################3 filepath = os.path.join(log_dir.saved_models.path, "model_final.h5") _model.save(filepath) print("Training is over! :D") meter.add_plot( x="step", ys=[("train_loss", "Train/Dijet"), ("dijet_loss", "Validation/Dijet"), ("zjet_loss", "Validation/Z+jet")], title="Loss(CrossEntropy)", xlabel="Step", ylabel="Loss") meter.add_plot( x="step", ys=[("train_acc", "Train/Dijet"), ("dijet_acc", "Validation/Dijet"), ("zjet_acc", "Validation/Z+jet")], title="Accuracy", xlabel="Step", ylabel="Acc.") meter.finish() config.finish() return log_dir
def main(): ########################## # Argument Parsing ########################## parser = argparse.ArgumentParser() parser.add_argument("--logdir", dest="log_dir", type=str, default="./logs/untitled-{}".format( datetime.now().strftime("%y%m%d-%H%M%S"))) parser.add_argument("--num_gpus", default=len(get_available_gpus()), type=int) parser.add_argument("--multi-gpu", default=False, action='store_true', dest='multi_gpu') # Hyperparameters parser.add_argument("--optimizer", default="Adam", type=str) parser.add_argument("--lr", default=0.003, type=float) parser.add_argument( "--clipnorm", default=-1, type=float, help="if it is greater than 0, then graidient clipping is activated") parser.add_argument("--clipvalue", default=-1, type=float) parser.add_argument("--use-class-weight", dest="use_class_weight", default=False, action="store_true") parser.add_argument("--batch_size", default=128, type=int) parser.add_argument("--valid_batch_size", default=1024, type=int) # Frequencies parser.add_argument("--valid_freq", type=int, default=32) parser.add_argument("--save_freq", type=int, default=32) parser.add_argument("-v", "--verbose", action="store_true") # Project parameters parser.add_argument("--min-pt", dest="min_pt", default=100, type=int) # Model Archtecture parser.add_argument("--act", dest="activation", default="elu", type=str) args = parser.parse_args() ################### # ################### log_dir = Directory(path=args.log_dir) log_dir.mkdir("script") log_dir.mkdir("checkpoint") log_dir.mkdir("learning_curve") log_dir.mkdir("roc_curve") log_dir.mkdir("model_response") backup_scripts(log_dir.script.path) config = Config(log_dir.path, "w") config.append(args) config["hostname"] = os.environ["HOSTNAME"] ############################### # Load ################################# if os.environ["HOSTNAME"] == "cms05.sscc.uos.ac.kr": ckpt_dir = "/store/slowmoyang/QGJets/SJ-keras4hep/Dev-Composite" elif os.environ["HOSTNAME"] == "gate2": ckpt_dir = "/scratch/slowmoyang/QGJets/SJ-keras4hep/Dev-Composite" else: raise NotImplementedError cnn_path = os.path.join( ckpt_dir, "VanillaConvNet_epoch-67_loss-0.4987_acc-0.7659_auc-0.8422.hdf5") rnn_path = os.path.join( ckpt_dir, "RNNGatherEmbedding_weights_epoch-121_loss-0.4963_acc-0.7658_auc-0.8431.hdf5" ) cnn_custom_objects = get_cnn_custom_objects() rnn_custom_objects = get_rnn_custom_objects() custom_objects = {} custom_objects.update(cnn_custom_objects) custom_objects.update(rnn_custom_objects) cnn = load_model(cnn_path, custom_objects=cnn_custom_objects) cnn.summary() print("\n" * 5) rnn = load_model(rnn_path, custom_objects=rnn_custom_objects) rnn.summary() print("\n" * 5) ###################################### # Build ###################################### inputs = cnn.inputs + rnn.inputs # cnn_logits = cnn.get_layer("cnn_gap2d_0").output # rnn_logits = rnn.get_layer("rnn_dense_6").output cnn_softmax = cnn.get_layer("cnn_softmax_0").output rnn_softmax = rnn.get_layer("rnn_softmax_0").output # logits = Add()([cnn_logits, rnn_logits]) logits = Add()([cnn_softmax, rnn_softmax]) y_pred = Softmax()(logits) model = Model(inputs=inputs, outputs=y_pred) model.summary() ################################################ # Freeze ################################################## for each in model.layers: each.trainable = False ################################################### # #################################################### dset = get_dataset_paths(config.min_pt) config["fit_generator_input"] = { "x": ["x_img", "x_kin", "x_pid", "x_len"], "y": ["y"] } train_iter = get_data_iter(path=dset["training"], batch_size=config.batch_size, fit_generator_input=config.fit_generator_input, fit_generator_mode=True) test_iter = get_data_iter(path=dset["test"], batch_size=config.valid_batch_size, fit_generator_input=config.fit_generator_input, fit_generator_mode=False) ########################################### # Evaluation ############################################ train_iter.fit_generator_mode = False train_iter.cycle = False evaluate(model=model, train_iter=train_iter, test_iter=test_iter, log_dir=log_dir) config.save()
def main(): ########################## # Argument Parsing ########################## parser = argparse.ArgumentParser() parser.add_argument("--logdir", dest="log_dir", type=str, default="./logs/untitled-{}".format( datetime.now().strftime("%y%m%d-%H%M%S"))) parser.add_argument("--num_gpus", default=len(get_available_gpus()), type=int) parser.add_argument("--multi-gpu", default=False, action='store_true', dest='multi_gpu') # Hyperparameters parser.add_argument("--epoch", dest="epochs", default=100, type=int) parser.add_argument("--batch_size", default=128, type=int) parser.add_argument("--valid_batch_size", default=1024, type=int) parser.add_argument("--optimizer", default="Adam", type=str) parser.add_argument("--lr", default=0.003, type=float) parser.add_argument( "--clipnorm", default=-1, type=float, help="if it is greater than 0, then graidient clipping is activated") parser.add_argument("--clipvalue", default=-1, type=float) parser.add_argument("--use-class-weight", dest="use_class_weight", default=False, action="store_true") # Frequencies parser.add_argument("--valid_freq", type=int, default=32) parser.add_argument("--save_freq", type=int, default=32) parser.add_argument("-v", "--verbose", action="store_true") # Project parameters parser.add_argument("--min-pt", dest="min_pt", default=100, type=int) # Model Archtecture parser.add_argument("--act", dest="activation", default="elu", type=str) args = parser.parse_args() ################### # ################### log_dir = Directory(path=args.log_dir) log_dir.mkdir("script") log_dir.mkdir("checkpoint") log_dir.mkdir("learning_curve") log_dir.mkdir("roc_curve") log_dir.mkdir("model_response") backup_scripts(log_dir.script.path) config = Config(log_dir.path, "w") config.append(args) config["hostname"] = os.environ["HOSTNAME"] ############################### # Load ################################# if os.environ["HOSTNAME"] == "cms05.sscc.uos.ac.kr": ckpt_dir = "/store/slowmoyang/QGJets/SJ-keras4hep/Dev-Composite" elif os.environ["HOSTNAME"] == "gate2": ckpt_dir = "/scratch/slowmoyang/QGJets/SJ-keras4hep/Dev-Composite" else: raise NotImplementedError cnn_path = os.path.join( ckpt_dir, "VanillaConvNet_epoch-67_loss-0.4987_acc-0.7659_auc-0.8422.hdf5") rnn_path = os.path.join( ckpt_dir, "RNNGatherEmbedding_weights_epoch-121_loss-0.4963_acc-0.7658_auc-0.8431.hdf5" ) cnn_custom_objects = get_cnn_custom_objects() rnn_custom_objects = get_rnn_custom_objects() custom_objects = {} custom_objects.update(cnn_custom_objects) custom_objects.update(rnn_custom_objects) cnn = load_model(cnn_path, custom_objects=cnn_custom_objects) cnn.summary() print("\n" * 5) rnn = load_model(rnn_path, custom_objects=rnn_custom_objects) rnn.summary() print("\n" * 5) ###################################### # Build ###################################### inputs = cnn.inputs + rnn.inputs # cnn_last_hidden = cnn.get_layer("cnn_conv2d_3").output # rnn_last_hidden = rnn.get_layer("rnn_dense_5").output # cnn_flatten = Flatten()(cnn_last_hidden) # joint = Concatenate(axis=-1)([cnn_flatten, rnn_last_hidden]) cnn_last_hidden = cnn.get_layer("cnn_batch_norm_2").output rnn_last_hidden = rnn.get_layer("rnn_dense_5").output cnn_gap = GlobalAveragePooling2D()(cnn_last_hidden) cnn_flatten = Flatten()(cnn_gap) joint = Concatenate(axis=-1)([cnn_flatten, rnn_last_hidden]) joint = BatchNormalization(axis=-1, name="joint_batch_norm")(joint) joint = Dense(128)(joint) joint = Activation("relu")(joint) logits = Dense(2)(joint) y_pred = Softmax()(logits) model = Model(inputs=inputs, outputs=y_pred) model.summary() ################################################ # Freeze ################################################## for each in model.layers: if each.name.startswith("cnn") or each.name.startswith("rnn"): each.trainable = False ################################################### # #################################################### dset = get_dataset_paths(config.min_pt) config["fit_generator_input"] = { "x": ["x_img", "x_kin", "x_pid", "x_len"], "y": ["y"] } train_iter = get_data_iter(path=dset["training"], batch_size=config.batch_size, fit_generator_input=config.fit_generator_input, fit_generator_mode=True) valid_iter = get_data_iter(path=dset["validation"], batch_size=config.valid_batch_size, fit_generator_input=config.fit_generator_input, fit_generator_mode=True) test_iter = get_data_iter(path=dset["test"], batch_size=config.valid_batch_size, fit_generator_input=config.fit_generator_input, fit_generator_mode=False) if config.use_class_weight: class_weight = get_class_weight(train_iter) config["class_weight"] = list(class_weight) else: class_weight = None ###################################### # ####################################### loss = 'categorical_crossentropy' # TODO capsulisation optimizer_kwargs = {} if config.clipnorm > 0: optimizer_kwargs["clipnorm"] = config.clipnorm if config.clipvalue > 0: optimizer_kwargs["clipvalue"] = config.clipvalue optimizer = getattr(optimizers, config.optimizer)(lr=config.lr, **optimizer_kwargs) metric_list = ["accuracy", roc_auc] model.compile(loss=loss, optimizer=optimizer, metrics=metric_list) ########################################################################### # Callbacks ########################################################################### ckpt_format_str = "weights_epoch-{epoch:02d}_loss-{val_loss:.4f}_acc-{val_acc:.4f}_auc-{val_roc_auc:.4f}.hdf5" ckpt_path = log_dir.checkpoint.concat(ckpt_format_str) csv_log_path = log_dir.concat("log_file.csv") learning_curve = LearningCurve(directory=log_dir.learning_curve.path) learning_curve.book(x="step", y="roc_auc", best="max") learning_curve.book(x="step", y="acc", best="max") learning_curve.book(x="step", y="loss", best="min") callback_list = [ callbacks.ModelCheckpoint(filepath=ckpt_path), # callbacks.EarlyStopping(monitor="val_loss" , patience=5), callbacks.ReduceLROnPlateau(verbose=1), callbacks.CSVLogger(csv_log_path), learning_curve, ] ############################################################################ # Training ############################################################################ model.fit_generator(train_iter, steps_per_epoch=len(train_iter), epochs=config.epochs, validation_data=valid_iter, validation_steps=len(valid_iter), callbacks=callback_list, shuffle=True, class_weight=class_weight) print("Training is over! :D") del model ########################################### # Evaluation ############################################ train_iter.fit_generator_mode = False train_iter.cycle = False good_ckpt = find_good_checkpoint(log_dir.checkpoint.path, which={ "max": ["auc", "acc"], "min": ["loss"] }) for idx, each in enumerate(good_ckpt, 1): print("[{}/{}] {}".format(idx, len(good_ckpt), each)) K.clear_session() evaluate(custom_objects=custom_objects, checkpoint_path=each, train_iter=train_iter, test_iter=test_iter, log_dir=log_dir) config.save()
def evaluate(saved_model_path, step, log_dir): # TEST config = Config(log_dir.path, "READ") custom_objects = {} #get_custom_objects(config.model_type, config.model) model = load_model(saved_model_path, custom_objects=custom_objects) out_hist = OutHist(dpath=log_dir.output_histogram.path, step=step, dname_list=["train", "test_dijet", "test_zjet"]) ########################## # training data ########################### train_iter = get_data_iter(path=config.dijet_training_set, seq_maxlen=config.seq_maxlen, batch_size=1024, cyclic=False) for batch in train_iter: y_pred = model.predict_on_batch(batch.x) out_hist.fill(dname="train", y_true=batch.y, y_pred=y_pred) ############################# # Test on dijet dataset ######################## dijet_iter = get_data_iter(path=config.dijet_test_set, seq_maxlen=config.seq_maxlen, extra=["pt", "eta"], batch_size=1024, cyclic=False) roc_dijet = ROCMeter(dpath=log_dir.roc.path, step=step, title="Test on Dijet", prefix="dijet_") heatmap_subdir = os.path.join( log_dir.heatmap.path, os.path.basename(saved_model_path).split(".")[0]) if not os.path.exists(heatmap_subdir): os.mkdir(heatmap_subdir) heatmap = Heatmap(data_set=config.dijet_test_set, out_dir=heatmap_subdir) for batch in dijet_iter: y_pred = model.predict_on_batch(batch.x) roc_dijet.append(y_true=batch.y, y_pred=y_pred) out_hist.fill(dname="test_dijet", y_true=batch.y, y_pred=y_pred) heatmap.fill(y_true=batch.y, y_pred=y_pred, pt=batch["pt"], eta=batch["eta"]) roc_dijet.finish() heatmap.finish() ################################## # Test on Z+jet dataset ################################### zjet_iter = get_data_iter(path=config.zjet_test_set, seq_maxlen=config.seq_maxlen, batch_size=1024, cyclic=False) roc_zjet = ROCMeter(dpath=log_dir.roc.path, step=step, title="Test on Z+jet", prefix="zjet_") for batch in zjet_iter: y_pred = model.predict_on_batch(batch.x) roc_zjet.append(y_true=batch.y, y_pred=y_pred) out_hist.fill("test_zjet", y_true=batch.y, y_pred=y_pred) roc_zjet.finish() out_hist.finish()