class Trainer: def __init__(self, config: Config): self.config = config self.model = None self.dataset = None self.optimizer = None self.visualize = Visualize(config) def start(self): self.model = self.load_model() self.training() def training(self): tc = self.config.trainer self.compile_model() self.dataset = self.load_dataset() self.fit(x=self.dataset[0][0], y=self.dataset[0][1], epochs=tc.epoch, batch_size=tc.batch_size, validation_data=self.dataset[1], is_visualize=tc.is_visualize, is_accuracy=tc.is_accuracy) self.evaluate(self.dataset[2][0], self.dataset[2][1]) self.save_model() self.save_result() def compile_model(self): self.optimizer = optimizers.get(self.config) self.loss = mean_square_error self.accuracy = accuracy def fit(self, x=None, y=None, epochs=1, batch_size=1, validation_data=None, is_shuffle=True, is_visualize=False, is_accuracy=False): if x is None or y is None: raise ValueError("There is no fitting data") n_train = len(x) self.losses = [] if validation_data is not None: self.losses_val = [] if is_accuracy: self.accuracies = [] if validation_data is not None and is_accuracy: self.accuracies_val = [] t = cp.asnumpy(self.model.t) logger.info("training start") cp.cuda.Stream.null.synchronize() start_time = time.time() for epoch in range(epochs): if is_shuffle: x, y = shuffle(x, y) error = 0. if is_accuracy: accuracy = 0. with tqdm(range(0, n_train, batch_size), desc="[Epoch: {}]".format(epoch+1)) as pbar: for i, ch in enumerate(pbar): x_bs = cp.array(x[i:i+batch_size]) y_bs = cp.array(y[i:i+batch_size]) self.model.params = self.optimizer(self.model.params, self.model.gradient(x_bs, y_bs)) y_pred = self.model(x_bs) error += self.loss(y_pred, y_bs) * len(y_bs) if is_accuracy: accuracy += self.accuracy(y_pred, y_bs) * len(y_bs) error /= n_train self.losses.append(error) if validation_data is None: if not is_accuracy: message = "Epoch:{} Training loss: {:.5f}".format(epoch+1, error) if is_visualize: params = tuple(cp.asnumpy(param) for param in self.model.params) self.visualize.plot_realtime(t, params, [self.losses]) else: accuracy /= n_train self.accuracies.append(accuracy) message = "Epoch:{} Training loss:{:.5f} Training accuracy:{:.5f}".format(epoch+1, error, accuracy) if is_visualize: params = tuple(cp.asnumpy(param) for param in self.model.params) self.visualize.plot_realtime(t, params, [self.losses], [self.accuracies]) else: error_val = 0. if is_accuracy: accuracy_val = 0. for i in range(0, len(validation_data[0]), batch_size): x_bs = cp.array(validation_data[0][i:i+batch_size]) y_bs = cp.array(validation_data[1][i:i+batch_size]) y_pred = self.model(x_bs) error_val += self.loss(y_pred, y_bs) * len(y_bs) if is_accuracy: accuracy_val += self.accuracy(y_pred, y_bs) * len(y_bs) error_val /= len(validation_data[0]) self.losses_val.append(error_val) if not is_accuracy: message = "Epoch:{} Training loss:{:.5f} Validation loss:{:.5f}".format(epoch+1, error, error_val) if is_visualize: params = tuple(cp.asnumpy(param) for param in self.model.params) self.visualize.plot_realtime(t, params, [self.losses, self.losses_val]) else: accuracy /= n_train self.accuracies.append(accuracy) accuracy_val /= len(validation_data[0]) self.accuracies_val.append(accuracy_val) message = "Epoch:{} Training loss:{:.5f} Validation loss:{:.5f} Training accuracy:{:.5f} Validation accuracy:{:.5f}".format(epoch+1, error, error_val, accuracy, accuracy_val) if is_visualize: params = tuple(cp.asnumpy(param) for param in self.model.params) self.visualize.plot_realtime(t, params, [self.losses, self.losses_val], [self.accuracies, self.accuracies_val]) logger.info(message) cp.cuda.Stream.null.synchronize() interval = time.time() - start_time logger.info("end of training") logger.info("time: {}".format(interval)) logger.info(message) def evaluate(self, x, y): batch_size = self.config.trainer.batch_size error = 0 is_accuracy = self.config.trainer.is_accuracy if is_accuracy: accuracy = 0 for i in range(0, len(x), batch_size): x_bs = cp.array(x[i:i+batch_size]) y_bs = cp.array(y[i:i+batch_size]) y_pred = self.model(x_bs) error += self.loss(y_pred, y_bs) * len(y_bs) if is_accuracy: accuracy += self.accuracy(y_pred, y_bs) * len(y_bs) error /= len(x) if is_accuracy: accuracy /= len(x) message = "Test loss:{} Test accuracy:{}".format(error, accuracy) else: message = "Test loss:{}".format(error) logger.info(message) def load_model(self): from model import NeuralODEModel model = NeuralODEModel(self.config) model.load(self.config.resource.model_path) return model def save_model(self): rc = self.config.resource model_id = datetime.now().strftime("%Y%m%d-%H%M%S") model_dir = os.path.join(rc.model_dir, "model_{}".format(model_id)) os.makedirs(model_dir, exist_ok=True) config_path = os.path.join(model_dir, "parameter.conf") model_path = os.path.join(model_dir, "model.json") self.model.save(config_path, model_path) def load_dataset(self): data_path = self.config.resource.data_path if os.path.exists(data_path): logger.debug("loading data from {}".format(data_path)) with open(data_path, "rt") as f: datasets = json.load(f) x = datasets.get("Input") y = datasets.get("Output") if x is None or y is None: raise TypeError("Dataset does not exists in {}".format(data_path)) if len(x[0]) != self.config.model.dim_in: raise ValueError("Input dimensions in config and dataset are not equal: {} != {}".format(self.config.model.dim_in, len(x[0]))) if len(y[0]) != self.config.model.dim_out: raise ValueError("Output dimensions in config and dataset are not equal: {} != {}".format(self.config.model.dim_out, len(y[0]))) x_train, x_test, y_train, y_test = train_test_split(np.array(x, dtype=np.float32), np.array(y, dtype=np.float32), test_size=self.config.trainer.test_size) x_train, x_val, y_train, y_val = train_test_split(x_train, y_train, test_size=self.config.trainer.validation_size) train = (x_train, y_train) validation = (x_val, y_val) test = (x_test, y_test) return (train, validation, test) else: raise FileNotFoundError("Dataset file can not loaded!") def save_result(self): rc = self.config.resource tc = self.config.trainer result_id = datetime.now().strftime("%Y%m%d-%H%M%S") result_dir = os.path.join(rc.result_dir, "result_train_{}".format(result_id)) os.makedirs(result_dir, exist_ok=True) result_path = os.path.join(result_dir, "learning_curve.csv") e = [i for i in range(1, tc.epoch+1)] try: self.visualize.save_plot_loss([self.losses, self.losses_val], xlabel="Epoch", ylabel="Loss", title="Loss", save_file=os.path.join(result_dir, "loss.png")) if tc.is_accuracy: result_csv = [e, self.losses, self.losses_val, self.accuracies, self.accuracies_val] columns = ["epoch", "loss_train", "loss_val", "accuracy_train", "accuracy_val"] self.visualize.save_plot_accuracy([self.accuracies, self.accuracies_val], xlabel="Epoch", ylabel="Accuracy", title="Accuracy", save_file=os.path.join(result_dir, "accuracy.png")) else: result_csv = [e, self.losses, self.losses_val] columns = ["epoch", "loss_train", "loss_val"] except AttributeError: self.visualize.save_plot_loss([self.losses], xlabel="Epoch", ylabel="Loss", title="Loss", save_file=os.path.join(result_dir, "loss.png")) if tc.is_accuracy: result_csv = [e, self.losses, self.accuracies] columns = ["epoch", "loss_train", "accuracy_train"] self.visualize.save_plot_accuracy([self.accuracies], xlabel="Epoch", ylabel="Accuracy", title="Accuracy", save_file=os.path.join(result_dir, "accuracy.png")) else: result_csv = [e, self.losses] columns = ["epoch", "loss_train"] logger.debug("save result to {}".format(result_path)) with open(result_path, "wt") as f: writer = csv.writer(f) writer.writerow(columns) writer.writerows(list(zip(*result_csv))) if tc.is_visualize: save_params_path = [os.path.join(result_dir, "alpha.png"), os.path.join(result_dir, "beta.png"), os.path.join(result_dir, "gamma.png"), os.path.join(result_dir, "params.png")] self.visualize.save_plot_params(cp.asnumpy(self.model.t), tuple(cp.asnumpy(param) for param in self.model.params), save_file=save_params_path)
class Trainer: def __init__(self, config: Config): self.config = config self.model = None self.dataset = None if config.trainer.is_visualize: self.visualize = Visualize() def start(self): self.model = self.load_model() self.training() def training(self): tc = self.config.trainer self.compile_model() self.dataset = self.load_dataset() self.fit(self.dataset[0], self.dataset[1], epochs=tc.epoch, batch_size=tc.batch_size, is_visualize=tc.is_visualize) self.save_model() self.save_result() def compile_model(self): self.optimizer_generator = optimizers.get(self.config, "generator") self.optimizer_discriminator = optimizers.get(self.config, "discriminator") self.loss = Loss(self.config).get(self.config.trainer.loss_type) self.accuracy = accuracy def fit(self, x=None, y=None, epochs=1, batch_size=1, is_visualize=False): if x is None or y is None: raise ValueError("There is no fitting data") n_train = len(x) self.losses_generator = [] self.losses_discriminator = [] self.losses_generator_reg = [] self.losses_discriminator_reg = [] self.accuracies = [] logger.info("training start") start_time = time.time() for epoch in range(epochs): x, y = shuffle(x, y) #Discriminatorの学習 noise = np.random.normal(loc=0., scale=0.5, size=(n_train, self.config.model.dim_noise)) x_fake = self.model.generator(np.hstack([noise, y])) y_fake = np.hstack( [y, np.zeros(shape=(n_train, 1), dtype=np.float32)]) y_real = np.hstack( [y, np.ones(shape=(n_train, 1), dtype=np.float32)]) x_train = np.concatenate([x, x_fake]) y_train = np.concatenate([y_real, y_fake]) x_train, y_train = shuffle(x_train, y_train) with tqdm(range(0, n_train * 2, batch_size), desc="[Epoch: {} Discriminator]".format(epoch + 1)) as pbar: for i, ch in enumerate(pbar): n_data = len(x_train[i:i + batch_size]) y_pred = self.model.discriminator(x_train[i:i + batch_size]) aT = np.zeros_like(x[i:i + batch_size]) bT = ( y_train[i:i + batch_size] / y_pred ) / n_data if self.config.trainer.loss_type == "crossentropy" else ( y_pred - y_train[i:i + batch_size]) / n_data self.model.discriminator.params = self.optimizer_discriminator( self.model.discriminator.params, self.model.discriminator.gradient(np.hstack([aT, bT]))) y_pred = self.model.discriminator(x_train) error = self.loss(y_pred, y_train, self.model.discriminator.params) self.losses_discriminator.append(error[0]) self.losses_discriminator_reg.append(error[1]) accuracy = self.accuracy(y_pred[:, :-1], y_train[:, :-1]) self.accuracies.append(accuracy) message1 = "Discriminator Epoch: {} Loss: {} Loss_reg: {} accuracy: {}".format( epoch + 1, error[0], error[1], accuracy) logger.info(message1) #Generatorの学習 noise = np.random.normal(loc=0., scale=0.5, size=(n_train, self.config.model.dim_noise)) x_train = np.hstack([noise, y]) y_train = np.hstack( [y, np.ones(shape=(n_train, 1), dtype=np.float32)]) x_train, y_train = shuffle(x_train, y_train) with tqdm(range(0, n_train, batch_size), desc="[Epoch: {} Generator]".format(epoch + 1)) as pbar: for i, ch in enumerate(pbar): n_data = len(x_train[i:i + batch_size]) y_pred = self.model(x_train[i:i + batch_size]) aT = np.zeros(shape=(n_data, self.model.discriminator.dim_in), dtype=np.float32) bT = ( y_train[i:i + batch_size] / y_pred ) / n_data if self.config.trainer.loss_type == "crossentropy" else ( y_pred - y_train[i:i + batch_size]) / n_data self.model.discriminator.gradient(np.hstack([aT, bT])) aT = np.zeros_like(x_train[i:i + batch_size]) bT = self.model.discriminator.a1[0] self.model.generator.params = self.optimizer_generator( self.model.generator.params, self.model.generator.gradient(np.hstack([aT, bT]))) y_pred = self.model(x_train) error = self.loss(y_pred, y_train, self.model.generator.params) self.losses_generator.append(error[0]) self.losses_generator_reg.append(error[1]) message2 = "Generator Epoch: {} Loss: {} Loss_reg: {}".format( epoch + 1, error[0], error[1]) logger.info(message2) if is_visualize: noise = np.random.normal(loc=0., scale=0.5, size=(n_train, self.config.model.dim_noise)) generated_points = self.model.generator(np.hstack([noise, y])) self.visualize.plot_realtime( [self.losses_generator, self.losses_generator_reg], [self.losses_discriminator, self.losses_discriminator_reg], self.accuracies, x, y, generated_points, y) interval = time.time() - start_time logger.info("end of training") logger.info("time: {}".format(interval)) logger.info(message1) logger.info(message2) def load_model(self): from model import ODEGANModel model = ODEGANModel(self.config) model.load(self.config.resource.generator_model_path, self.config.resource.discriminator_model_path) return model def save_model(self): rc = self.config.resource model_id = datetime.now().strftime("%Y%m%d-%H%M%S") model_dir = os.path.join(rc.model_dir, "model_{}".format(model_id)) os.makedirs(model_dir, exist_ok=True) config_path = os.path.join(model_dir, "parameter.conf") generator_model_path = os.path.join(model_dir, "generator.json") discriminator_model_path = os.path.join(model_dir, "discriminator.json") self.model.save(config_path, generator_model_path, discriminator_model_path) def load_dataset(self): data_path = self.config.resource.data_path if os.path.exists(data_path): logger.debug("loading data from {}".format(data_path)) with open(data_path, "rt") as f: datasets = json.load(f) x = datasets.get("Input") y = datasets.get("Output") if x is None or y is None: raise TypeError( "Dataset does not exists in {}".format(data_path)) if len(x[0]) != self.config.model.dim_in: raise ValueError( "Input dimensions in config and dataset are not equal: {} != {}" .format(self.config.model.dim_in, len(x[0]))) if len(y[0]) != self.config.model.dim_out: raise ValueError( "Output dimensions in config and dataset are not equal: {} != {}" .format(self.config.model.dim_out, len(y[0]))) return (np.array(x), np.array(y)) else: raise FileNotFoundError("Dataset file can not loaded!") def save_result(self): rc = self.config.resource tc = self.config.trainer result_id = datetime.now().strftime("%Y%m%d-%H%M%S") result_dir = os.path.join(rc.result_dir, "result_train_{}".format(result_id)) os.makedirs(result_dir, exist_ok=True) result_path = os.path.join(result_dir, "learning_curve.csv") e = [i for i in range(1, tc.epoch + 1)] self.visualize.save_plot_loss( [self.losses_generator, self.losses_generator_reg], xlabel='Epoch', ylabel='Loss', title='Loss of generator', save_file=os.path.join(result_dir, "loss_generator.png")) self.visualize.save_plot_loss( [self.losses_discriminator, self.losses_discriminator_reg], xlabel='Epoch', ylabel='Loss', title='Loss of discriminator', save_file=os.path.join(result_dir, "loss_discriminator.png")) self.visualize.save_plot_accuracy(self.accuracies, xlabel='Epoch', ylabel='Accuracy', title='Accuracy', save_file=os.path.join( result_dir, "accuracy.png")) result_csv = [ e, self.losses_generator, self.losses_generator_reg, self.losses_discriminator, self.losses_discriminator_reg, self.accuracies ] columns = [ 'epoch', 'loss_generator', 'loss_generator_reg', 'loss_discriminator', 'loss_discriminator_reg', 'accuracy' ] logger.debug("save result to {}".format(result_path)) with open(result_path, "wt") as f: writer = csv.writer(f) writer.writerow(columns) writer.writerows(list(zip(*result_csv))) result_generator_dir = os.path.join(result_dir, "generator") result_discriminator_dir = os.path.join(result_dir, "discriminator") os.makedirs(result_generator_dir, exist_ok=True) os.makedirs(result_discriminator_dir, exist_ok=True) save_params_path = [ os.path.join(result_generator_dir, "alpha_x.png"), os.path.join(result_generator_dir, "beta_x.png"), os.path.join(result_generator_dir, "gamma_x.png"), os.path.join(result_generator_dir, "A_x.png"), os.path.join(result_generator_dir, "alpha_y.png"), os.path.join(result_generator_dir, "beta_y.png"), os.path.join(result_generator_dir, "gamma_y.png"), os.path.join(result_generator_dir, "A_y.png"), os.path.join(result_generator_dir, "params.png") ] self.visualize.save_plot_params(self.model.generator.t, self.model.generator.params, save_file=save_params_path) save_params_path = [ os.path.join(result_discriminator_dir, "alpha_x.png"), os.path.join(result_discriminator_dir, "beta_x.png"), os.path.join(result_discriminator_dir, "gamma_x.png"), os.path.join(result_discriminator_dir, "A_x.png"), os.path.join(result_discriminator_dir, "alpha_y.png"), os.path.join(result_discriminator_dir, "beta_y.png"), os.path.join(result_discriminator_dir, "gamma_y.png"), os.path.join(result_discriminator_dir, "A_y.png"), os.path.join(result_discriminator_dir, "params.png") ] self.visualize.save_plot_params(self.model.discriminator.t, self.model.discriminator.params, save_file=save_params_path)