def __init__(self, loss, regularization=None, learning_rate=0.01, reg_param=0.05): self.learning_rate = learning_rate # Select regularizer if regularization == 'l1': regularizer = L1Regularization(reg_param) elif regularization == 'l2': regularizer = L2Regularization(reg_param) elif regularization is None: regularizer = None else: raise ValueError( 'Regularizer {} is not defined'.format(regularization)) # Select loss function if loss == 'hinge': self.loss = HingeLoss(regularizer) elif loss == 'squared': self.loss = SquaredLoss(regularizer) else: raise ValueError('Loss function {} is not defined'.format(loss)) self.model = None # storage for frq self.losses = [] self.predictions = [] self.convergence = False self.iterations = 0
def Model_Linear_Gelu_1_HingeLoss(): name = '1_Gelu_HingeLoss' model = Network() model.add(Linear('fc1', 784, 256, 0.01)) model.add(Gelu('a1')) model.add(Linear('fc2', 256, 10, 0.01)) loss = HingeLoss(name='loss') return name, model, loss
def Model_Linear_Gelu_2_HingeLoss(): name = '2_Gelu_HingeLoss' model = Network() model.add(Linear('fc1', 784, 441, 0.01)) model.add(Gelu('a1')) model.add(Linear('fc2', 441, 196, 0.01)) model.add(Gelu('a2')) model.add(Linear('fc3', 196, 10, 0.01)) loss = HingeLoss(name='loss') return name, model, loss
from layers import Relu, Sigmoid, Linear, Gelu from loss import EuclideanLoss, SoftmaxCrossEntropyLoss, HingeLoss from solve_net import train_net, test_net from load_data import load_mnist_2d import time NAME = "1layersSigmoid.Hinge" train_data, test_data, train_label, test_label = load_mnist_2d('data') # Your model defintion here # You should explore different model architecture model = Network() model.add(Linear('fc1', 784, 256, 0.01, 'sigmoid', 'act1')) model.add(Linear('fc2', 256, 10, 0.01, None, 'act2')) loss = HingeLoss(name='loss') # Training configuration # You should adjust these hyperparameters # NOTE: one iteration means model forward-backwards one batch of samples. # one epoch means model has gone through all the training samples. # 'disp_freq' denotes number of iterations in one epoch to display information. config = { 'learning_rate': 0.1, 'weight_decay': 0.0005, 'momentum': 0.9, 'batch_size': 500, 'max_epoch': 50, 'disp_freq': 50, 'test_epoch': 5
plt.figure() plt.plot(epochs, train, label="Training") plt.plot(epochs, test, label="Testing") plt.xlabel("Epochs") plt.ylabel(label) plt.legend() plt.savefig(file) # Your model defintion here # You should explore different model architecture from mlp2 import model # loss = SoftmaxCrossEntropyLoss(name="CE") # loss = EuclideanLoss(name='MSE') loss = HingeLoss(name="Hinge5", threshold=5) model_name = str(model) + "_" + loss.name print(model_name) # Training configuration # You should adjust these hyperparameters # NOTE: one iteration means model forward-backwards one batch of samples. # one epoch means model has gone through all the training samples. # 'disp_freq' denotes number of iterations in one epoch to display information. config = { 'learning_rate': 0.05, 'weight_decay': 0.0, 'momentum': 0.0, 'batch_size': 100, 'max_epoch': 100,
def main(): parser = argparse.ArgumentParser() parser.add_argument( "--config", "-c", default="configs/default.yml", help="training configuration file", ) args = parser.parse_args() # parse config yaml with open(args.config) as f: configs = yaml.load(f, Loader=yaml.FullLoader) configs["config_path"] = args.config # fix seed fix_seed(configs["seed"]) # initialize logger log_path = Path(configs["log_dir"]) / configs["experiment_name"] tb_path = Path(configs["tensorboard_dir"]) / configs["experiment_name"] logger = Logger(log_path, tb_path) logger.debug("(experiment)") logger.debug(f"name: {configs['experiment_name']}", 1) logger.debug(f"directory {configs['log_dir']}", 1) logger.debug(f"tensorboard: {configs['tensorboard_dir']}", 1) logger.debug(f"geometric_info: {configs['geometric_info']}", 1) logger.debug(f"log_interval: {configs['log_interval']}", 1) logger.debug(f"log_samples_interval: {configs['log_samples_interval']}", 1) logger.debug(f"snapshot_interval: {configs['snapshot_interval']}", 1) logger.debug(f"evaluation_interval: {configs['evaluation_interval']}", 1) # loss loss: Loss if configs["loss"] == "adversarial-loss": loss = AdversarialLoss() elif configs["loss"] == "hinge-loss": loss = HingeLoss() else: logger.error(f"Specified loss is not supported {configs['loss']}") sys.exit(1) logger.debug(f"loss: {configs['loss']}", 1) # prepare dataset dataset = VideoDataset( configs["dataset"]["name"], Path(configs["dataset"]["path"]), eval(f'preprocess_{configs["dataset"]["name"]}_dataset'), configs["video_length"], configs["image_size"], configs["dataset"]["number_limit"], geometric_info=configs["geometric_info"]["name"], ) dataloader = VideoDataLoader( dataset, batch_size=configs["batchsize"], num_workers=configs["dataset"]["n_workers"], shuffle=True, drop_last=True, pin_memory=True, worker_init_fn=_worker_init_fn, ) logger.debug("(dataset)") logger.debug(f"name: {dataset.name}", 1) logger.debug(f"size: {len(dataset)}", 1) logger.debug(f"batchsize: {dataloader.batch_size}", 1) logger.debug(f"workers: {dataloader.num_workers}", 1) # prepare models ggen = GeometricVideoGenerator( configs["ggen"]["dim_z_content"], configs["ggen"]["dim_z_motion"], configs["geometric_info"]["channel"], configs["geometric_info"]["name"], configs["ggen"]["ngf"], configs["video_length"], ) cgen = ColorVideoGenerator( ggen.channel, configs["cgen"]["dim_z_color"], configs["geometric_info"]["name"], configs["cgen"]["ngf"], configs["video_length"], ) idis = ImageDiscriminator( ggen.channel, cgen.channel, configs["idis"]["use_noise"], configs["idis"]["noise_sigma"], configs["idis"]["ndf"], ) vdis = VideoDiscriminator( ggen.channel, cgen.channel, configs["vdis"]["use_noise"], configs["vdis"]["noise_sigma"], configs["vdis"]["ndf"], ) gdis = GradientDiscriminator( ggen.channel, cgen.channel, configs["gdis"]["use_noise"], configs["gdis"]["noise_sigma"], configs["gdis"]["ndf"], ) models = { "ggen": ggen, "cgen": cgen, "idis": idis, "vdis": vdis, "gdis": gdis } logger.debug("(models)") for m in models.values(): logger.debug(str(m), 1) # init weights for m in models.values(): m.apply(util.init_weights) # optimizers logger.debug("(optimizers)") optimizers = {} for name, model in models.items(): lr = configs[name]["optimizer"]["lr"] betas = (0.5, 0.999) decay = configs[name]["optimizer"]["decay"] optimizers[name] = optim.Adam(model.parameters(), lr=lr, betas=betas, weight_decay=decay) logger.debug( json.dumps( {name: { "betas": betas, "lr": lr, "weight_decay": decay }}), 1) # start training trainer = Trainer(dataloader, logger, models, optimizers, loss, configs) trainer.train()