def __init__(self,
loss,
regularization=None,
learning_rate=0.01,
reg_param=0.05):
self.learning_rate = learning_rate
# Select regularizer
if regularization == 'l1':
regularizer = L1Regularization(reg_param)
elif regularization == 'l2':
regularizer = L2Regularization(reg_param)
elif regularization is None:
regularizer = None
else:
raise ValueError(
'Regularizer {} is not defined'.format(regularization))
# Select loss function
if loss == 'hinge':
self.loss = HingeLoss(regularizer)
elif loss == 'squared':
self.loss = SquaredLoss(regularizer)
else:
raise ValueError('Loss function {} is not defined'.format(loss))
self.model = None
# storage for frq
self.losses = []
self.predictions = []
self.convergence = False
self.iterations = 0
def Model_Linear_Gelu_1_HingeLoss():
name = '1_Gelu_HingeLoss'
model = Network()
model.add(Linear('fc1', 784, 256, 0.01))
model.add(Gelu('a1'))
model.add(Linear('fc2', 256, 10, 0.01))
loss = HingeLoss(name='loss')
return name, model, loss
def Model_Linear_Gelu_2_HingeLoss():
name = '2_Gelu_HingeLoss'
model = Network()
model.add(Linear('fc1', 784, 441, 0.01))
model.add(Gelu('a1'))
model.add(Linear('fc2', 441, 196, 0.01))
model.add(Gelu('a2'))
model.add(Linear('fc3', 196, 10, 0.01))
loss = HingeLoss(name='loss')
return name, model, loss
from layers import Relu, Sigmoid, Linear, Gelu
from loss import EuclideanLoss, SoftmaxCrossEntropyLoss, HingeLoss
from solve_net import train_net, test_net
from load_data import load_mnist_2d
import time
NAME = "1layersSigmoid.Hinge"
train_data, test_data, train_label, test_label = load_mnist_2d('data')
# Your model defintion here
# You should explore different model architecture
model = Network()
model.add(Linear('fc1', 784, 256, 0.01, 'sigmoid', 'act1'))
model.add(Linear('fc2', 256, 10, 0.01, None, 'act2'))
loss = HingeLoss(name='loss')
# Training configuration
# You should adjust these hyperparameters
# NOTE: one iteration means model forward-backwards one batch of samples.
# one epoch means model has gone through all the training samples.
# 'disp_freq' denotes number of iterations in one epoch to display information.
config = {
'learning_rate': 0.1,
'weight_decay': 0.0005,
'momentum': 0.9,
'batch_size': 500,
'max_epoch': 50,
'disp_freq': 50,
'test_epoch': 5
plt.figure()
plt.plot(epochs, train, label="Training")
plt.plot(epochs, test, label="Testing")
plt.xlabel("Epochs")
plt.ylabel(label)
plt.legend()
plt.savefig(file)
# Your model defintion here
# You should explore different model architecture
from mlp2 import model
# loss = SoftmaxCrossEntropyLoss(name="CE")
# loss = EuclideanLoss(name='MSE')
loss = HingeLoss(name="Hinge5", threshold=5)
model_name = str(model) + "_" + loss.name
print(model_name)
# Training configuration
# You should adjust these hyperparameters
# NOTE: one iteration means model forward-backwards one batch of samples.
# one epoch means model has gone through all the training samples.
# 'disp_freq' denotes number of iterations in one epoch to display information.
config = {
'learning_rate': 0.05,
'weight_decay': 0.0,
'momentum': 0.0,
'batch_size': 100,
'max_epoch': 100,
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
"--config",
"-c",
default="configs/default.yml",
help="training configuration file",
)
args = parser.parse_args()
# parse config yaml
with open(args.config) as f:
configs = yaml.load(f, Loader=yaml.FullLoader)
configs["config_path"] = args.config
# fix seed
fix_seed(configs["seed"])
# initialize logger
log_path = Path(configs["log_dir"]) / configs["experiment_name"]
tb_path = Path(configs["tensorboard_dir"]) / configs["experiment_name"]
logger = Logger(log_path, tb_path)
logger.debug("(experiment)")
logger.debug(f"name: {configs['experiment_name']}", 1)
logger.debug(f"directory {configs['log_dir']}", 1)
logger.debug(f"tensorboard: {configs['tensorboard_dir']}", 1)
logger.debug(f"geometric_info: {configs['geometric_info']}", 1)
logger.debug(f"log_interval: {configs['log_interval']}", 1)
logger.debug(f"log_samples_interval: {configs['log_samples_interval']}", 1)
logger.debug(f"snapshot_interval: {configs['snapshot_interval']}", 1)
logger.debug(f"evaluation_interval: {configs['evaluation_interval']}", 1)
# loss
loss: Loss
if configs["loss"] == "adversarial-loss":
loss = AdversarialLoss()
elif configs["loss"] == "hinge-loss":
loss = HingeLoss()
else:
logger.error(f"Specified loss is not supported {configs['loss']}")
sys.exit(1)
logger.debug(f"loss: {configs['loss']}", 1)
# prepare dataset
dataset = VideoDataset(
configs["dataset"]["name"],
Path(configs["dataset"]["path"]),
eval(f'preprocess_{configs["dataset"]["name"]}_dataset'),
configs["video_length"],
configs["image_size"],
configs["dataset"]["number_limit"],
geometric_info=configs["geometric_info"]["name"],
)
dataloader = VideoDataLoader(
dataset,
batch_size=configs["batchsize"],
num_workers=configs["dataset"]["n_workers"],
shuffle=True,
drop_last=True,
pin_memory=True,
worker_init_fn=_worker_init_fn,
)
logger.debug("(dataset)")
logger.debug(f"name: {dataset.name}", 1)
logger.debug(f"size: {len(dataset)}", 1)
logger.debug(f"batchsize: {dataloader.batch_size}", 1)
logger.debug(f"workers: {dataloader.num_workers}", 1)
# prepare models
ggen = GeometricVideoGenerator(
configs["ggen"]["dim_z_content"],
configs["ggen"]["dim_z_motion"],
configs["geometric_info"]["channel"],
configs["geometric_info"]["name"],
configs["ggen"]["ngf"],
configs["video_length"],
)
cgen = ColorVideoGenerator(
ggen.channel,
configs["cgen"]["dim_z_color"],
configs["geometric_info"]["name"],
configs["cgen"]["ngf"],
configs["video_length"],
)
idis = ImageDiscriminator(
ggen.channel,
cgen.channel,
configs["idis"]["use_noise"],
configs["idis"]["noise_sigma"],
configs["idis"]["ndf"],
)
vdis = VideoDiscriminator(
ggen.channel,
cgen.channel,
configs["vdis"]["use_noise"],
configs["vdis"]["noise_sigma"],
configs["vdis"]["ndf"],
)
gdis = GradientDiscriminator(
ggen.channel,
cgen.channel,
configs["gdis"]["use_noise"],
configs["gdis"]["noise_sigma"],
configs["gdis"]["ndf"],
)
models = {
"ggen": ggen,
"cgen": cgen,
"idis": idis,
"vdis": vdis,
"gdis": gdis
}
logger.debug("(models)")
for m in models.values():
logger.debug(str(m), 1)
# init weights
for m in models.values():
m.apply(util.init_weights)
# optimizers
logger.debug("(optimizers)")
optimizers = {}
for name, model in models.items():
lr = configs[name]["optimizer"]["lr"]
betas = (0.5, 0.999)
decay = configs[name]["optimizer"]["decay"]
optimizers[name] = optim.Adam(model.parameters(),
lr=lr,
betas=betas,
weight_decay=decay)
logger.debug(
json.dumps(
{name: {
"betas": betas,
"lr": lr,
"weight_decay": decay
}}), 1)
# start training
trainer = Trainer(dataloader, logger, models, optimizers, loss, configs)
trainer.train()