def init(self):
# Print losses and accuracy to screen
tracker.set_scalar('loss.*', True)
tracker.set_scalar('accuracy.*', True)
# We need to set the metrics to calculate them for the epoch for training and validation
self.state_modules = [self.accuracy]
def run(self):
tracker.set_queue('train.loss', is_print=True)
tracker.set_scalar('valid.loss', is_print=True)
for s in self.training_loop:
self.train()
if self.training_loop.is_interval(self.valid_internal):
self.validate()
def __init__(self, configs: Configs):
self.device = torch.device('cpu')
if torch.cuda.is_available():
self.device = torch.device('cuda:0')
self.dataset = TinyShakespeareDataset(configs.seq_len)
self.dataloader = DataLoader(self.dataset,
batch_size=configs.batch_size,
collate_fn=transpose_batch,
shuffle=True)
if configs.glu_variant == 'GLU':
ffn = FeedForward(configs.d_model, configs.d_ff, configs.dropout,
nn.Sigmoid(), True, False, False, False)
elif configs.glu_variant == 'Bilinear':
ffn = FeedForward(configs.d_model, configs.d_ff, configs.dropout,
nn.Identity(), True, False, False, False)
elif configs.glu_variant == 'ReGLU':
ffn = FeedForward(configs.d_model, configs.d_ff, configs.dropout,
nn.ReLU(), True, False, False, False)
elif configs.glu_variant == 'GEGLU':
ffn = FeedForward(configs.d_model, configs.d_ff, configs.dropout,
nn.GELU(), True, False, False, False)
elif configs.glu_variant == 'SwiGLU':
ffn = FeedForward(configs.d_model, configs.d_ff, configs.dropout,
nn.SiLU(), True, False, False, False)
elif configs.glu_variant == 'ReLU':
ffn = FeedForward(configs.d_model, configs.d_ff, configs.dropout,
nn.ReLU())
elif configs.glu_variant == 'GELU':
ffn = FeedForward(configs.d_model, configs.d_ff, configs.dropout,
nn.GELU())
else:
raise ValueError(f'Unknown variant {configs.glu_variant}')
n_chars = len(self.dataset.stoi)
self.model = AutoregressiveModel(
EmbeddingsWithPositionalEncoding(configs.d_model, n_chars),
Encoder(
TransformerLayer(d_model=configs.d_model,
self_attn=MultiHeadAttention(
configs.n_heads, configs.d_model,
configs.dropout),
src_attn=None,
feed_forward=ffn,
dropout_prob=configs.dropout),
configs.n_layers), nn.Linear(configs.d_model, n_chars))
self.model.to(self.device)
self.optimizer = Noam(self.model.parameters(),
lr=1.0,
warmup=2_000,
d_model=configs.d_model)
self.loss_func = nn.CrossEntropyLoss()
self.epochs = configs.epochs
self.grad_norm_clip = configs.grad_norm_clip
# Set tracker configurations
tracker.set_scalar("loss.*", True)
def init(self):
# Set tracker configurations
tracker.set_scalar("accuracy.*", True)
tracker.set_scalar("loss.*", True)
# Add a hook to log module outputs
hook_model_outputs(self.mode, self.model, 'model')
# This will keep the accuracy metric stats and memories separate for training and validation.
self.state_modules = [self.accuracy, self.memory]
def __init__(self, *, model: nn.Module,
optimizer: Optional[torch.optim.Adam], loss_func: Callable,
accuracy_func: Callable):
self.accuracy_func = accuracy_func
self.loss_func = loss_func
self.optimizer = optimizer
self.model = model
tracker.set_queue("*.loss", 20, True)
tracker.set_scalar("*.accuracy", True)
def setup_and_add():
for t in range(10):
tracker.set_scalar(f"loss1.{t}", is_print=t == 0)
experiment.start()
for i in monit.loop(1000):
for t in range(10):
tracker.add({f'loss1.{t}': i})
tracker.save()
def init(self):
"""
Initializations
"""
self.state_modules = []
hook_model_outputs(self.mode, self.generator, 'generator')
hook_model_outputs(self.mode, self.discriminator, 'discriminator')
tracker.set_scalar("loss.generator.*", True)
tracker.set_scalar("loss.discriminator.*", True)
tracker.set_image("generated", True, 1 / 100)
def init(self):
# Set tracker configurations
tracker.set_scalar("accuracy.*", True)
tracker.set_scalar("loss.*", True)
# Add a hook to log module outputs
hook_model_outputs(self.mode, self.model, 'model')
# Add accuracy as a state module.
# The name is probably confusing, since it's meant to store
# states between training and validation for RNNs.
# This will keep the accuracy metric stats separate for training and validation.
self.state_modules = [self.accuracy, self.memory]
def run(self):
pytorch_utils.add_model_indicators(self.model)
tracker.set_queue("train.loss", 20, True)
tracker.set_histogram("valid.loss", True)
tracker.set_scalar("valid.accuracy", True)
for _ in self.training_loop:
self.train()
self.test()
if self.is_log_parameters:
pytorch_utils.store_model_indicators(self.model)
def init(self):
self.state_modules = []
self.generator = Generator().to(self.device)
self.discriminator = Discriminator().to(self.device)
self.generator_loss = GeneratorLogitsLoss(self.label_smoothing).to(self.device)
self.discriminator_loss = DiscriminatorLogitsLoss(self.label_smoothing).to(self.device)
hook_model_outputs(self.mode, self.generator, 'generator')
hook_model_outputs(self.mode, self.discriminator, 'discriminator')
tracker.set_scalar("loss.generator.*", True)
tracker.set_scalar("loss.discriminator.*", True)
tracker.set_image("generated", True, 1 / 100)
def add_save():
arr = torch.zeros((1000, 1000))
experiment.start()
for i in monit.loop(N):
for t in range(10):
arr += 1
for t in range(10):
if i == 0:
tracker.set_scalar(f"loss1.{t}", is_print=t == 0)
for t in range(10):
tracker.add({f'loss1.{t}': i})
tracker.save()
def __init__(self, *, discriminator: Module, generator: Module,
discriminator_optimizer: Optional[torch.optim.Adam],
generator_optimizer: Optional[torch.optim.Adam],
discriminator_loss: DiscriminatorLogitsLoss,
generator_loss: GeneratorLogitsLoss):
self.generator = generator
self.discriminator = discriminator
self.generator_loss = generator_loss
self.discriminator_loss = discriminator_loss
self.generator_optimizer = generator_optimizer
self.discriminator_optimizer = discriminator_optimizer
tracker.set_scalar("loss.generator.*", True)
tracker.set_scalar("loss.discriminator.*", True)
def run(self):
# Training and testing
pytorch_utils.add_model_indicators(self.model)
tracker.set_queue("train.loss", 20, True)
tracker.set_histogram("valid.loss", True)
tracker.set_scalar("valid.accuracy", True)
tracker.set_indexed_scalar('valid.sample_loss')
tracker.set_indexed_scalar('valid.sample_pred')
test_data = np.array([d[0].numpy() for d in self.valid_dataset])
experiment.save_numpy("valid.data", test_data)
for _ in self.training_loop:
self.train()
self.valid()
if self.is_log_parameters:
pytorch_utils.store_model_indicators(self.model)
def init(self):
# Initialize encoder & decoder
self.encoder = EncoderRNN(self.d_z,
self.enc_hidden_size).to(self.device)
self.decoder = DecoderRNN(self.d_z, self.dec_hidden_size,
self.n_distributions).to(self.device)
# Set optimizer. Things like type of optimizer and learning rate are configurable
optimizer = OptimizerConfigs()
optimizer.parameters = list(self.encoder.parameters()) + list(
self.decoder.parameters())
self.optimizer = optimizer
# Create sampler
self.sampler = Sampler(self.encoder, self.decoder)
# `npz` file path is `data/sketch/[DATASET NAME].npz`
path = lab.get_data_path() / 'sketch' / f'{self.dataset_name}.npz'
# Load the numpy file
dataset = np.load(str(path), encoding='latin1', allow_pickle=True)
# Create training dataset
self.train_dataset = StrokesDataset(dataset['train'],
self.max_seq_length)
# Create validation dataset
self.valid_dataset = StrokesDataset(dataset['valid'],
self.max_seq_length,
self.train_dataset.scale)
# Create training data loader
self.train_loader = DataLoader(self.train_dataset,
self.batch_size,
shuffle=True)
# Create validation data loader
self.valid_loader = DataLoader(self.valid_dataset, self.batch_size)
# Add hooks to monitor layer outputs on Tensorboard
hook_model_outputs(self.mode, self.encoder, 'encoder')
hook_model_outputs(self.mode, self.decoder, 'decoder')
# Configure the tracker to print the total train/validation loss
tracker.set_scalar("loss.total.*", True)
self.state_modules = []
def __init__(self, *, discriminator: Module, generator: Module,
discriminator_optimizer: Optional[torch.optim.Adam],
generator_optimizer: Optional[torch.optim.Adam],
discriminator_loss: DiscriminatorLogitsLoss,
generator_loss: GeneratorLogitsLoss, discriminator_k: int):
self.discriminator_k = discriminator_k
self.generator = generator
self.discriminator = discriminator
self.generator_loss = generator_loss
self.discriminator_loss = discriminator_loss
self.generator_optimizer = generator_optimizer
self.discriminator_optimizer = discriminator_optimizer
hook_model_outputs(self.generator, 'generator')
hook_model_outputs(self.discriminator, 'discriminator')
tracker.set_scalar("loss.generator.*", True)
tracker.set_scalar("loss.discriminator.*", True)
tracker.set_image("generated", True, 1 / 100)
def __init__(self, *, name: str, model: nn.Module,
optimizer: Optional[torch.optim.Adam], loss_func: Callable,
accuracy_func: Callable,
data_loader: torch.utils.data.DataLoader,
is_increment_global_step: bool, log_interval: Optional[int]):
r"""
Arguments:
loss_func(Callable): A module with a call signature
``(output: torch.Tensor, target: torch.Tensor) -> torch.Tensor``
accuracy_func(Callable): A module with a call signature
``(output: torch.Tensor, target: torch.Tensor) -> int``
"""
self.accuracy_func = accuracy_func
self.loss_func = loss_func
self.log_interval = log_interval
self.is_increment_global_step = is_increment_global_step
self.optimizer = optimizer
self.data_loader = data_loader
self.name = name
self.model = model
tracker.set_queue(".loss", 20, True)
tracker.set_scalar(".accuracy", True)
def init(self):
# Print indicators to screen
tracker.set_scalar('loss.*', True)
tracker.set_scalar('loss_reg.*', True)
tracker.set_scalar('accuracy.*', True)
tracker.set_scalar('steps.*', True)
# We need to set the metrics to calculate them for the epoch for training and validation
self.state_modules = [self.accuracy]
# Initialize the model
self.model = ParityPonderGRU(self.n_elems, self.n_hidden, self.max_steps).to(self.device)
# $L_{Rec}$
self.loss_rec = ReconstructionLoss(nn.BCEWithLogitsLoss(reduction='none')).to(self.device)
# $L_{Reg}$
self.loss_reg = RegularizationLoss(self.lambda_p, self.max_steps).to(self.device)
# Training and validation loaders
self.train_loader = DataLoader(ParityDataset(self.batch_size * self.n_batches, self.n_elems),
batch_size=self.batch_size)
self.valid_loader = DataLoader(ParityDataset(self.batch_size * 32, self.n_elems),
batch_size=self.batch_size)
def __init__(self, configs: Configs):
# Get the device
self.device = torch.device('cpu')
if torch.cuda.is_available():
self.device = torch.device('cuda:0')
# Initialize the dataset
self.dataset = TinyShakespeareDataset(configs.seq_len)
# Initialize the dataloader
self.dataloader = DataLoader(self.dataset,
batch_size=configs.batch_size,
collate_fn=transpose_batch,
shuffle=True)
# FFN with Gated Linear Unit
# $$FFN_{GLU}(x)(x, W_1, V, W_2) = (\sigma(x W_1) \otimes x V) W_2$$
if configs.glu_variant == 'GLU':
ffn = FeedForward(configs.d_model, configs.d_ff, configs.dropout,
nn.Sigmoid(), True, False, False, False)
# FFN with Bilinear hidden layer
# $$FFN_{Bilinear}(x)(x, W_1, V, W_2) = (x W_1 \otimes x V) W_2$$
elif configs.glu_variant == 'Bilinear':
ffn = FeedForward(configs.d_model, configs.d_ff, configs.dropout,
nn.Identity(), True, False, False, False)
# FFN with ReLU gate
# $$FFN_{ReGLU}(x)(x, W_1, V, W_2) = (\max(0, x W_1) \otimes x V) W_2$$
elif configs.glu_variant == 'ReGLU':
ffn = FeedForward(configs.d_model, configs.d_ff, configs.dropout,
nn.ReLU(), True, False, False, False)
# FFN with GELU gate
# $$FFN_{GEGLU}(x)(x, W_1, V, W_2) = (\text{GELU}(x W_1) \otimes x V) W_2$$
elif configs.glu_variant == 'GEGLU':
ffn = FeedForward(configs.d_model, configs.d_ff, configs.dropout,
nn.GELU(), True, False, False, False)
# FFN with Swish gate
# $$FFN_{SwiGLU}(x)(x, W_1, V, W_2) = (\text{Swish}_1(x W_1) \otimes x V) W_2$$
# where $\text{Swish}_\beta(x) = x \sigma(\beta x)$
elif configs.glu_variant == 'SwiGLU':
ffn = FeedForward(configs.d_model, configs.d_ff, configs.dropout,
nn.SiLU(), True, False, False, False)
# FFN with ReLU activation
# $$FFN_{ReLU}(x)(x, W_1, W_2, b_1, b_2) = \text{ReLU}_1(x W_1 + b_1) W_2 + b_2$$
elif configs.glu_variant == 'ReLU':
ffn = FeedForward(configs.d_model, configs.d_ff, configs.dropout,
nn.ReLU())
# FFN with ReLU activation
# $$FFN_{GELU}(x)(x, W_1, W_2, b_1, b_2) = \text{GELU}_1(x W_1 + b_1) W_2 + b_2$$
elif configs.glu_variant == 'GELU':
ffn = FeedForward(configs.d_model, configs.d_ff, configs.dropout,
nn.GELU())
else:
raise ValueError(f'Unknown variant {configs.glu_variant}')
# Number of different characters
n_chars = len(self.dataset.stoi)
# Initialize [Multi-Head Attention module](../mha.html)
mha = MultiHeadAttention(configs.n_heads, configs.d_model,
configs.dropout)
# Initialize the [Transformer Block](../models.html#TransformerLayer)
transformer_layer = TransformerLayer(d_model=configs.d_model,
self_attn=mha,
src_attn=None,
feed_forward=ffn,
dropout_prob=configs.dropout)
# Initialize the model with an
# [embedding layer](../models.html#EmbeddingsWithPositionalEncoding)
# (with fixed positional encoding)
# [transformer encoder](../models.html#Encoder) and
# a linear layer to generate logits.
self.model = AutoregressiveModel(
EmbeddingsWithPositionalEncoding(configs.d_model, n_chars),
Encoder(transformer_layer, configs.n_layers),
nn.Linear(configs.d_model, n_chars))
# Move the model to the current device
self.model.to(self.device)
# Initialize [Noam optimizer](../../optimizers/noam.html)
self.optimizer = Noam(self.model.parameters(),
lr=1.0,
warmup=2_000,
d_model=configs.d_model)
# Cross-entropy loss
self.loss_func = nn.CrossEntropyLoss()
# Number of training epochs;
# *note that our dataset definition repeats the data `seq_len` times in a single epoch
self.epochs = configs.epochs
# Gradient clipping norm
self.grad_norm_clip = configs.grad_norm_clip
# Set tracker configurations
tracker.set_scalar("loss.*", True)
def init(self):
tracker.set_queue("loss.*", 20, True)
tracker.set_scalar("accuracy.*", True)
hook_model_outputs(self.mode, self.model, 'model')
self.state_modules = [self.accuracy_func]
def init(self):
super().init()
# Initialize tracking indicators
tracker.set_scalar("lb_loss.*", False)
tracker.set_scalar("route.*", False)
tracker.set_scalar("dropped.*", False)
def main():
# set indicator types
tracker.set_queue("train_loss", 20, True)
tracker.set_histogram("valid_loss", True)
tracker.set_scalar("valid_accuracy", True)
epochs = 10
train_batch_size = 64
test_batch_size = 1000
use_cuda = True
cuda_device = 0
seed = 5
train_log_interval = 10
learning_rate = 0.01
# get device
is_cuda = use_cuda and torch.cuda.is_available()
if not is_cuda:
device = torch.device("cpu")
else:
if cuda_device < torch.cuda.device_count():
device = torch.device(f"cuda:{cuda_device}")
else:
print(f"Cuda device index {cuda_device} higher than "
f"device count {torch.cuda.device_count()}")
device = torch.device(f"cuda:{torch.cuda.device_count() - 1}")
# data transform
data_transform = transforms.Compose(
[transforms.ToTensor(),
transforms.Normalize((0.1307, ), (0.3081, ))])
# train loader
train_loader = torch.utils.data.DataLoader(datasets.MNIST(
str(lab.get_data_path()),
train=True,
download=True,
transform=data_transform),
batch_size=train_batch_size,
shuffle=True)
# test loader
test_loader = torch.utils.data.DataLoader(datasets.MNIST(
str(lab.get_data_path()),
train=False,
download=True,
transform=data_transform),
batch_size=test_batch_size,
shuffle=False)
# model
model = Net().to(device)
# optimizer
optimizer = optim.Adam(model.parameters(), lr=learning_rate)
# set seeds
torch.manual_seed(seed)
# only for logging purposes
configs = {
'epochs': epochs,
'train_batch_size': train_batch_size,
'test_batch_size': test_batch_size,
'use_cuda': use_cuda,
'cuda_device': cuda_device,
'seed': seed,
'train_log_interval': train_log_interval,
'learning_rate': learning_rate,
'device': device,
'train_loader': train_loader,
'test_loader': test_loader,
'model': model,
'optimizer': optimizer,
}
# create the experiment
experiment.create(name='tracker')
# experiment configs
experiment.calculate_configs(configs)
# pyTorch model
experiment.add_pytorch_models(dict(model=model))
experiment.start()
# training loop
for epoch in range(1, epochs + 1):
train(model, optimizer, train_loader, device, train_log_interval)
test(model, test_loader, device)
logger.log()
# save the model
experiment.save_checkpoint()
def main():
# ✨ Set the types of the stats/indicators.
# They default to scalars if not specified
tracker.set_queue('loss.train', 20, True)
tracker.set_histogram('loss.valid', True)
tracker.set_scalar('accuracy.valid', True)
# Configurations
configs = {
'epochs': 10,
'train_batch_size': 64,
'valid_batch_size': 100,
'use_cuda': True,
'seed': 5,
'train_log_interval': 10,
'learning_rate': 0.01,
}
is_cuda = configs['use_cuda'] and torch.cuda.is_available()
if not is_cuda:
device = torch.device("cpu")
else:
device = torch.device(f"cuda:0")
data_transform = transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.1307,), (0.3081,))
])
train_loader = torch.utils.data.DataLoader(
datasets.MNIST(str(lab.get_data_path()),
train=True,
download=True,
transform=data_transform),
batch_size=configs['train_batch_size'], shuffle=True)
valid_loader = torch.utils.data.DataLoader(
datasets.MNIST(str(lab.get_data_path()),
train=False,
download=True,
transform=data_transform),
batch_size=configs['valid_batch_size'], shuffle=False)
model = Net().to(device)
optimizer = optim.Adam(model.parameters(), lr=configs['learning_rate'])
torch.manual_seed(configs['seed'])
# ✨ Create the experiment
experiment.create(name='mnist_labml_tracker')
# ✨ Save configurations
experiment.configs(configs)
# ✨ Set PyTorch models for checkpoint saving and loading
experiment.add_pytorch_models(dict(model=model))
# ✨ Start and monitor the experiment
with experiment.start():
#
for epoch in range(1, configs['epochs'] + 1):
train(model, optimizer, train_loader, device, configs['train_log_interval'])
validate(model, valid_loader, device)
logger.log()
# ✨ Save the models
experiment.save_checkpoint()
def init(self):
tracker.set_queue("loss.*", 20, True)
tracker.set_scalar("accuracy.*", True)
self.state_modules = [self.accuracy_func]
def startup(self):
pytorch_utils.add_model_indicators(self.model)
tracker.set_queue("train.loss", 20, True)
tracker.set_histogram("valid.loss", True)
tracker.set_scalar("valid.accuracy", True)
