def init(self): # Print losses and accuracy to screen tracker.set_scalar('loss.*', True) tracker.set_scalar('accuracy.*', True) # We need to set the metrics to calculate them for the epoch for training and validation self.state_modules = [self.accuracy]
def run(self): tracker.set_queue('train.loss', is_print=True) tracker.set_scalar('valid.loss', is_print=True) for s in self.training_loop: self.train() if self.training_loop.is_interval(self.valid_internal): self.validate()
def __init__(self, configs: Configs): self.device = torch.device('cpu') if torch.cuda.is_available(): self.device = torch.device('cuda:0') self.dataset = TinyShakespeareDataset(configs.seq_len) self.dataloader = DataLoader(self.dataset, batch_size=configs.batch_size, collate_fn=transpose_batch, shuffle=True) if configs.glu_variant == 'GLU': ffn = FeedForward(configs.d_model, configs.d_ff, configs.dropout, nn.Sigmoid(), True, False, False, False) elif configs.glu_variant == 'Bilinear': ffn = FeedForward(configs.d_model, configs.d_ff, configs.dropout, nn.Identity(), True, False, False, False) elif configs.glu_variant == 'ReGLU': ffn = FeedForward(configs.d_model, configs.d_ff, configs.dropout, nn.ReLU(), True, False, False, False) elif configs.glu_variant == 'GEGLU': ffn = FeedForward(configs.d_model, configs.d_ff, configs.dropout, nn.GELU(), True, False, False, False) elif configs.glu_variant == 'SwiGLU': ffn = FeedForward(configs.d_model, configs.d_ff, configs.dropout, nn.SiLU(), True, False, False, False) elif configs.glu_variant == 'ReLU': ffn = FeedForward(configs.d_model, configs.d_ff, configs.dropout, nn.ReLU()) elif configs.glu_variant == 'GELU': ffn = FeedForward(configs.d_model, configs.d_ff, configs.dropout, nn.GELU()) else: raise ValueError(f'Unknown variant {configs.glu_variant}') n_chars = len(self.dataset.stoi) self.model = AutoregressiveModel( EmbeddingsWithPositionalEncoding(configs.d_model, n_chars), Encoder( TransformerLayer(d_model=configs.d_model, self_attn=MultiHeadAttention( configs.n_heads, configs.d_model, configs.dropout), src_attn=None, feed_forward=ffn, dropout_prob=configs.dropout), configs.n_layers), nn.Linear(configs.d_model, n_chars)) self.model.to(self.device) self.optimizer = Noam(self.model.parameters(), lr=1.0, warmup=2_000, d_model=configs.d_model) self.loss_func = nn.CrossEntropyLoss() self.epochs = configs.epochs self.grad_norm_clip = configs.grad_norm_clip # Set tracker configurations tracker.set_scalar("loss.*", True)
def init(self): # Set tracker configurations tracker.set_scalar("accuracy.*", True) tracker.set_scalar("loss.*", True) # Add a hook to log module outputs hook_model_outputs(self.mode, self.model, 'model') # This will keep the accuracy metric stats and memories separate for training and validation. self.state_modules = [self.accuracy, self.memory]
def __init__(self, *, model: nn.Module, optimizer: Optional[torch.optim.Adam], loss_func: Callable, accuracy_func: Callable): self.accuracy_func = accuracy_func self.loss_func = loss_func self.optimizer = optimizer self.model = model tracker.set_queue("*.loss", 20, True) tracker.set_scalar("*.accuracy", True)
def setup_and_add(): for t in range(10): tracker.set_scalar(f"loss1.{t}", is_print=t == 0) experiment.start() for i in monit.loop(1000): for t in range(10): tracker.add({f'loss1.{t}': i}) tracker.save()
def init(self): """ Initializations """ self.state_modules = [] hook_model_outputs(self.mode, self.generator, 'generator') hook_model_outputs(self.mode, self.discriminator, 'discriminator') tracker.set_scalar("loss.generator.*", True) tracker.set_scalar("loss.discriminator.*", True) tracker.set_image("generated", True, 1 / 100)
def init(self): # Set tracker configurations tracker.set_scalar("accuracy.*", True) tracker.set_scalar("loss.*", True) # Add a hook to log module outputs hook_model_outputs(self.mode, self.model, 'model') # Add accuracy as a state module. # The name is probably confusing, since it's meant to store # states between training and validation for RNNs. # This will keep the accuracy metric stats separate for training and validation. self.state_modules = [self.accuracy, self.memory]
def run(self): pytorch_utils.add_model_indicators(self.model) tracker.set_queue("train.loss", 20, True) tracker.set_histogram("valid.loss", True) tracker.set_scalar("valid.accuracy", True) for _ in self.training_loop: self.train() self.test() if self.is_log_parameters: pytorch_utils.store_model_indicators(self.model)
def init(self): self.state_modules = [] self.generator = Generator().to(self.device) self.discriminator = Discriminator().to(self.device) self.generator_loss = GeneratorLogitsLoss(self.label_smoothing).to(self.device) self.discriminator_loss = DiscriminatorLogitsLoss(self.label_smoothing).to(self.device) hook_model_outputs(self.mode, self.generator, 'generator') hook_model_outputs(self.mode, self.discriminator, 'discriminator') tracker.set_scalar("loss.generator.*", True) tracker.set_scalar("loss.discriminator.*", True) tracker.set_image("generated", True, 1 / 100)
def add_save(): arr = torch.zeros((1000, 1000)) experiment.start() for i in monit.loop(N): for t in range(10): arr += 1 for t in range(10): if i == 0: tracker.set_scalar(f"loss1.{t}", is_print=t == 0) for t in range(10): tracker.add({f'loss1.{t}': i}) tracker.save()
def __init__(self, *, discriminator: Module, generator: Module, discriminator_optimizer: Optional[torch.optim.Adam], generator_optimizer: Optional[torch.optim.Adam], discriminator_loss: DiscriminatorLogitsLoss, generator_loss: GeneratorLogitsLoss): self.generator = generator self.discriminator = discriminator self.generator_loss = generator_loss self.discriminator_loss = discriminator_loss self.generator_optimizer = generator_optimizer self.discriminator_optimizer = discriminator_optimizer tracker.set_scalar("loss.generator.*", True) tracker.set_scalar("loss.discriminator.*", True)
def run(self): # Training and testing pytorch_utils.add_model_indicators(self.model) tracker.set_queue("train.loss", 20, True) tracker.set_histogram("valid.loss", True) tracker.set_scalar("valid.accuracy", True) tracker.set_indexed_scalar('valid.sample_loss') tracker.set_indexed_scalar('valid.sample_pred') test_data = np.array([d[0].numpy() for d in self.valid_dataset]) experiment.save_numpy("valid.data", test_data) for _ in self.training_loop: self.train() self.valid() if self.is_log_parameters: pytorch_utils.store_model_indicators(self.model)
def init(self): # Initialize encoder & decoder self.encoder = EncoderRNN(self.d_z, self.enc_hidden_size).to(self.device) self.decoder = DecoderRNN(self.d_z, self.dec_hidden_size, self.n_distributions).to(self.device) # Set optimizer. Things like type of optimizer and learning rate are configurable optimizer = OptimizerConfigs() optimizer.parameters = list(self.encoder.parameters()) + list( self.decoder.parameters()) self.optimizer = optimizer # Create sampler self.sampler = Sampler(self.encoder, self.decoder) # `npz` file path is `data/sketch/[DATASET NAME].npz` path = lab.get_data_path() / 'sketch' / f'{self.dataset_name}.npz' # Load the numpy file dataset = np.load(str(path), encoding='latin1', allow_pickle=True) # Create training dataset self.train_dataset = StrokesDataset(dataset['train'], self.max_seq_length) # Create validation dataset self.valid_dataset = StrokesDataset(dataset['valid'], self.max_seq_length, self.train_dataset.scale) # Create training data loader self.train_loader = DataLoader(self.train_dataset, self.batch_size, shuffle=True) # Create validation data loader self.valid_loader = DataLoader(self.valid_dataset, self.batch_size) # Add hooks to monitor layer outputs on Tensorboard hook_model_outputs(self.mode, self.encoder, 'encoder') hook_model_outputs(self.mode, self.decoder, 'decoder') # Configure the tracker to print the total train/validation loss tracker.set_scalar("loss.total.*", True) self.state_modules = []
def __init__(self, *, discriminator: Module, generator: Module, discriminator_optimizer: Optional[torch.optim.Adam], generator_optimizer: Optional[torch.optim.Adam], discriminator_loss: DiscriminatorLogitsLoss, generator_loss: GeneratorLogitsLoss, discriminator_k: int): self.discriminator_k = discriminator_k self.generator = generator self.discriminator = discriminator self.generator_loss = generator_loss self.discriminator_loss = discriminator_loss self.generator_optimizer = generator_optimizer self.discriminator_optimizer = discriminator_optimizer hook_model_outputs(self.generator, 'generator') hook_model_outputs(self.discriminator, 'discriminator') tracker.set_scalar("loss.generator.*", True) tracker.set_scalar("loss.discriminator.*", True) tracker.set_image("generated", True, 1 / 100)
def __init__(self, *, name: str, model: nn.Module, optimizer: Optional[torch.optim.Adam], loss_func: Callable, accuracy_func: Callable, data_loader: torch.utils.data.DataLoader, is_increment_global_step: bool, log_interval: Optional[int]): r""" Arguments: loss_func(Callable): A module with a call signature ``(output: torch.Tensor, target: torch.Tensor) -> torch.Tensor`` accuracy_func(Callable): A module with a call signature ``(output: torch.Tensor, target: torch.Tensor) -> int`` """ self.accuracy_func = accuracy_func self.loss_func = loss_func self.log_interval = log_interval self.is_increment_global_step = is_increment_global_step self.optimizer = optimizer self.data_loader = data_loader self.name = name self.model = model tracker.set_queue(".loss", 20, True) tracker.set_scalar(".accuracy", True)
def init(self): # Print indicators to screen tracker.set_scalar('loss.*', True) tracker.set_scalar('loss_reg.*', True) tracker.set_scalar('accuracy.*', True) tracker.set_scalar('steps.*', True) # We need to set the metrics to calculate them for the epoch for training and validation self.state_modules = [self.accuracy] # Initialize the model self.model = ParityPonderGRU(self.n_elems, self.n_hidden, self.max_steps).to(self.device) # $L_{Rec}$ self.loss_rec = ReconstructionLoss(nn.BCEWithLogitsLoss(reduction='none')).to(self.device) # $L_{Reg}$ self.loss_reg = RegularizationLoss(self.lambda_p, self.max_steps).to(self.device) # Training and validation loaders self.train_loader = DataLoader(ParityDataset(self.batch_size * self.n_batches, self.n_elems), batch_size=self.batch_size) self.valid_loader = DataLoader(ParityDataset(self.batch_size * 32, self.n_elems), batch_size=self.batch_size)
def __init__(self, configs: Configs): # Get the device self.device = torch.device('cpu') if torch.cuda.is_available(): self.device = torch.device('cuda:0') # Initialize the dataset self.dataset = TinyShakespeareDataset(configs.seq_len) # Initialize the dataloader self.dataloader = DataLoader(self.dataset, batch_size=configs.batch_size, collate_fn=transpose_batch, shuffle=True) # FFN with Gated Linear Unit # $$FFN_{GLU}(x)(x, W_1, V, W_2) = (\sigma(x W_1) \otimes x V) W_2$$ if configs.glu_variant == 'GLU': ffn = FeedForward(configs.d_model, configs.d_ff, configs.dropout, nn.Sigmoid(), True, False, False, False) # FFN with Bilinear hidden layer # $$FFN_{Bilinear}(x)(x, W_1, V, W_2) = (x W_1 \otimes x V) W_2$$ elif configs.glu_variant == 'Bilinear': ffn = FeedForward(configs.d_model, configs.d_ff, configs.dropout, nn.Identity(), True, False, False, False) # FFN with ReLU gate # $$FFN_{ReGLU}(x)(x, W_1, V, W_2) = (\max(0, x W_1) \otimes x V) W_2$$ elif configs.glu_variant == 'ReGLU': ffn = FeedForward(configs.d_model, configs.d_ff, configs.dropout, nn.ReLU(), True, False, False, False) # FFN with GELU gate # $$FFN_{GEGLU}(x)(x, W_1, V, W_2) = (\text{GELU}(x W_1) \otimes x V) W_2$$ elif configs.glu_variant == 'GEGLU': ffn = FeedForward(configs.d_model, configs.d_ff, configs.dropout, nn.GELU(), True, False, False, False) # FFN with Swish gate # $$FFN_{SwiGLU}(x)(x, W_1, V, W_2) = (\text{Swish}_1(x W_1) \otimes x V) W_2$$ # where $\text{Swish}_\beta(x) = x \sigma(\beta x)$ elif configs.glu_variant == 'SwiGLU': ffn = FeedForward(configs.d_model, configs.d_ff, configs.dropout, nn.SiLU(), True, False, False, False) # FFN with ReLU activation # $$FFN_{ReLU}(x)(x, W_1, W_2, b_1, b_2) = \text{ReLU}_1(x W_1 + b_1) W_2 + b_2$$ elif configs.glu_variant == 'ReLU': ffn = FeedForward(configs.d_model, configs.d_ff, configs.dropout, nn.ReLU()) # FFN with ReLU activation # $$FFN_{GELU}(x)(x, W_1, W_2, b_1, b_2) = \text{GELU}_1(x W_1 + b_1) W_2 + b_2$$ elif configs.glu_variant == 'GELU': ffn = FeedForward(configs.d_model, configs.d_ff, configs.dropout, nn.GELU()) else: raise ValueError(f'Unknown variant {configs.glu_variant}') # Number of different characters n_chars = len(self.dataset.stoi) # Initialize [Multi-Head Attention module](../mha.html) mha = MultiHeadAttention(configs.n_heads, configs.d_model, configs.dropout) # Initialize the [Transformer Block](../models.html#TransformerLayer) transformer_layer = TransformerLayer(d_model=configs.d_model, self_attn=mha, src_attn=None, feed_forward=ffn, dropout_prob=configs.dropout) # Initialize the model with an # [embedding layer](../models.html#EmbeddingsWithPositionalEncoding) # (with fixed positional encoding) # [transformer encoder](../models.html#Encoder) and # a linear layer to generate logits. self.model = AutoregressiveModel( EmbeddingsWithPositionalEncoding(configs.d_model, n_chars), Encoder(transformer_layer, configs.n_layers), nn.Linear(configs.d_model, n_chars)) # Move the model to the current device self.model.to(self.device) # Initialize [Noam optimizer](../../optimizers/noam.html) self.optimizer = Noam(self.model.parameters(), lr=1.0, warmup=2_000, d_model=configs.d_model) # Cross-entropy loss self.loss_func = nn.CrossEntropyLoss() # Number of training epochs; # *note that our dataset definition repeats the data `seq_len` times in a single epoch self.epochs = configs.epochs # Gradient clipping norm self.grad_norm_clip = configs.grad_norm_clip # Set tracker configurations tracker.set_scalar("loss.*", True)
def init(self): tracker.set_queue("loss.*", 20, True) tracker.set_scalar("accuracy.*", True) hook_model_outputs(self.mode, self.model, 'model') self.state_modules = [self.accuracy_func]
def init(self): super().init() # Initialize tracking indicators tracker.set_scalar("lb_loss.*", False) tracker.set_scalar("route.*", False) tracker.set_scalar("dropped.*", False)
def main(): # set indicator types tracker.set_queue("train_loss", 20, True) tracker.set_histogram("valid_loss", True) tracker.set_scalar("valid_accuracy", True) epochs = 10 train_batch_size = 64 test_batch_size = 1000 use_cuda = True cuda_device = 0 seed = 5 train_log_interval = 10 learning_rate = 0.01 # get device is_cuda = use_cuda and torch.cuda.is_available() if not is_cuda: device = torch.device("cpu") else: if cuda_device < torch.cuda.device_count(): device = torch.device(f"cuda:{cuda_device}") else: print(f"Cuda device index {cuda_device} higher than " f"device count {torch.cuda.device_count()}") device = torch.device(f"cuda:{torch.cuda.device_count() - 1}") # data transform data_transform = transforms.Compose( [transforms.ToTensor(), transforms.Normalize((0.1307, ), (0.3081, ))]) # train loader train_loader = torch.utils.data.DataLoader(datasets.MNIST( str(lab.get_data_path()), train=True, download=True, transform=data_transform), batch_size=train_batch_size, shuffle=True) # test loader test_loader = torch.utils.data.DataLoader(datasets.MNIST( str(lab.get_data_path()), train=False, download=True, transform=data_transform), batch_size=test_batch_size, shuffle=False) # model model = Net().to(device) # optimizer optimizer = optim.Adam(model.parameters(), lr=learning_rate) # set seeds torch.manual_seed(seed) # only for logging purposes configs = { 'epochs': epochs, 'train_batch_size': train_batch_size, 'test_batch_size': test_batch_size, 'use_cuda': use_cuda, 'cuda_device': cuda_device, 'seed': seed, 'train_log_interval': train_log_interval, 'learning_rate': learning_rate, 'device': device, 'train_loader': train_loader, 'test_loader': test_loader, 'model': model, 'optimizer': optimizer, } # create the experiment experiment.create(name='tracker') # experiment configs experiment.calculate_configs(configs) # pyTorch model experiment.add_pytorch_models(dict(model=model)) experiment.start() # training loop for epoch in range(1, epochs + 1): train(model, optimizer, train_loader, device, train_log_interval) test(model, test_loader, device) logger.log() # save the model experiment.save_checkpoint()
def main(): # ✨ Set the types of the stats/indicators. # They default to scalars if not specified tracker.set_queue('loss.train', 20, True) tracker.set_histogram('loss.valid', True) tracker.set_scalar('accuracy.valid', True) # Configurations configs = { 'epochs': 10, 'train_batch_size': 64, 'valid_batch_size': 100, 'use_cuda': True, 'seed': 5, 'train_log_interval': 10, 'learning_rate': 0.01, } is_cuda = configs['use_cuda'] and torch.cuda.is_available() if not is_cuda: device = torch.device("cpu") else: device = torch.device(f"cuda:0") data_transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,)) ]) train_loader = torch.utils.data.DataLoader( datasets.MNIST(str(lab.get_data_path()), train=True, download=True, transform=data_transform), batch_size=configs['train_batch_size'], shuffle=True) valid_loader = torch.utils.data.DataLoader( datasets.MNIST(str(lab.get_data_path()), train=False, download=True, transform=data_transform), batch_size=configs['valid_batch_size'], shuffle=False) model = Net().to(device) optimizer = optim.Adam(model.parameters(), lr=configs['learning_rate']) torch.manual_seed(configs['seed']) # ✨ Create the experiment experiment.create(name='mnist_labml_tracker') # ✨ Save configurations experiment.configs(configs) # ✨ Set PyTorch models for checkpoint saving and loading experiment.add_pytorch_models(dict(model=model)) # ✨ Start and monitor the experiment with experiment.start(): # for epoch in range(1, configs['epochs'] + 1): train(model, optimizer, train_loader, device, configs['train_log_interval']) validate(model, valid_loader, device) logger.log() # ✨ Save the models experiment.save_checkpoint()
def init(self): tracker.set_queue("loss.*", 20, True) tracker.set_scalar("accuracy.*", True) self.state_modules = [self.accuracy_func]
def startup(self): pytorch_utils.add_model_indicators(self.model) tracker.set_queue("train.loss", 20, True) tracker.set_histogram("valid.loss", True) tracker.set_scalar("valid.accuracy", True)