def main(): conf = {'batch_size': 20} with experiment.record(name='sample', exp_conf=conf, writers={'web_api', 'screen'}): for i in range(10_000): values = {'loss': random()} # if i > 1000: # raise RuntimeError('Testing error') # for j in range(0, 100): # values[f'grad.fc.{j}.l1'] = random() # values[f'grad.fc.{j}.l2'] = random() # values[f'grad.fc.{j}.mean'] = random() # # # values[f'param.fc.{j}.l1'] = random() # # values[f'param.fc.{j}.l2'] = random() # # values[f'param.fc.{j}.mean'] = random() # # # # values[f'module.fc.{j}.l1'] = random() # # values[f'module.fc.{j}.l2'] = random() # # values[f'module.fc.{j}.mean'] = random() # # # # values[f'time.fc.{j}.l1'] = random() # # values[f'time.fc.{j}.l2'] = random() # # values[f'time.fc.{j}.mean'] = random() tracker.save(i, values) if i % 1000 == 0: tracker.new_line()
def main(): conf = {'batch_size': 20} for i in range(2): with experiment.record(name=f'sample_{i}', exp_conf=conf, writers={'screen'}): for epoch in range(100): tracker.save(i, loss=random()) tracker.new_line()
def repeat_values(): conf = {'batch_size': 20} with experiment.record(name='sample', exp_conf=conf, writers={'web_api', 'screen'}): for i in range(10): tracker.add_global_step(1) tracker.save('loss', 1) tracker.save('loss', 5) # tracker.save() if i % 1000 == 0: tracker.new_line()
def main(): # Init our model mnist_model = MNISTModel() # Init DataLoader from MNIST Dataset train_ds = MNIST(str(lab.get_data_path()), train=True, download=True, transform=transforms.ToTensor()) train_loader = DataLoader(train_ds, batch_size=32) # Initialize a trainer trainer = pl.Trainer(gpus=1, max_epochs=3, progress_bar_refresh_rate=20, logger=LabMLLightningLogger()) # Train the model ⚡ with experiment.record(name='mnist_lightening', disable_screen=True): trainer.fit(mnist_model, train_loader)
def main(): conf = {'batch_size': 20} with experiment.record(name='sample', exp_conf=conf, writers={'web_api'}): for i in range(10000000): values = {'loss': random()} continue for j in range(0, 100): values[f'grad.fc.{j}.l1'] = random() values[f'grad.fc.{j}.l2'] = random() values[f'grad.fc.{j}.mean'] = random() # values[f'param.fc.{j}.l1'] = random() # values[f'param.fc.{j}.l2'] = random() # values[f'param.fc.{j}.mean'] = random() # # values[f'module.fc.{j}.l1'] = random() # values[f'module.fc.{j}.l2'] = random() # values[f'module.fc.{j}.mean'] = random() # # values[f'time.fc.{j}.l1'] = random() # values[f'time.fc.{j}.l2'] = random() # values[f'time.fc.{j}.mean'] = random() tracker.save(i, values)
import time from numpy.random import random from labml import tracker, experiment conf = {'batch_size': 20} def train(n: int): return 0.999**n + random() / 10, 1 - .999**n + random() / 10 with experiment.record(name='sample', exp_conf=conf): for i in range(100000): time.sleep(0.2) loss, accuracy = train(i) tracker.save(i, {'loss': loss, 'accuracy': accuracy})
def _synthetic_experiment(is_adam: bool): """ ## Synthetic Experiment This is the synthetic experiment described in the paper, that shows a scenario where *Adam* fails. The paper (and Adam) formulates the problem of optimizing as minimizing the expected value of a function, $\mathbb{E}[f(\theta)]$ with respect to the parameters $\theta$. In the stochastic training setting we do not get hold of the function $f$ it self; that is, when you are optimizing a NN $f$ would be the function on entire batch of data. What we actually evaluate is a mini-batch so the actual function is realization of the stochastic $f$. This is why we are talking about an expected value. So let the function realizations be $f_1, f_2, ..., f_T$ for each time step of training. We measure the performance of the optimizer as the regret, $$R(T) = \sum_{t=1}^T \big[ f_t(\theta_t) - f_t(\theta^*) \big]$$ where $theta_t$ is the parameters at time step $t$, and $\theta^*$ is the optimal parameters that minimize $\mathbb{E}[f(\theta)]$. Now lets define the synthetic problem, \begin{align} f_t(x) = \begin{cases} 1010 x, & \text{for $t \mod 101 = 1$} \\ -10 x, & \text{otherwise} \end{cases} \end{align} where $-1 \le x \le +1$. The optimal solution is $x = -1$. This code will try running *Adam* and *AMSGrad* on this problem. """ # Define $x$ parameter x = nn.Parameter(torch.tensor([.0])) # Optimal, $x^* = -1$ x_star = nn.Parameter(torch.tensor([-1]), requires_grad=False) def func(t: int, x_: nn.Parameter): """ ### $f_t(x)$ """ if t % 101 == 1: return (1010 * x_).sum() else: return (-10 * x_).sum() # Initialize the relevant optimizer if is_adam: optimizer = Adam([x], lr=1e-2, betas=(0.9, 0.99)) else: optimizer = AMSGrad([x], lr=1e-2, betas=(0.9, 0.99)) # $R(T)$ total_regret = 0 from labml import monit, tracker, experiment # Create experiment to record results with experiment.record(name='synthetic', comment='Adam' if is_adam else 'AMSGrad'): # Run for $10^7$ steps for step in monit.loop(10_000_000): # $f_t(\theta_t) - f_t(\theta^*)$ regret = func(step, x) - func(step, x_star) # $R(T) = \sum_{t=1}^T \big[ f_t(\theta_t) - f_t(\theta^*) \big]$ total_regret += regret.item() # Track results every 1,000 steps if (step + 1) % 1000 == 0: tracker.save(loss=regret, x=x, regret=total_regret / (step + 1)) # Calculate gradients regret.backward() # Optimize optimizer.step() # Clear gradients optimizer.zero_grad() # Make sure $-1 \le x \le +1$ x.data.clamp_(-1., +1.)
from labml import tracker, experiment from numpy.random import random conf = {'batch_size': 20} with experiment.record(name='sample', exp_conf=conf, writers={'web_api'}): for i in range(10000000): values = {'loss': random()} for j in range(0, 100): values[f'grad.fc.{j}.l1'] = random() values[f'grad.fc.{j}.l2'] = random() values[f'grad.fc.{j}.mean'] = random() # values[f'param.fc.{j}.l1'] = random() # values[f'param.fc.{j}.l2'] = random() # values[f'param.fc.{j}.mean'] = random() # # values[f'module.fc.{j}.l1'] = random() # values[f'module.fc.{j}.l2'] = random() # values[f'module.fc.{j}.mean'] = random() # # values[f'time.fc.{j}.l1'] = random() # values[f'time.fc.{j}.l2'] = random() # values[f'time.fc.{j}.mean'] = random() tracker.save(i, values)
from numpy.random import random from labml import tracker, experiment def train(i): return 0.999**i + random() / 10, 1 - .999**i + random() / 10 conf = {'batch_size': 20} with experiment.record(name='sample', exp_conf=conf, token='903c84fba8ca49ca9f215922833e08cf'): for i in range(10000): loss, accuracy = train(i) tracker.save(i, {'loss': loss, 'accuracy': accuracy})
from fastai.vision.all import untar_data, URLs, ImageDataLoaders, get_image_files, Resize, error_rate, resnet34, \ cnn_learner from labml import lab, experiment from labml.utils.fastai import LabMLFastAICallback path = untar_data( URLs.PETS, dest=lab.get_data_path(), fname=lab.get_data_path() / URLs.path(URLs.PETS).name) / 'images' def is_cat(x): return x[0].isupper() dls = ImageDataLoaders.from_name_func(path, get_image_files(path), valid_pct=0.2, seed=42, label_func=is_cat, item_tfms=Resize(224)) # Train the model ⚡ learn = cnn_learner(dls, resnet34, metrics=error_rate, cbs=LabMLFastAICallback()) with experiment.record(name='pets', exp_conf=learn.labml_configs()): learn.fine_tune(5)
if mode == 'lr_finder': trainer.train_dataloader = data_module.train_dataloader # Run learning rate finder lr_finder = trainer.tuner.lr_find(model, train_loader, min_lr=1e-6, max_lr=100, num_training=500) # Plot with fig = lr_finder.plot(suggest=True, show=True) fig.savefig('lr_finder.png') fig.show() # Pick point based on plot, or get suggestion new_lr = lr_finder.suggestion() print(f"Suggested LR: {new_lr}") exit() wandb.log(params) with experiment.record(name=model_name, exp_conf=dict(params), disable_screen=True, token='ae914b4ab3de48eb84b3a4a757c928b9'): trainer.fit(model, datamodule=data_module) try: print(f"Best Model path: {checkpoint_callback1.best_model_path} Best Score: {checkpoint_callback1.best_model_score:.4f}") except: pass chk_path = checkpoint_callback1.best_model_path model2 = LightningDR.load_from_checkpoint(chk_path, model=base, loss_fn=criterion, optim=optimizer, plist=plist, batch_size=batch_size, lr_scheduler=lr_reduce_scheduler, cyclic_scheduler=cyclic_scheduler, num_class=num_class, target_type=target_type, learning_rate = learning_rate, random_id=random_id) trainer.test(model=model2, test_dataloaders=test_loader) # CAM Generation model2.eval()
from numpy.random import random from labml import tracker, experiment def train(i): return 0.999**i + random() / 10, 1 - .999**i + random() / 10 conf = {'batch_size': 20} with experiment.record(name='sample', exp_conf=conf, token='49d688f6624d468394ca029ecabc8a84'): for i in range(10000): loss, accuracy = train(i) tracker.save(i, {'loss': loss, 'accuracy': accuracy})