def lr_find(self, files=None, bs=None, n_jobs=-1, verbose=1, **kwargs): bs = bs or self.bs files = files or self.files train_ds = RandomTileDataset(files, label_fn=self.label_fn, n_jobs=n_jobs, verbose=verbose, **self.mw_kwargs, **self.ds_kwargs) dls = DataLoaders.from_dsets(train_ds, train_ds, bs=bs) pre = None if self.pretrained == 'new' else self.pretrained model = torch.hub.load(self.repo, self.arch, pretrained=pre, n_classes=dls.c, in_channels=self.in_channels) if torch.cuda.is_available(): dls.cuda(), model.cuda() learn = Learner(dls, model, metrics=self.metrics, wd=self.wd, loss_func=self.loss_fn, opt_func=_optim_dict[self.optim]) if self.mpt: learn.to_fp16() sug_lrs = learn.lr_find(**kwargs) return sug_lrs, learn.recorder
def fit(self, i, n_iter=None, lr_max=None, **kwargs): n_iter = n_iter or self.n_iter lr_max = lr_max or self.lr name = self.ensemble_dir / f'{self.arch}_model-{i}.pth' pre = None if self.pretrained == 'new' else self.pretrained model = self.get_model(pretrained=pre) files_train, files_val = self.splits[i] dls = self.get_dls(files_train, files_val) self.learn = Learner(dls, model, metrics=self.metrics, wd=self.wd, loss_func=self.loss_fn, opt_func=_optim_dict[self.optim], cbs=self.cbs) self.learn.model_dir = self.ensemble_dir.parent / '.tmp' if self.mpt: self.learn.to_fp16() print(f'Starting training for {name.name}') epochs = calc_iterations(n_iter=n_iter, ds_length=len(dls.train_ds), bs=self.bs) self.learn.fit_one_cycle(epochs, lr_max) print(f'Saving model at {name}') name.parent.mkdir(exist_ok=True, parents=True) self.save_model(name, self.learn.model) self.models[i] = name self.recorder[i] = self.learn.recorder
def fit(self, i, n_iter=None, base_lr=None, **kwargs): n_iter = n_iter or self.n_iter base_lr = base_lr or self.base_lr name = self.ensemble_dir / f'{self.model_name}-fold{i}.pth' model = self._create_model() files_train, files_val = self.splits[i] dls = self._get_dls(files_train, files_val) log_name = f'{name.name}_{time.strftime("%Y%m%d-%H%M%S")}.csv' log_dir = self.ensemble_dir / 'logs' log_dir.mkdir(exist_ok=True, parents=True) cbs = self.cbs.append(CSVLogger(fname=log_dir / log_name)) self.learn = Learner(dls, model, metrics=self.metrics, wd=self.weight_decay, loss_func=self.loss_fn, opt_func=_optim_dict[self.optim], cbs=self.cbs) self.learn.model_dir = self.ensemble_dir.parent / '.tmp' if self.mixed_precision_training: self.learn.to_fp16() print(f'Starting training for {name.name}') epochs = calc_iterations(n_iter=n_iter, ds_length=len(dls.train_ds), bs=self.batch_size) #self.learn.fit_one_cycle(epochs, lr_max) self.learn.fine_tune(epochs, base_lr=base_lr) print(f'Saving model at {name}') name.parent.mkdir(exist_ok=True, parents=True) save_smp_model(self.learn.model, self.arch, name, stats=self.stats) self.models[i] = name self.recorder[i] = self.learn.recorder
def classify(image: PILImage, learner: Learner) -> Tuple[AnimalType, float]: with learner.no_bar(): results = learner.predict(image) _, category, probabilities = results is_a_cat = category == 1 animal_type = AnimalType.cat if is_a_cat else AnimalType.dog percent = np.round(100 * probabilities) return animal_type, percent[category]
def predict_mnist(): dls = read_data_loaders() lr = 1. learn = Learner(dls=dls, model=torch.nn.Linear(28 * 28, 1), opt_func=SGD, loss_func=mnist_loss, metrics=batch_accuracy) learn.fit(10, lr=lr)
def make_xla_child_learner(rank, sync_valid, learner_args, add_args, ctrl_args): "create a learner using passed parameters" device = xm.xla_device() world_size = xm.xrt_world_size() dls = build_distributed_dataloaders(learner_args.pop('base_dls'), rank, world_size, sync_valid=sync_valid) model = learner_args.pop('wrapped_model').to(device) master_cbs = learner_args.pop('master_cbs') if master_cbs is None: master_cbs = L() learner = Learner(dls, model, **learner_args) learner.__stored_args__ = {**learner.__stored_args__, **add_args} learner.to_multi_xla(device, rank, sync_valid=sync_valid) if not ctrl_args['use_progress'] and 'progress' in L( learner.cbs).attrgot('name'): learner.remove_cbs(ProgressCallback) if rank == 0: learner.add_cbs(master_cbs) return learner
def objective(trial: optuna.trial.Trial) -> float: model = nn.Sequential(nn.Linear(20, 1), nn.Sigmoid()) learn = Learner( data, model, loss_func=F.nll_loss, metrics=[accuracy], ) learn.fit(1, cbs=FastAIV2PruningCallback(trial)) return 1.0
def lr_find(self, files=None, **kwargs): files = files or self.files dls = self._get_dls(files) model = self._create_model() learn = Learner(dls, model, metrics=self.metrics, wd=self.weight_decay, loss_func=self.loss_fn, opt_func=_optim_dict[self.optim]) if self.mixed_precision_training: learn.to_fp16() sug_lrs = learn.lr_find(**kwargs) return sug_lrs, learn.recorder
def fit(self, X: TabularDataLoaders): """Creates the learner and trains it.""" emb_szs = get_emb_sz(X.train_ds, {}) self.emb_szs = {col: sz for col, sz in zip(self.cat_names, emb_szs)} n_conts = len(X.cont_names) n_cats = sum(list(map(lambda e: e[1], emb_szs))) in_sz = n_conts + n_cats out_sz = n_conts + len(X.cat_names) # Create the embedding model model = AutoEmbedder(in_sz, out_sz, emb_szs, [2000, 1000]) self.learn = Learner(X, model, loss_func=EmbeddingLoss(model), wd=1.0) # TODO hide training progress? with self.learn.no_bar(): self.learn.fit_one_cycle(20, lr_max=3e-3)
def lr_find(self, files=None, **kwargs): files = files or self.files dls = self.get_dls(files) pre = None if self.pretrained == 'new' else self.pretrained model = self.get_model(pretrained=pre) learn = Learner(dls, model, metrics=self.metrics, wd=self.wd, loss_func=self.loss_fn, opt_func=_optim_dict[self.optim]) if self.mpt: learn.to_fp16() sug_lrs = learn.lr_find(**kwargs) return sug_lrs, learn.recorder
def run_training(learn: Learner, resume_ckpt: Path, min_lr=0.005, head_runs=1, full_runs=1): if resume_ckpt: print(f'Loading {resume_ckpt}...') try: learn.model.load_state_dict(torch.load(resume_ckpt)) except Exception as e: print(f'Error while trying to load {resume_ckpt}: {e}') monitor.display_average_stats_per_gpu() print(f"Training for {head_runs}+{full_runs} epochs at min LR {min_lr}") learn.fine_tune(full_runs, min_lr, freeze_epochs=head_runs)
def get_learner_rnn_feedback(model_name): # Training settings bs = 1 seq_len = 1 # Datasets and Dataloaders trn_lds = LinearFeedbackDataset(df_small_trn, trn_tfms, PATH, IMAGES_FOLDER) val_lds = LinearFeedbackDataset(df_small_val, trn_tfms, PATH, IMAGES_FOLDER) trn_dl = FastaiDataLoader(trn_lds, batch_sampler=trn_lds.batch_sampler()) val_dl = FastaiDataLoader(val_lds, batch_sampler=val_lds.batch_sampler()) # Model model_folder = "CNNtoRNNFeedback" model = CNNtoRNNFeedback(1024, 200, 2, seq_len, bs, 14, use_ground_truth=False) layer_groups = [ list(model.encoder.children())[:6], list(model.encoder.children())[6:], [model.lstm, model.linear], ] # opt_fn is used like this: optimizer = opt_fn(trainable_params(model), lr=1e-1) opt_fn = partial(optim.SGD, momentum=0.9) criterion = F.l1_loss learner = Learner( MockedData(trn_dl, val_dl), CustomModel(model, layer_groups), metrics=METRICS, opt_fn=opt_fn, crit=criterion, tmp_name=os.path.join(ROOT, PATH, 'tmp'), models_name=os.path.join(ROOT, PATH, 'models', model_folder), ) # clip and reg_fn needs shouldn't be passed to the constructor because it sets as None anyway... # learner.reg_fn = partial(seq2seq_reg, alpha=2, beta=1) learner.clip = 0.4 learner.load(model_name) learner.model.eval() return learner
def fastai_tabular_model(data, **kwargs): # Create a fine-tunable learner return Learner(data, Model(), loss_func=nn.MSELoss(), splitter=splitter, **kwargs)
def get_learner_rnn(model_name): # mem comsuption: 7200 MB bs = 1 seq_len = 1 # Datasets and Dataloaders trn_ds = BatchifiedDataset(df_large_trn, bs, seq_len, trn_tfms, PATH, IMAGES_FOLDER) val_ds = BatchifiedDataset(df_large_val, bs, seq_len, trn_tfms, PATH, IMAGES_FOLDER) trn_dl = FastaiDataLoader(trn_ds, batch_sampler=trn_ds.batch_sampler()) val_dl = FastaiDataLoader(val_ds, batch_sampler=val_ds.batch_sampler()) # Model model_folder = "CNNtoRNN_new" model = CNNtoRNN( encode_size=128, # 1024 hidden_size=32, # 200 num_layers=2, bs=bs, output_size=14) layer_groups = [ list(model.encoder.children())[:6], list(model.encoder.children())[6:], [model.encoder_linear, model.lstm, model.linear], ] # opt_fn is used like this: optimizer = opt_fn(trainable_params(model), lr=1e-1) opt_fn = partial(optim.SGD, momentum=0.9) criterion = F.mse_loss learner = Learner( MockedData(trn_dl, val_dl), CustomModel(model, layer_groups), metrics=METRICS, opt_fn=opt_fn, crit=criterion, tmp_name=os.path.join(ROOT, PATH, 'tmp'), models_name=os.path.join(ROOT, PATH, 'models', model_folder), ) # clip and reg_fn needs shouldn't be passed to the constructor because it sets as None anyway... # learner.reg_fn = partial(seq2seq_reg, alpha=2, beta=1) learner.clip = 0.4 learner.load(model_name) learner.model.eval() return learner
def main(): # Parse command-line arguments args = parse_args() # Enable auto logging mlflow.fastai.autolog() # Create Learner model learn = Learner(get_data_loaders(), Model(), loss_func=nn.MSELoss(), splitter=splitter) # Start MLflow session with mlflow.start_run(): # Train and fit with default or supplied command line arguments learn.fit_one_cycle(args.epochs, args.lr)
def fastai_learner(): model = Model() loss = Loss() dblock = DataBlock(get_items=get_items, get_y=np.sum) dls = dblock.datasets(None).dataloaders() learner = Learner(dls, model, loss) return learner
def predict(pdf_tiles: list, learner: Learner): # Get predicted labels and confidences for the given image tiles predictions = [learner.predict(tile) for tile in pdf_tiles] labels = [prediction[0] for prediction in predictions] confidences = [ float(prediction[2][prediction[1]].numpy()) for prediction in predictions ] return labels, confidences
def predict(self, files, model_no, path=None, **kwargs): model_path = self.models[model_no] model = self.load_model(model_path) ds_kwargs = self.ds_kwargs # Adding extra padding (overlap) for models that have the same input and output shape if ds_kwargs['padding'][0] == 0: ds_kwargs['padding'] = (self.extra_padding, ) * 2 ds = TileDataset(files, **ds_kwargs) dls = DataLoaders.from_dsets(ds, batch_size=self.bs, after_batch=self.get_batch_tfms(), shuffle=False, drop_last=False, **self.dl_kwargs) if torch.cuda.is_available(): dls.cuda() learn = Learner(dls, model, loss_func=self.loss_fn) if self.mpt: learn.to_fp16() if path: path = path / f'model_{model_no}' return learn.predict_tiles(dl=dls.train, path=path, **kwargs)
def pack_models(path: str) -> None: model = LinearModel() loss = Loss() dblock = DataBlock(get_items=get_items, get_y=np.sum) dls = dblock.datasets(None).dataloaders() learner = Learner(dls, model, loss) FastAIModel(learner).save(path)
def predict(self, files, model_no, bs=None, **kwargs): bs = bs or self.bs model_path = self.models[model_no] model = self.load_model(model_path) batch_tfms = Normalize.from_stats(*self.stats) ds = TileDataset(files, **self.ds_kwargs) dls = DataLoaders.from_dsets(ds, batch_size=bs, after_batch=batch_tfms, shuffle=False, drop_last=False, num_workers=0) if torch.cuda.is_available(): dls.cuda(), model.cuda() learn = Learner(dls, model, loss_func=self.loss_fn) if self.mpt: learn.to_fp16() results = learn.predict_tiles(dl=dls.train, **kwargs) pth_tmp = self.path / '.tmp' / model_path.name save_tmp(pth_tmp, files, results) return results
def get_learner( data, arch, lr, loss_func=nn.MSELoss(), cb_funcs=None, opt_func=Adam, **kwargs ): init_cnn(arch) dls = DataLoaders.from_dsets( data.train_ds, data.valid_ds, bs=data.train_dl.batch_size, ) return Learner(dls, arch, loss_func, lr=lr, cbs=cb_funcs, opt_func=opt_func)
def pack_models(path): from bentoml.frameworks.fastai import FastaiModelArtifact model = Model() loss = Loss() dblock = DataBlock(get_items=get_items, get_y=np.sum) dls = dblock.datasets(None).dataloaders() learner = Learner(dls, model, loss) FastaiModelArtifact("model").pack(learner).save(path)
def get_autoencoder(dls, verbose=True): ae = Autoencoder(z_size=64, input_dimension=(128, 128, 3)) model = torch.nn.Sequential(ae) if verbose: print(model) callbacks = ActivationStats(with_hist=True) #learn = Learner(dls, model, loss_func = nn.BCELoss(), metrics=[rmse], cbs=callbacks) learn = Learner(dls, model, loss_func=torch.nn.MSELoss(), metrics=[rmse], cbs=callbacks) return learn
def print_dialogue_batch(learner: Learner, modeldata: ModelData, input_field, output_field, num_batches=1, num_sentences=-1, is_test=False, num_beams=1, smoothing_function=None, weights=None): weights = (1 / 3., 1 / 3., 1 / 3.) if weights is None else weights smoothing_function = SmoothingFunction( ).method1 if smoothing_function is None else smoothing_function predictions, targets, inputs = learner.predict_with_targs_and_inputs( is_test=is_test, num_beams=num_beams) blue_scores = [] for batch_num, (input, target, prediction) in enumerate(zip(inputs, targets, predictions)): input = np.transpose( input, [1, 2, 0]) # transpose number of utterances to beams [sl, bs, nb] inputs_str: BatchBeamTokens = modeldata.itos(input, input_field) inputs_str: List[str] = ["\n".join(conv) for conv in inputs_str] predictions_str: BatchBeamTokens = modeldata.itos( prediction, output_field) targets_str: BatchBeamTokens = modeldata.itos(target, output_field) for index, (inp, targ, pred) in enumerate( zip(inputs_str, targets_str, predictions_str)): if targ[0].split() == pred[0].split()[1:]: blue_score = 1 else: blue_score = sentence_bleu( [targ[0].split()], pred[0].split()[1:], smoothing_function=smoothing_function, weights=weights) print( f'BATCH: {batch_num} SAMPLE : {index}\nINPUT:\n{"".join(inp)}\nTARGET:\n{ "".join(targ)}\nPREDICTON:\n{"".join(pred)}\nblue: {blue_score}\n\n' ) blue_scores.append(blue_score) if 0 < num_sentences <= index - 1: break if 0 < num_batches <= batch_num - 1: break print( f'bleu score: mean: {np.mean(blue_scores)}, std: {np.std(blue_scores)}' )
def get_learner(dls): model = torch.nn.Sequential(ConvLayer(3, 24, stride=2), ConvLayer(24, 32, stride=2), ConvLayer(32, 64, stride=2), ConvLayer(64, 128, stride=2), ConvLayer(128, 256, stride=2), torch.nn.AdaptiveAvgPool2d(1), Flatten(), torch.nn.Linear(256, 50), torch.nn.ReLU(), torch.nn.Linear(50, dls.c), torch.nn.Tanh()) #print(model) callbacks = ActivationStats(with_hist=True) learn = Learner(dls, model, loss_func=MSELossFlat(), metrics=[rmse], cbs=callbacks) #valley = learn.lr_find() return learn
def tsimage_learner(dls, arch=None, pretrained=False, loss_func=None, opt_func=Adam, lr=defaults.lr, cbs=None, metrics=None, path=None, model_dir='models', wd=None, wd_bn_bias=False, train_bn=True, moms=(0.95, 0.85, 0.95), **kwargs): if arch is None: from .models.XResNet1d import xresnet34 arch = xresnet34 elif isinstance(arch, str): arch = get_arch(arch) model = build_tsimage_model(arch, dls=dls, pretrained=pretrained, **kwargs) learn = Learner(dls=dls, model=model, loss_func=loss_func, opt_func=opt_func, lr=lr, cbs=cbs, metrics=metrics, path=path, model_dir=model_dir, wd=wd, wd_bn_bias=wd_bn_bias, train_bn=train_bn, moms=moms) # keep track of args for loggers store_attr('arch', self=learn) return learn
def print_batch(learner: Learner, modeldata: ModelData, input_field, output_field, num_batches=1, num_sentences=-1, is_test=False, num_beams=1, weights=None, smoothing_function=None): predictions, targets, inputs = learner.predict_with_targs_and_inputs( is_test=is_test, num_beams=num_beams) weights = (1 / 3., 1 / 3., 1 / 3.) if weights is None else weights smoothing_function = SmoothingFunction( ).method1 if smoothing_function is None else smoothing_function blue_scores = [] for batch_num, (input, target, prediction) in enumerate(zip(inputs, targets, predictions)): inputs_str: BatchBeamTokens = modeldata.itos(input, input_field) predictions_str: BatchBeamTokens = modeldata.itos( prediction, output_field) targets_str: BatchBeamTokens = modeldata.itos(target, output_field) for index, (inp, targ, pred) in enumerate( zip(inputs_str, targets_str, predictions_str)): blue_score = sentence_bleu([targ], pred, smoothing_function=smoothing_function, weights=weights) print( f'batch: {batch_num} sample : {index}\ninput: {" ".join(inp)}\ntarget: { " ".join(targ)}\nprediction: {" ".join(pred)}\nbleu: {blue_score}\n\n' ) blue_scores.append(blue_score) if 0 < num_sentences <= index - 1: break if 0 < num_batches <= batch_num - 1: break print(f'mean bleu score: {np.mean(blue_scores)}')
def get_segmentation_learner(dls, number_classes, segmentation_type, architecture_name, backbone_name, loss_func=None, opt_func=Adam, lr=defaults.lr, splitter=trainable_params, cbs=None, pretrained=True, normalize=True, image_size=None, metrics=None, path=None, model_dir='models', wd=None, wd_bn_bias=False, train_bn=True, moms=(0.95, 0.85, 0.95)): """This function return a learner for the provided architecture and backbone Parameters: dls (DataLoader): the dataloader to use with the learner number_classes (int): the number of clases in the project. It should be >=2 segmentation_type (str): just `Semantic Segmentation` accepted for now architecture_name (str): name of the architecture. The following ones are supported: `unet`, `deeplabv3+`, `hrnet`, `maskrcnn` and `u2^net` backbone_name (str): name of the backbone loss_func (): loss function. opt_func (): opt function. lr (): learning rates splitter (): splitter function for freazing the learner cbs (List[cb]): list of callbacks pretrained (bool): it defines if a trained backbone is needed normalize (bool): image_size (int): REQUIRED for MaskRCNN. It indicates the desired size of the image. metrics (List[metric]): list of metrics path (): path parameter model_dir (str): the path in which save models wd (float): wieght decay wd_bn_bias (bool): train_bn (bool): moms (Tuple(float)): tuple of different momentuns Returns: learner: value containing the learner object """ number_classes_name = "" if number_classes == 2: number_classes_name = "binary" elif number_classes > 2: number_classes_name = "multiple" else: raise Exception("The number of classes must be >=2") check_architecture_configuration(number_classes=number_classes_name, segmentation_type=segmentation_type, architecture_name=architecture_name, backbone_name=backbone_name) learner = None if architecture_name == "unet": # TODO -> Revisar arch learner = unet_learner(dls=dls, arch=unet_backbone_name[backbone_name], metrics=metrics, wd=wd, loss_func=loss_func, opt_func=opt_func, lr=lr, splitter=splitter, cbs=cbs, path=path, model_dir=model_dir, wd_bn_bias=wd_bn_bias, train_bn=train_bn, pretrained=pretrained, normalize=normalize, moms=moms) elif architecture_name == "deeplabv3+": model = DeepLabV3Plus(backbone_name=backbone_name, nclass=number_classes, pretrained=pretrained) learner = Learner(dls=dls, model=model, loss_func=loss_func, opt_func=opt_func, lr=lr, splitter=splitter, cbs=cbs, metrics=metrics, path=path, model_dir=model_dir, wd=wd, wd_bn_bias=wd_bn_bias, train_bn=train_bn) elif architecture_name == "hrnet": model = HRNet(nclass=number_classes, backbone_name=backbone_name, pretrained=pretrained) learner = Learner(dls=dls, model=model, loss_func=loss_func, opt_func=opt_func, lr=lr, splitter=splitter, cbs=cbs, metrics=metrics, path=path, model_dir=model_dir, wd=wd, wd_bn_bias=wd_bn_bias, train_bn=train_bn) elif architecture_name == "maskrcnn": if image_size is None: raise Exception( "MaskRCNN need to define image_size. This values are for reescaling the image" ) model = maskrcnn_resnet50_fpn(num_classes=number_classes, min_size=image_size, max_size=image_size) learner = mask_rcnn.MaskRCNNLearner(dls=dls, model=model, loss_func=loss_func, opt_func=opt_func, lr=lr, splitter=splitter, cbs=cbs, metrics=metrics, path=path, model_dir=model_dir, wd=wd, wd_bn_bias=wd_bn_bias, train_bn=train_bn) elif architecture_name == "u2^net": model = None if backbone_name == "small": model = u2net.U2NETP(3, 1) elif backbone_name == "normal": model = u2net.U2NET(3, 1) learner = u2net.USquaredNetLearner(dls=dls, model=model, opt_func=opt_func, lr=lr, splitter=splitter, cbs=cbs, metrics=metrics, path=path, model_dir=model_dir, wd=wd, wd_bn_bias=wd_bn_bias, train_bn=train_bn) return learner
x = F.relu(self.conv2(x)) x = x.view(-1, 320) x = F.relu(self.fc1(x)) x = F.log_softmax(self.fc2(x), dim=1) return x # %% net = ConvNet() print(summary(net, torch.zeros((1, 1, 28, 28)), show_input=True)) print(summary(net, torch.zeros((1, 1, 28, 28)), show_input=False)) # %% learner = Learner( mnist_dls, ConvNet(), loss_func=F.nll_loss, metrics=[accuracy, Precision(average="macro"), Recall(average="macro")], ) # %%[markdown] # # These are too many epochs, but we want to see the behavior of the net when it # is trained for some time. # %% learner.fit(n_epoch=20) # %% pprint(list(learner.metrics)) # %%
def run_training(learn: Learner, min_lr=0.05, head_runs=1, full_runs=1): monitor.display_average_stats_per_gpu() print(f"Training for {head_runs}+{full_runs} epochs at min LR {min_lr}") learn.fine_tune(full_runs, min_lr, freeze_epochs=head_runs)