def get_validation_objf( dataloader: torch.utils.data.DataLoader, model: AcousticModel, ali_model: Optional[AcousticModel], P: k2.Fsa, device: torch.device, graph_compiler: MmiTrainingGraphCompiler, scaler: GradScaler, den_scale: float = 1, ): total_objf = 0. total_frames = 0. # for display only total_all_frames = 0. # all frames including those seqs that failed. model.eval() from torchaudio.datasets.utils import bg_iterator for batch_idx, batch in enumerate(bg_iterator(dataloader, 2)): objf, frames, all_frames = get_objf(batch=batch, model=model, ali_model=ali_model, P=P, device=device, graph_compiler=graph_compiler, is_training=False, is_update=False, den_scale=den_scale, scaler=scaler) total_objf += objf total_frames += frames total_all_frames += all_frames return total_objf, total_frames, total_all_frames
def validate(model, criterion, data_loader, device, epoch): with torch.no_grad(): model.eval() sums = defaultdict(lambda: 0.0) start = time() for waveform, specgram, target in bg_iterator(data_loader, maxsize=2): waveform = waveform.to(device) specgram = specgram.to(device) target = target.to(device) output = model(waveform, specgram) output, target = output.squeeze(1), target.squeeze(1) loss = criterion(output, target) sums["loss"] += loss.item() avg_loss = sums["loss"] / len(data_loader) metric = MetricLogger("validation") metric["epoch"] = epoch metric["loss"] = avg_loss metric["time"] = time() - start metric() return avg_loss
def test_bg_iterator(self): data = COMMONVOICE(self.path, version="cv-corpus-4-2019-12-10", language="tatar") data = dataset_utils.bg_iterator(data, 5) for _ in data: pass
def train_one_epoch(model, criterion, optimizer, data_loader, device, epoch): model.train() sums = defaultdict(lambda: 0.0) start1 = time() metric = MetricLogger("train_iteration") metric["epoch"] = epoch for waveform, specgram, target in bg_iterator(data_loader, maxsize=2): start2 = time() waveform = waveform.to(device) specgram = specgram.to(device) target = target.to(device) output = model(waveform, specgram) output, target = output.squeeze(1), target.squeeze(1) loss = criterion(output, target) loss_item = loss.item() sums["loss"] += loss_item metric["loss"] = loss_item optimizer.zero_grad() loss.backward() if args.clip_grad > 0: gradient = torch.nn.utils.clip_grad_norm_( model.parameters(), args.clip_grad ) sums["gradient"] += gradient.item() metric["gradient"] = gradient.item() optimizer.step() metric["iteration"] = sums["iteration"] metric["time"] = time() - start2 metric() sums["iteration"] += 1 avg_loss = sums["loss"] / len(data_loader) metric = MetricLogger("train_epoch") metric["epoch"] = epoch metric["loss"] = sums["loss"] / len(data_loader) metric["gradient"] = avg_loss metric["time"] = time() - start1 metric()
def evaluate( model, criterion, data_loader, decoder, language_model, device, epoch, disable_logger=False, ): with torch.no_grad(): model.eval() start = time() metric = MetricLogger("validation", disable=disable_logger) metric["epoch"] = epoch for inputs, targets, tensors_lengths, target_lengths in bg_iterator( data_loader, maxsize=2): inputs = inputs.to(device, non_blocking=True) targets = targets.to(device, non_blocking=True) # keep batch first for data parallel outputs = model(inputs).transpose(-1, -2).transpose(0, 1) # CTC # outputs: input length, batch size, number of classes (including blank) # targets: batch size, max target length # input_lengths: batch size # target_lengths: batch size metric["cumulative loss"] += criterion(outputs, targets, tensors_lengths, target_lengths).item() metric["dataset length"] += len(inputs) metric["iteration"] += 1 compute_error_rates(outputs, targets, decoder, language_model, metric) metric[ "average loss"] = metric["cumulative loss"] / metric["iteration"] metric["validation time"] = time() - start metric() return metric["average loss"]
def test_commonvoice_bg(self): data = COMMONVOICE(self.path, url="tatar") data = bg_iterator(data, 5) for _ in data: pass
def test_bg_iterator(self): data = COMMONVOICE(self.path, url="tatar") data = dataset_utils.bg_iterator(data, 5) for _ in data: pass
def test_bg_iterator(self): data = dataset_utils.bg_iterator(Dataset(), 5) for _ in data: pass
def train_one_epoch( model, criterion, optimizer, scheduler, data_loader, decoder, language_model, device, epoch, clip_grad, disable_logger=False, reduce_lr_on_plateau=False, ): model.train() metric = MetricLogger("train", disable=disable_logger) metric["epoch"] = epoch for inputs, targets, tensors_lengths, target_lengths in bg_iterator( data_loader, maxsize=2): start = time() inputs = inputs.to(device, non_blocking=True) targets = targets.to(device, non_blocking=True) # keep batch first for data parallel outputs = model(inputs).transpose(-1, -2).transpose(0, 1) # CTC # outputs: input length, batch size, number of classes (including blank) # targets: batch size, max target length # input_lengths: batch size # target_lengths: batch size loss = criterion(outputs, targets, tensors_lengths, target_lengths) optimizer.zero_grad() loss.backward() if clip_grad > 0: metric["gradient"] = torch.nn.utils.clip_grad_norm_( model.parameters(), clip_grad) optimizer.step() compute_error_rates(outputs, targets, decoder, language_model, metric) try: metric["lr"] = scheduler.get_last_lr()[0] except AttributeError: metric["lr"] = optimizer.param_groups[0]["lr"] metric["batch size"] = len(inputs) metric["n_channel"] = inputs.shape[1] metric["n_time"] = inputs.shape[-1] metric["dataset length"] += metric["batch size"] metric["iteration"] += 1 metric["loss"] = loss.item() metric["cumulative loss"] += metric["loss"] metric[ "average loss"] = metric["cumulative loss"] / metric["iteration"] metric["iteration time"] = time() - start metric["epoch time"] += metric["iteration time"] metric() if reduce_lr_on_plateau and isinstance(scheduler, ReduceLROnPlateau): scheduler.step(metric["average loss"]) elif not isinstance(scheduler, ReduceLROnPlateau): scheduler.step()
def test_commonvoice_bg(self): path = os.path.join(self.path, "commonvoice") data = COMMONVOICE(path, "train.tsv", "tatar") data = bg_iterator(data, 5) for d in data: pass
'history_training': history_training, 'history_validation': history_validation, }, False) # In[ ]: with tqdm(total=args.epochs, unit_scale=1, disable=args.distributed) as pbar: for epoch in range(args.start_epoch, args.epochs): torch.cuda.reset_max_memory_allocated() model.train() sum_loss = 0. total_norm = 0. for inputs, targets, tensors_lengths, target_lengths in bg_iterator(loader_training, maxsize=2): loss = forward_loss( inputs, targets, tensors_lengths, target_lengths) sum_loss += loss.item() optimizer.zero_grad() loss.backward() norm = 0. if clip_norm > 0: norm = torch.nn.utils.clip_grad_norm_( model.parameters(), clip_norm) total_norm += norm elif args.gradient: for p in list(filter(lambda p: p.grad is not None, model.parameters())):