Пример #1
0
def download_from_url(url, dst):
    """
    kindly used from https://gist.github.com/wy193777/0e2a4932e81afc6aa4c8f7a2984f34e2
    @param: url to download file
    @param: dst place to put the file
    """

    file_size = int(requests.head(url).headers["Content-Length"])
    if os.path.exists(dst):
        first_byte = os.path.getsize(dst)
    else:
        first_byte = 0
    if first_byte >= file_size:
        return file_size
    header = {"Range": "bytes=%s-%s" % (first_byte, file_size)}
    pbar = tqdm(
        total=file_size, initial=first_byte,
        unit='B', unit_scale=True, desc=url.split('/')[-1])
    req = requests.get(url, headers=header, stream=True)
    with(open(dst, 'ab')) as f:
        for chunk in req.iter_content(chunk_size=1024):
            if chunk:
                f.write(chunk)
                pbar.update(1024)
    pbar.close()
    return file_size
Пример #2
0
def reduce_noise(
    audio_clip,
    noise_clip,
    n_grad_freq=2,
    n_grad_time=4,
    n_fft=2048,
    win_length=2048,
    hop_length=512,
    n_std_thresh=1.5,
    prop_decrease=1.0,
    pad_clipping=True,
    use_tensorflow=False,
    verbose=False,
):
    """Remove noise from audio based upon a clip containing only noise

    Args:
        audio_clip (array): The first parameter.
        noise_clip (array): The second parameter.
        n_grad_freq (int): how many frequency channels to smooth over with the mask.
        n_grad_time (int): how many time channels to smooth over with the mask.
        n_fft (int): number audio of frames between STFT columns.
        win_length (int): Each frame of audio is windowed by `window()`. The window will be of length `win_length` and then padded with zeros to match `n_fft`..
        hop_length (int):number audio of frames between STFT columns.
        n_std_thresh (int): how many standard deviations louder than the mean dB of the noise (at each frequency level) to be considered signal
        prop_decrease (float): To what extent should you decrease noise (1 = all, 0 = none)
        pad_clipping (bool): Pad the signals with zeros to ensure that the reconstructed data is equal length to the data
        use_tensorflow (bool): Use tensorflow as a backend for convolution and fft to speed up computation
        verbose (bool): Whether to plot the steps of the algorithm

    Returns:
        array: The recovered signal with noise subtracted

    """
    # load tensorflow if you are using it as a backend
    if use_tensorflow:
        use_tensorflow = load_tensorflow(verbose)

    if verbose:
        pbar = tqdm(total=7)
    else:
        pbar = None

    update_pbar(pbar, "STFT on noise")
    # STFT over noise
    noise_stft = _stft(noise_clip,
                       n_fft,
                       hop_length,
                       win_length,
                       use_tensorflow=use_tensorflow)
    noise_stft_db = _amp_to_db(np.abs(noise_stft))  # convert to dB
    # Calculate statistics over noise
    update_pbar(pbar, "STFT on signal")
    mean_freq_noise = np.mean(noise_stft_db, axis=1)
    std_freq_noise = np.std(noise_stft_db, axis=1)
    noise_thresh = mean_freq_noise + std_freq_noise * n_std_thresh
    # STFT over signal
    update_pbar(pbar, "STFT on signal")

    # pad signal with zeros to avoid extra frames being clipped if desired
    if pad_clipping:
        nsamp = len(audio_clip)
        audio_clip = np.pad(audio_clip, [0, hop_length], mode="constant")

    sig_stft = _stft(audio_clip,
                     n_fft,
                     hop_length,
                     win_length,
                     use_tensorflow=use_tensorflow)
    # spectrogram of signal in dB
    sig_stft_db = _amp_to_db(np.abs(sig_stft))
    update_pbar(pbar, "Generate mask")

    # calculate the threshold for each frequency/time bin
    db_thresh = np.repeat(
        np.reshape(noise_thresh, [1, len(mean_freq_noise)]),
        np.shape(sig_stft_db)[1],
        axis=0,
    ).T
    # mask if the signal is above the threshold
    sig_mask = sig_stft_db < db_thresh
    update_pbar(pbar, "Smooth mask")
    # Create a smoothing filter for the mask in time and frequency
    smoothing_filter = _smoothing_filter(n_grad_freq, n_grad_time)

    # convolve the mask with a smoothing filter
    sig_mask = convolve_gaussian(sig_mask, smoothing_filter, use_tensorflow)

    sig_mask = sig_mask * prop_decrease
    update_pbar(pbar, "Apply mask")
    # mask the signal

    sig_stft_amp = mask_signal(sig_stft, sig_mask)

    update_pbar(pbar, "Recover signal")
    # recover the signal
    recovered_signal = _istft(sig_stft_amp,
                              n_fft,
                              hop_length,
                              win_length,
                              use_tensorflow=use_tensorflow)
    # fix the recovered signal length if padding signal
    if pad_clipping:
        recovered_signal = librosa.util.fix_length(recovered_signal, nsamp)

    recovered_spec = _amp_to_db(
        np.abs(
            _stft(
                recovered_signal,
                n_fft,
                hop_length,
                win_length,
                use_tensorflow=use_tensorflow,
            )))
    if verbose:
        plot_reduction_steps(
            noise_stft_db,
            mean_freq_noise,
            std_freq_noise,
            noise_thresh,
            smoothing_filter,
            sig_stft_db,
            sig_mask,
            recovered_spec,
        )
    return recovered_signal
Пример #3
0
def train_simultaneous_decoder_objectives(encoder,
                                          decoder,
                                          train_dl,
                                          test_dl,
                                          Y,
                                          Y_test,
                                          Y_test_avg,
                                          epochs=config.num_multi_epochs):
    global NUM_VOXELS
    # encoder.eval()
    encoder.eval()
    encoder.trainable = False
    decoder.train()
    decoder.trainable = True
    print(decoder)
    Y = torch.from_numpy(Y).float()  #.cuda()
    # Y = Y.reshape(-1, NUM_VOXELS, 1, 1) # turn fmri into 1 x NUMVOXELS grayscale image
    Y_test = torch.from_numpy(Y_test).float()
    Y_test_avg = torch.from_numpy(Y_test_avg).float()  #.cuda()
    test_fmri_dl = make_test_fmri_dl(Y_test_avg)

    msecriterion = nn.MSELoss()
    # maecriterion = nn.L1Loss()
    # ssimcriterion = piq.SSIMLoss(data_range=1.)#
    # ssimcriterion = pytorch_ssim.SSIM()
    # perceptualcriterion = lpips.LPIPS(net='alex').cuda()
    # mdsicriterion = piqa.MDSI().cuda()
    coscriterion = nn.CosineSimilarity()

    # enc_optim = optim.AdamW(lr=0, params=encoder.parameters())
    optimizer = optim.Adam(
        lr=1e-3,
        params=list(decoder.parameters())  # + list(encoder.parameters())
        ,
        weight_decay=1e-3)

    # scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=5, gamma=0.8)

    epoch_dec_losses = []
    epoch_decenc_losses = []
    epoch_encdec_losses = []

    imagenet = imagenet_dl()
    scaler = torch.cuda.amp.GradScaler(enabled=True)
    objectives = ["d"] * 80 + [
        "ed"
    ] * 20  # ["d"] * 60 + ["de"] * 10 + ["ed"] * 30 + ["gan"] * 0
    for epoch in tqdm(range(epochs)):
        decoder.trainable = True
        decoder.train()
        dec_losses = []
        decenc_losses = []
        encdec_losses = []

        for i, batch in enumerate(train_dl):
            # TODO:
            # - use test set of MRIs in decenc
            # - transformer decoder?
            # - imagenet val set in encdec

            inputs, mris, idxs = batch
            batch_size = len(inputs)
            inputs = inputs.permute(0, 3, 1, 2).float().cuda()
            # Y_batch = Y[idxs].cuda() # if we go to next batch in train_dl, but then pick random objective of training decenc on testset statistics fmri, we're going to be biased against training on the trainset mri->mri mapping..
            Y_batch = mris.float().cuda()

            # not sure why it is so memory intensive to do all 3.. doing a random choice of any
            objective = random.choice(objectives) if epoch > 0 else "d"

            # enc_optim.zero_grad()

            # dec:
            # D: fMRI -> image

            if objective == "d":
                with torch.cuda.amp.autocast():
                    dec_outputs = decoder(
                        Y_batch).float().cuda()  # [b, c, h, w]
                    # print(dec_outputs.shape, inputs.shape)
                    dec_loss = msecriterion(dec_outputs, inputs)
                    # dec_loss = mdsicriterion(dec_outputs, inputs.permute(0, 3, 1, 2))
                    # dec_loss += maecriterion(dec_outputs, inputs)# + ssimcriterion(dec_outputs, inputs.permute(0, 3, 1, 2))
                    # dec_loss -= ssimcriterion(dec_outputs, inputs)
                    # dec_loss += perceptualcriterion(dec_outputs, inputs)
                    # perceptualloss = perceptualcriterion.forward(dec_outputs, inputs, normalize=True).cuda()
                    # dec_loss += 0.01 * torch.sum(perceptualloss)
                    # msecriterion(dec_outputs.permute(0, 2, 3, 1), inputs) + -ssimcriterion(dec_outputs.permute(0, 2, 3, 1), inputs) \
                    #

                    loss = dec_loss
                    dec_losses.append(dec_loss.item())
            # print("d", dec_outputs.permute(0, 2, 3, 1).shape, inputs.shape)

            # decenc:
            # E . D: mri -> mri
            elif objective == "de":
                fmri_set = random.choice(["trainset", "testset"])
                if fmri_set == "testset":
                    print(">testset fmri")
                    del Y_batch
                    Y_batch = next(iter(test_fmri_dl)).float().cuda()
                with torch.cuda.amp.autocast():
                    dec_outputs = decoder(
                        Y_batch).float().cuda()  # [b, c, h, w]
                    decenc_outputs = encoder(
                        dec_outputs)  #.reshape(batch_size, NUM_VOXELS, 1, 1)
                    decenc_loss = msecriterion(decenc_outputs, Y_batch)
                    decenc_loss += (
                        1 - torch.mean(coscriterion(decenc_outputs, Y_batch)))
                    loss = decenc_loss
                    decenc_losses.append(decenc_loss.item())
            # print("de", decenc_outputs.shape, Y_batch.shape)

            # encdec:
            # D. E: img -> img

            elif objective == "ed":
                # enc: b h w c -> b c h w ->
                # dec then b c h w -> b h w c
                img_src = random.choice(["trainset", "trainset", "imagenet"])
                if img_src == "imagenet":
                    print(">imagenet batch")
                    del inputs
                    inputs = next(iter(imagenet)).float().cuda()

                with torch.cuda.amp.autocast():
                    encdec_outputs = decoder(
                        encoder(
                            inputs)  #.reshape(batch_size, NUM_VOXELS, 1, 1)
                    )
                    encdec_loss = msecriterion(encdec_outputs, inputs)
                    # encdec_loss += perceptualcriterion(dec_outputs, inputs)
                    # encdec_loss = mdsicriterion(encdec_outputs, inputs.permute(0, 3, 1, 2))
                    # encdec_loss += maecriterion(encdec_outputs, inputs)
                    # encdec_loss -= ssimcriterion(encdec_outputs, inputs)
                    # encdec_loss = contentloss(encdec_outputs, inputs.permute(0, 3, 1, 2))
                    # msecriterion(encdec_outputs, inputs) -ssimcriterion(encdec_outputs, inputs)

                    loss = encdec_loss
                    encdec_losses.append(encdec_loss.item())
            # print("ed", encdec_outputs.shape, inputs.shape)

            elif objective == "gan":
                pass

            # loss = torch.sum(dec_loss) + torch.sum(decenc_loss) + torch.sum(encdec_loss)
            # scaled_grad_params = torch.autograd.grad(outputs=scaler.scale(loss),
            #                                         inputs=decoder.parameters(),
            #                                         create_graph=True
            # )

            # inv_scale = 1./scaler.get_scale()
            # grad_params = [p * inv_scale for p in scaled_grad_params]

            # with torch.cuda.amp.autocast():
            #     grad_norm = 0
            #     for grad in grad_params:
            #         grad_norm += grad.pow(2).sum()
            #     grad_norm = grad_norm.sqrt()
            #     loss = loss + grad_norm

            scaler.scale(loss).backward()
            scaler.step(optimizer)
            scaler.update()
            # scheduler.step()

            optimizer.zero_grad()
            # enc_optim.step()

        print(
            f"epoch {epoch} mri->img: {np.mean(dec_losses)}  mri->mri: {np.mean(decenc_losses)}   img->img: {np.mean(encdec_losses)}"
        )
        epoch_dec_losses.append(np.mean(dec_losses))
        epoch_decenc_losses.append(np.mean(decenc_losses))
        epoch_encdec_losses.append(np.mean(encdec_losses))

        # if epoch % 5 == 0:
        #     with torch.no_grad():
        #         eval_decoder(decoder, test_dl, X_test, Y_test_avg, avg=True)

        if epoch % 20 == 0:
            print("running through whole un-averaged testset")
            with torch.no_grad():
                # eval_decoder(decoder, test_dl, X_test, Y_test_avg, Y_test=Y, avg=False)
                decode_test_set(test_dl, Y_test, X, decoder)

        if epoch % 20 == 0:
            print("dumping trainset results")
            with torch.no_grad():
                decode_training_set(train_dl, Y, X, decoder)

    import matplotlib.pyplot as plt
    # plt.plot(range(len(epoch_dec_losses)), epoch_dec_losses, label='dec')
    # plt.plot(range(len(epoch_decenc_losses)), epoch_decenc_losses, label='decenc')
    # plt.plot(range(len(epoch_encdec_losses)), epoch_encdec_losses, label='encdec')
    # plt.legend()
    # plt.show()
    return decoder
Пример #4
0
from tqdm.autonotebook import tqdm
import time
x = range(10000)
tk0 = tqdm(x, total=len(x))

for i in tk0:
    time.sleep(0.1)
    def fit(self, X, lazy=False, collector=False):
        """
        Build the SubCMedians model associated to dataset $X$

        Parameters
        ----------
        X : numpy.ndarray
            Data set to be clustered, rows represent instances (points) and
            columns represent features (dimensions)
        lazy: bool, default=False
            If true only tries to update the model iif the SAE increased when
            the datasample was updated, otherwise the optimization iteration is
            permormed at each sample update
        collector: bool, default=False
            If true all the models, and conserved modifications are kept.

        Returns
        -------
        subcmedians
            Fitted subcmedians instance

        """
        X = np.asarray(X)
        if self.random_state is not None:
            np.random.seed(self.random_state)
        industrius = (not lazy)
        if X.shape[0] < self.H:
            self.H = X.shape[0] - 1
        self.S = X[:self.H, :].copy()
        self._sae_history = []
        self._nb_centers_history = []
        self._genome_size_history = []
        if collector:
            self.subspaces_history = []
            self.cluster_centers_history = []
            self.changes_accepted_history = []
            self.changes_history = []
            self.gain_sae_history = []
        i = self.H
        h = 0
        self._model_candidate(self.S[h])
        sae = self.sae_score(self.S)
        for t in tqdm(range(self.nb_iter)):
            # update dataset and SAE
            ae_old_point = self.sae_score(self.S[h, :])
            point = X[i, :]
            self.S[h, :] = point
            ae_new_point = self.sae_score(self.S[h, :])
            sae = sae - ae_old_point + ae_new_point
            # generate candidate
            if industrius or ae_old_point < ae_new_point:
                self._model_candidate(point)
            if collector:
                self.changes_history.append([
                    self.model._candidate_deletion,
                    self.model._candidate_insertion
                ])
            sae_ = self.sae_score(self.S)
            gain = sae - sae_
            if gain >= 0:
                sae = sae_
                self.model.apply_changes()
                if collector:
                    self.changes_accepted_history.append(1)
            else:
                self.model.reverse_changes()
                if collector:
                    self.changes_accepted_history.append(0)
            self._sae_history.append(sae)
            self._nb_centers_history.append(self.model.geno.nb_centers)
            self._genome_size_history.append(self.model.geno.G)
            if collector:
                self.subspaces_history.append(self.model.geno.to_pandas())
                self.cluster_centers_history.append(
                    self.model.pheno.to_pandas())
                self.gain_sae_history.append(gain)
            h = (h + 1) % self.H
            i = (i + 1) % X.shape[0]
        self.cluster_centers_ = self.model.get_cluster_centers()
        self.subspaces_ = self.model.get_cluster_subspaces()
        self.sae_ = sae
        return (self)
Пример #6
0
    def range_test(
        self,
        train_loader,
        val_loader=None,
        start_lr=None,
        end_lr=10,
        num_iter=100,
        step_mode="exp",
        smooth_f=0.05,
        diverge_th=5,
        accumulation_steps=1,
    ):
        """Performs the learning rate range test.
        Arguments:
            train_loader (torch.utils.data.DataLoader): the training set data laoder.
            val_loader (torch.utils.data.DataLoader, optional): if `None` the range test
                will only use the training loss. When given a data loader, the model is
                evaluated after each iteration on that dataset and the evaluation loss
                is used. Note that in this mode the test takes significantly longer but
                generally produces more precise results. Default: None.
            start_lr (float, optional): the starting learning rate for the range test.
                Default: None (uses the learning rate from the optimizer).
            end_lr (float, optional): the maximum learning rate to test. Default: 10.
            num_iter (int, optional): the number of iterations over which the test
                occurs. Default: 100.
            step_mode (str, optional): one of the available learning rate policies,
                linear or exponential ("linear", "exp"). Default: "exp".
            smooth_f (float, optional): the loss smoothing factor within the [0, 1[
                interval. Disabled if set to 0, otherwise the loss is smoothed using
                exponential smoothing. Default: 0.05.
            diverge_th (int, optional): the test is stopped when the loss surpasses the
                threshold:  diverge_th * best_loss. Default: 5.
            accumulation_steps (int, optional): steps for gradient accumulation. If it
                is 1, gradients are not accumulated. Default: 1.
        Example (fastai approach):
            >>> lr_finder = LRFinder(net, optimizer, criterion, device="cuda")
            >>> lr_finder.range_test(dataloader, end_lr=100, num_iter=100)
        Example (Leslie Smith's approach):
            >>> lr_finder = LRFinder(net, optimizer, criterion, device="cuda")
            >>> lr_finder.range_test(trainloader, val_loader=val_loader, end_lr=1, num_iter=100, step_mode="linear")
        Gradient accumulation is supported; example:
            >>> train_data = ...    # prepared dataset
            >>> desired_bs, real_bs = 32, 4         # batch size
            >>> accumulation_steps = desired_bs // real_bs     # required steps for accumulation
            >>> dataloader = torch.utils.data.DataLoader(train_data, batch_size=real_bs, shuffle=True)
            >>> acc_lr_finder = LRFinder(net, optimizer, criterion, device="cuda")
            >>> acc_lr_finder.range_test(dataloader, end_lr=10, num_iter=100, accumulation_steps=accumulation_steps)
        Reference:
        [Training Neural Nets on Larger Batches: Practical Tips for 1-GPU, Multi-GPU & Distributed setups](
        https://medium.com/huggingface/ec88c3e51255)
        [thomwolf/gradient_accumulation](https://gist.github.com/thomwolf/ac7a7da6b1888c2eeac8ac8b9b05d3d3)
        """

        # Reset test results
        self.history = {"lr": [], "loss": []}
        self.best_loss = None

        # Move the model to the proper device
        self.model.to(self.device)

        # Check if the optimizer is already attached to a scheduler
        self._check_for_scheduler()

        # Set the starting learning rate
        if start_lr:
            self._set_learning_rate(start_lr)

        # Initialize the proper learning rate policy
        if step_mode.lower() == "exp":
            lr_schedule = ExponentialLR(self.optimizer, end_lr, num_iter)
        elif step_mode.lower() == "linear":
            lr_schedule = LinearLR(self.optimizer, end_lr, num_iter)
        else:
            raise ValueError(
                "expected one of (exp, linear), got {}".format(step_mode))

        if smooth_f < 0 or smooth_f >= 1:
            raise ValueError("smooth_f is outside the range [0, 1[")

        # Create an iterator to get data batch by batch
        iter_wrapper = DataLoaderIterWrapper(train_loader)
        for iteration in tqdm(range(num_iter)):
            # Train on batch and retrieve loss
            loss, accu = self._train_batch(iter_wrapper, accumulation_steps)
            if val_loader:
                loss = self._validate(val_loader)

            # Update the learning rate
            lr_schedule.step()
            self.history["lr"].append(lr_schedule.get_lr()[0])
            self.accuracy["lr"].append(lr_schedule.get_lr()[0])

            # Track the best loss and smooth it if smooth_f is specified
            if iteration == 0:
                self.best_loss = loss
                self.best_Accu = accu
                self.max_lr = lr_schedule.get_lr()[0]
            else:
                if smooth_f > 0:
                    loss = smooth_f * loss + (
                        1 - smooth_f) * self.history["loss"][-1]
                if loss < self.best_loss:
                    self.best_loss = loss
                if accu > self.best_Accu:
                    self.best_Accu = accu
                    self.max_lr = lr_schedule.get_lr()[0]

            # Check if the loss has diverged; if it has, stop the test
            self.history["loss"].append(loss)
            self.accuracy["Accuracy"].append(accu)
            #if loss > diverge_th * self.best_loss:
            #    print("Stopping early, the loss has diverged")
            #    break

        print(
            "Learning rate search finished. See the graph with {finder_name}.plot()"
        )
        print("Max Accuracy = " + str(self.best_Accu) + " at LR = " +
              str(self.max_lr))
Пример #7
0
    def evaluate(self,
                 eval_data,
                 load_best_model=True,
                 model_file=None,
                 show_progress=False):
        r"""Evaluate the model based on the eval data.

        Args:
            eval_data (DataLoader): the eval data
            load_best_model (bool, optional): whether load the best model in the training process, default: True.
                                              It should be set True, if users want to test the model after training.
            model_file (str, optional): the saved model file, default: None. If users want to test the previously
                                        trained model file, they can set this parameter.
            show_progress (bool): Show the progress of evaluate epoch. Defaults to ``False``.

        Returns:
            dict: eval result, key is the eval metric and value in the corresponding metric value.
        """
        if not eval_data:
            return

        if load_best_model:
            if model_file:
                checkpoint_file = model_file
            else:
                checkpoint_file = self.saved_model_file
            checkpoint = torch.load(checkpoint_file)
            self.model.load_state_dict(checkpoint['state_dict'])
            message_output = 'Loading model structure and parameters from {}'.format(
                checkpoint_file)
            self.logger.info(message_output)

        self.model.eval()

        if eval_data.dl_type == DataLoaderType.FULL:
            if self.item_tensor is None:
                self.item_tensor = eval_data.get_item_feature().to(
                    self.device).repeat(eval_data.step)
            self.tot_item_num = eval_data.dataset.item_num

        batch_matrix_list = []
        iter_data = (tqdm(
            enumerate(eval_data),
            total=len(eval_data),
            desc=set_color(f"Evaluate   ", 'pink'),
        ) if show_progress else enumerate(eval_data))
        for batch_idx, batched_data in iter_data:
            if eval_data.dl_type == DataLoaderType.FULL:
                interaction, scores = self._full_sort_batch_eval(batched_data)
            else:
                interaction = batched_data
                batch_size = interaction.length
                if batch_size <= self.test_batch_size:
                    scores = self.model.predict(interaction.to(self.device))
                else:
                    scores = self._spilt_predict(interaction, batch_size)

            batch_matrix = self.evaluator.collect(interaction, scores)
            batch_matrix_list.append(batch_matrix)
        result = self.evaluator.evaluate(batch_matrix_list, eval_data)

        return result
Пример #8
0
text = ("funny comedy music laugh humor song songs jokes musical hilarious")
doc = nlp(text)

for token1, token2 in combinations(doc, 2):
    print(
        f"similarity between {token1} and {token2} is {token1.similarity(token2)}"
    )

# %%
import pandas as pd
from gensim.models import Word2Vec
from tqdm.autonotebook import tqdm

data = pd.read_csv("train.csv")
sentences = []
for review in tqdm(data["review"]):
    doc = nlp(review, disable=["tagger", "ner"])
    for sent in doc.sents:
        sentences.append([t.text for t in sent])

# train a Word2vec model
model = Word2Vec(sentences)

# %%
for token1, token2 in combinations(text.split(), 2):
    print(
        f"similarity between {token1} and {token2} is {model.wv.similarity(token1, token2)}"
    )

# %%
Пример #9
0
final_output = []

# Instantiate TweetDataset with the test data
test_dataset = TweetDataset(tweet=df_test.text.values,
                            sentiment=df_test.sentiment.values,
                            selected_text=df_test.selected_text.values)

# Instantiate DataLoader with `test_dataset`
data_loader = torch.utils.data.DataLoader(test_dataset,
                                          shuffle=False,
                                          batch_size=VALID_BATCH_SIZE,
                                          num_workers=1)

# Turn of gradient calculations
with torch.no_grad():
    tk0 = tqdm(data_loader, total=len(data_loader), ncols=80)
    # Predict the span containing the sentiment for each batch
    for bi, d in enumerate(tk0):
        ids = d["ids"]
        token_type_ids = d["token_type_ids"]
        mask = d["mask"]
        sentiment = d["sentiment"]
        orig_selected = d["orig_selected"]
        orig_tweet = d["orig_tweet"]
        targets_start = d["targets_start"]
        targets_end = d["targets_end"]
        offsets = d["offsets"].numpy()

        ids = ids.to(device, dtype=torch.long)
        token_type_ids = token_type_ids.to(device, dtype=torch.long)
        mask = mask.to(device, dtype=torch.long)
Пример #10
0
    def generate_tiles_dataset(self,
                               output_dir=None,
                               save_format="jpg",
                               remove_first=True):
        """
            Actually generates training images from the dataset.sampled_tiles (= regions of interest)
            The filestructure is compatible with keras.ImageDataGenerator.flow_from_directory() method

            For more information on how to parse this, check this script:

            https://gist.github.com/fchollet/0830affa1f7f19fd47b06d4cf89ed44d

            In summary, this is our directory structure:

            ```markdown
            output_dir/
                aircrafts/
                    ac001.jpg
                    ac002.jpg
                    ...
                background/
                    bg001.jpg
                    bg002.jpg
                    ...
            ```

        Args:
            output_dir(str): the output path
            save_format: "jpg" the image format
            remove_first(bool): erase output dir first?

        Returns:

        """
        LOGGER.info(
            "Generating a dataset of tiles at location {}".format(output_dir))

        for label in self.found_labels:
            if remove_first:
                shutil.rmtree(os.path.join(output_dir, label))
            if not os.path.exists(os.path.join(output_dir, label)):
                os.makedirs(os.path.join(output_dir, label))

        def _generate_tiles(item, tiles):
            image = item.image
            tiles = list_utils.filter_tiles_by_item(tiles, item.key)
            for tile in tiles:
                tile_data = tile.get_data(image)
                tile_label = tile.label
                tile_basename = "{}_{}.{}".format(item.key, tile.key,
                                                  save_format)
                io.imsave(os.path.join(output_dir, tile_label, tile_basename),
                          tile_data)

        items = self.items
        sampled_tiles = self.sampled_tiles

        LOGGER.info("Dumping tiles to {}".format(output_dir))

        for item in tqdm(items, desc="Saving tiles to {}".format(output_dir)):
            _generate_tiles(item, sampled_tiles)
Пример #11
0
    def range_test(
        self,
        train_loader,
        val_loader=None,
        end_lr=10,
        num_iter=100,
        step_mode="exp",
        smooth_f=0.05,
        diverge_th=5,
    ):
        """Performs the learning rate range test.
        Arguments:
            train_loader (torch.utils.data.DataLoader): the training set data laoder.
            val_loader (torch.utils.data.DataLoader, optional): if `None` the range test
                will only use the training loss. When given a data loader, the model is
                evaluated after each iteration on that dataset and the evaluation loss
                is used. Note that in this mode the test takes significantly longer but
                generally produces more precise results. Default: None.
            end_lr (float, optional): the maximum learning rate to test. Default: 10.
            num_iter (int, optional): the number of iterations over which the test
                occurs. Default: 100.
            step_mode (str, optional): one of the available learning rate policies,
                linear or exponential ("linear", "exp"). Default: "exp".
            smooth_f (float, optional): the loss smoothing factor within the [0, 1[
                interval. Disabled if set to 0, otherwise the loss is smoothed using
                exponential smoothing. Default: 0.05.
            diverge_th (int, optional): the test is stopped when the loss surpasses the
                threshold:  diverge_th * best_loss. Default: 5.
        """
        # Reset test results
        self.history = {"lr": [], "loss": []}
        self.best_loss = None

        # Move the model to the proper device
        self.model.to(self.device)

        # Initialize the proper learning rate policy
        if step_mode.lower() == "exp":
            lr_schedule = ExponentialLR(self.optimizer, end_lr, num_iter)
        elif step_mode.lower() == "linear":
            lr_schedule = LinearLR(self.optimizer, end_lr, num_iter)
        else:
            raise ValueError(
                "expected one of (exp, linear), got {}".format(step_mode))

        if smooth_f < 0 or smooth_f >= 1:
            raise ValueError("smooth_f is outside the range [0, 1[")

        # Create an iterator to get data batch by batch
        iterator = iter(train_loader)
        for iteration in tqdm(range(num_iter)):
            # Get a new set of inputs and labels
            try:
                inputs, labels = next(iterator)
            except StopIteration:
                iterator = iter(train_loader)
                inputs, labels = next(iterator)

            # Train on batch and retrieve loss
            loss = self._train_batch(inputs, labels)
            if val_loader:
                loss = self._validate(val_loader)

            # Update the learning rate
            lr_schedule.step()
            self.history["lr"].append(lr_schedule.get_lr()[0])

            # Track the best loss and smooth it if smooth_f is specified
            if iteration == 0:
                self.best_loss = loss
            else:
                if smooth_f > 0:
                    loss = smooth_f * loss + (
                        1 - smooth_f) * self.history["loss"][-1]
                if loss < self.best_loss:
                    self.best_loss = loss

            # Check if the loss has diverged; if it has, stop the test
            self.history["loss"].append(loss)
            if loss > diverge_th * self.best_loss:
                print("Stopping early, the loss has diverged")
                break

        print(
            "Learning rate search finished. See the graph with {finder_name}.plot()"
        )
Пример #12
0
    def _parallel_analysis_component_selection(self,
                                               timeseries,
                                               L,
                                               K,
                                               rank,
                                               singular_values,
                                               iterations=100):
        '''
        Performs parallel analysis to help select the appropriate number of MSSA
        components to keep. The algorithm follows these steps:
        1. Calculate the eigenvalues via SVD/PCA on your real dataset.
        2. For a given number of iterations:
            3. Construct a random noise matrix the same shape as your real data.
            4. Perform decomposition of the random noise data.
            5. Calculate the eigenvalues for the noise data and track them per
               iteration.
        6. Calculate the percentile at a user-specified threshold of the noise
           eigenvalues at each position.
        7. Select only the number of components in the real data whose eigenvalues
           exceed those at the specified percentile of the noise eigenvalues.
        '''

        def _bootstrap_eigenvalues(ts_std, ts_shape, L, K, rank):

            # create random normal differences with equivalent standard deviations
            ts_rnorm = np.random.normal(
                np.zeros(ts_shape[1]),
                ts_std,
                size=ts_shape
            )

            # create noise trajectory matrix
            rnorm_trajectory_matrix = ts_matrix_to_trajectory_matrix(
                ts_rnorm,
                L,
                K
            )

            # decompose the noise trajectory matrix
            U, s, V, rank = decompose_trajectory_matrix(
                rnorm_trajectory_matrix,
                rank,
                svd_method=self.svd_method
            )

            # return the eigenvalues
            return s ** 2

        # calculate real eigenvalues
        eigenvalues = singular_values ** 2

        # calculate standard deviations column-wise
        ts_std = np.std(timeseries, axis=0)

        # bootstrap the eigenvalues
        noise_eigenvalues = [
            _bootstrap_eigenvalues(
                ts_std,
                timeseries.shape,
                L,
                K,
                rank
            )
            for i in tqdm(range(iterations), disable=(not self.verbose))
        ]
        noise_eigenvalues = np.concatenate(noise_eigenvalues, axis=0)

        # calculate the 95th percentile of the noise eigenvalues
        eig_pctl = np.percentile(noise_eigenvalues, 95, axis=0)

        # find the first index where the noise eigenvalue 95th percentile is >= real
        adjusted_rank = np.where(eig_pctl > eigenvalues)[0][0]

        return adjusted_rank
Пример #13
0
 def transform(self, df):
     return pd.Series({
         idx: self.transform_row_(row)
         for idx, row in tqdm(df.iterrows())
     })
Пример #14
0
    def start_training(self):
        if self.system_dict["params"]["num_gpus"] == 0:
            os.environ['CUDA_VISIBLE_DEVICES'] = '-1'

        if torch.cuda.is_available():
            torch.cuda.manual_seed(42)
        else:
            torch.manual_seed(42)

        self.system_dict["params"]["saved_path"] = self.system_dict["params"]["saved_path"] + "/" + self.system_dict["params"]["project_name"] + "/";
        self.system_dict["params"]["log_path"] = self.system_dict["params"]["log_path"] + "/" + self.system_dict["params"]["project_name"] + "/tensorboard/";
        os.makedirs(self.system_dict["params"]["saved_path"], exist_ok=True)
        os.makedirs(self.system_dict["params"]["log_path"], exist_ok=True)

        training_params = {'batch_size': self.system_dict["params"]["batch_size"],
                   'shuffle': True,
                   'drop_last': True,
                   'collate_fn': collater,
                   'num_workers': self.system_dict["params"]["num_workers"]}

        val_params = {'batch_size': self.system_dict["params"]["batch_size"],
                      'shuffle': False,
                      'drop_last': True,
                      'collate_fn': collater,
                      'num_workers': self.system_dict["params"]["num_workers"]}

        input_sizes = [512, 640, 768, 896, 1024, 1280, 1280, 1536]
        training_set = CocoDataset(self.system_dict["dataset"]["train"]["root_dir"], 
                                    self.system_dict["dataset"]["train"]["coco_dir"], 
                                    self.system_dict["dataset"]["train"]["img_dir"], 
                                    set_dir=self.system_dict["dataset"]["train"]["set_dir"],
                                    transform=transforms.Compose([Normalizer(mean=self.system_dict["params"]["mean"], std=self.system_dict["params"]["std"]),
                                                                 Augmenter(),
                                                                 Resizer(input_sizes[self.system_dict["params"]["compound_coef"]])]))
        training_generator = DataLoader(training_set, **training_params)

        if(self.system_dict["dataset"]["val"]["status"]):
            val_set = CocoDataset(self.system_dict["dataset"]["val"]["root_dir"], 
                                    self.system_dict["dataset"]["val"]["coco_dir"], 
                                    self.system_dict["dataset"]["val"]["img_dir"], 
                                    set_dir=self.system_dict["dataset"]["val"]["set_dir"],
                                    transform=transforms.Compose([Normalizer(self.system_dict["params"]["mean"], self.system_dict["params"]["std"]),
                                                             Resizer(input_sizes[self.system_dict["params"]["compound_coef"]])]))
            val_generator = DataLoader(val_set, **val_params)

        print("");
        print("");
        model = EfficientDetBackbone(num_classes=len(self.system_dict["params"]["obj_list"]), 
                                        compound_coef=self.system_dict["params"]["compound_coef"],
                                        ratios=eval(self.system_dict["params"]["anchors_ratios"]), 
                                        scales=eval(self.system_dict["params"]["anchors_scales"]));

        os.makedirs("pretrained_weights", exist_ok=True);

        if(self.system_dict["params"]["compound_coef"] == 0):
            if(not os.path.isfile(self.system_dict["params"]["load_weights"])):
                print("Downloading weights");
                cmd = "wget https://github.com/zylo117/Yet-Another-Efficient-Pytorch/releases/download/1.0/efficientdet-d0.pth -O " + \
                            self.system_dict["params"]["load_weights"];
                os.system(cmd);
        elif(self.system_dict["params"]["compound_coef"] == 1):
            if(not os.path.isfile(self.system_dict["params"]["load_weights"])):
                print("Downloading weights");
                cmd = "wget https://github.com/zylo117/Yet-Another-Efficient-Pytorch/releases/download/1.0/efficientdet-d1.pth -O " + \
                            self.system_dict["params"]["load_weights"]
                os.system(cmd);
        elif(self.system_dict["params"]["compound_coef"] == 2):
            if(not os.path.isfile(self.system_dict["params"]["load_weights"])):
                print("Downloading weights");
                cmd = "wget https://github.com/zylo117/Yet-Another-Efficient-Pytorch/releases/download/1.0/efficientdet-d2.pth -O " + \
                            self.system_dict["params"]["load_weights"]
                os.system(cmd);
        elif(self.system_dict["params"]["compound_coef"] == 3):
            if(not os.path.isfile(self.system_dict["params"]["load_weights"])):
                print("Downloading weights");
                cmd = "wget https://github.com/zylo117/Yet-Another-Efficient-Pytorch/releases/download/1.0/efficientdet-d3.pth -O " + \
                            self.system_dict["params"]["load_weights"]
                os.system(cmd);
        elif(self.system_dict["params"]["compound_coef"] == 4):
            if(not os.path.isfile(self.system_dict["params"]["load_weights"])):
                print("Downloading weights");
                cmd = "wget https://github.com/zylo117/Yet-Another-Efficient-Pytorch/releases/download/1.0/efficientdet-d4.pth -O " + \
                            self.system_dict["params"]["load_weights"]
                os.system(cmd);
        elif(self.system_dict["params"]["compound_coef"] == 5):
            if(not os.path.isfile(self.system_dict["params"]["load_weights"])):
                print("Downloading weights");
                cmd = "wget https://github.com/zylo117/Yet-Another-Efficient-Pytorch/releases/download/1.0/efficientdet-d5.pth -O " + \
                            self.system_dict["params"]["load_weights"]
                os.system(cmd);
        elif(self.system_dict["params"]["compound_coef"] == 6):
            if(not os.path.isfile(self.system_dict["params"]["load_weights"])):
                print("Downloading weights");
                cmd = "wget https://github.com/zylo117/Yet-Another-Efficient-Pytorch/releases/download/1.0/efficientdet-d6.pth -O " + \
                            self.system_dict["params"]["load_weights"]
                os.system(cmd);
        elif(self.system_dict["params"]["compound_coef"] == 7):
            if(not os.path.isfile(self.system_dict["params"]["load_weights"])):
                print("Downloading weights");
                cmd = "wget https://github.com/zylo117/Yet-Another-Efficient-Pytorch/releases/download/1.0/efficientdet-d7.pth -O " + \
                            self.system_dict["params"]["load_weights"]
                os.system(cmd);



        

        # load last weights
        if self.system_dict["params"]["load_weights"] is not None:
            if self.system_dict["params"]["load_weights"].endswith('.pth'):
                weights_path = self.system_dict["params"]["load_weights"]
            else:
                weights_path = get_last_weights(self.system_dict["params"]["saved_path"])
            try:
                last_step = int(os.path.basename(weights_path).split('_')[-1].split('.')[0])
            except:
                last_step = 0


            try:
                ret = model.load_state_dict(torch.load(weights_path), strict=False)
            except RuntimeError as e:
                print(f'[Warning] Ignoring {e}')
                print(
                    '[Warning] Don\'t panic if you see this, this might be because you load a pretrained weights with different number of classes. The rest of the weights should be loaded already.')

            print(f'[Info] loaded weights: {os.path.basename(weights_path)}, resuming checkpoint from step: {last_step}')
        else:
            last_step = 0
            print('[Info] initializing weights...')
            init_weights(model)

        print("");
        print("");

        # freeze backbone if train head_only
        if self.system_dict["params"]["head_only"]:
            def freeze_backbone(m):
                classname = m.__class__.__name__
                for ntl in ['EfficientNet', 'BiFPN']:
                    if ntl in classname:
                        for param in m.parameters():
                            param.requires_grad = False

            model.apply(freeze_backbone)
            print('[Info] freezed backbone')

        print("");
        print("");

        if self.system_dict["params"]["num_gpus"] > 1 and self.system_dict["params"]["batch_size"] // self.system_dict["params"]["num_gpus"] < 4:
            model.apply(replace_w_sync_bn)
            use_sync_bn = True
        else:
            use_sync_bn = False

        writer = SummaryWriter(self.system_dict["params"]["log_path"] + f'/{datetime.datetime.now().strftime("%Y%m%d-%H%M%S")}/')

        model = ModelWithLoss(model, debug=self.system_dict["params"]["debug"])

        if self.system_dict["params"]["num_gpus"] > 0:
            model = model.cuda()
            if self.system_dict["params"]["num_gpus"] > 1:
                model = CustomDataParallel(model, self.system_dict["params"]["num_gpus"])
                if use_sync_bn:
                    patch_replication_callback(model)

        if self.system_dict["params"]["optim"] == 'adamw':
            optimizer = torch.optim.AdamW(model.parameters(), self.system_dict["params"]["lr"])
        else:
            optimizer = torch.optim.SGD(model.parameters(), self.system_dict["params"]["lr"], momentum=0.9, nesterov=True)

        scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, patience=3, verbose=True)

        epoch = 0
        best_loss = 1e5
        best_epoch = 0
        step = max(0, last_step)
        model.train()

        num_iter_per_epoch = len(training_generator)


        try:
            for epoch in range(self.system_dict["params"]["num_epochs"]):
                last_epoch = step // num_iter_per_epoch
                if epoch < last_epoch:
                    continue

                epoch_loss = []
                progress_bar = tqdm(training_generator)
                for iter, data in enumerate(progress_bar):
                    if iter < step - last_epoch * num_iter_per_epoch:
                        progress_bar.update()
                        continue
                    try:
                        imgs = data['img']
                        annot = data['annot']

                        if self.system_dict["params"]["num_gpus"] == 1:
                            # if only one gpu, just send it to cuda:0
                            # elif multiple gpus, send it to multiple gpus in CustomDataParallel, not here
                            imgs = imgs.cuda()
                            annot = annot.cuda()

                        optimizer.zero_grad()
                        cls_loss, reg_loss = model(imgs, annot, obj_list=self.system_dict["params"]["obj_list"])
                        cls_loss = cls_loss.mean()
                        reg_loss = reg_loss.mean()

                        loss = cls_loss + reg_loss
                        if loss == 0 or not torch.isfinite(loss):
                            continue

                        loss.backward()
                        # torch.nn.utils.clip_grad_norm_(model.parameters(), 0.1)
                        optimizer.step()

                        epoch_loss.append(float(loss))

                        progress_bar.set_description(
                            'Step: {}. Epoch: {}/{}. Iteration: {}/{}. Cls loss: {:.5f}. Reg loss: {:.5f}. Total loss: {:.5f}'.format(
                                step, epoch, self.system_dict["params"]["num_epochs"], iter + 1, num_iter_per_epoch, cls_loss.item(),
                                reg_loss.item(), loss.item()))
                        writer.add_scalars('Loss', {'train': loss}, step)
                        writer.add_scalars('Regression_loss', {'train': reg_loss}, step)
                        writer.add_scalars('Classfication_loss', {'train': cls_loss}, step)

                        # log learning_rate
                        current_lr = optimizer.param_groups[0]['lr']
                        writer.add_scalar('learning_rate', current_lr, step)

                        step += 1

                        if step % self.system_dict["params"]["save_interval"] == 0 and step > 0:
                            self.save_checkpoint(model, f'efficientdet-d{self.system_dict["params"]["compound_coef"]}_trained.pth')
                            #print('checkpoint...')

                    except Exception as e:
                        print('[Error]', traceback.format_exc())
                        print(e)
                        continue
                scheduler.step(np.mean(epoch_loss))

                if epoch % self.system_dict["params"]["val_interval"] == 0 and self.system_dict["dataset"]["val"]["status"]:
                    print("Running validation");
                    model.eval()
                    loss_regression_ls = []
                    loss_classification_ls = []
                    for iter, data in enumerate(val_generator):
                        with torch.no_grad():
                            imgs = data['img']
                            annot = data['annot']

                            if self.system_dict["params"]["num_gpus"] == 1:
                                imgs = imgs.cuda()
                                annot = annot.cuda()

                            cls_loss, reg_loss = model(imgs, annot, obj_list=self.system_dict["params"]["obj_list"])
                            cls_loss = cls_loss.mean()
                            reg_loss = reg_loss.mean()

                            loss = cls_loss + reg_loss
                            if loss == 0 or not torch.isfinite(loss):
                                continue

                            loss_classification_ls.append(cls_loss.item())
                            loss_regression_ls.append(reg_loss.item())

                    cls_loss = np.mean(loss_classification_ls)
                    reg_loss = np.mean(loss_regression_ls)
                    loss = cls_loss + reg_loss

                    print(
                        'Val. Epoch: {}/{}. Classification loss: {:1.5f}. Regression loss: {:1.5f}. Total loss: {:1.5f}'.format(
                            epoch, self.system_dict["params"]["num_epochs"], cls_loss, reg_loss, loss))
                    writer.add_scalars('Loss', {'val': loss}, step)
                    writer.add_scalars('Regression_loss', {'val': reg_loss}, step)
                    writer.add_scalars('Classfication_loss', {'val': cls_loss}, step)

                    if loss + self.system_dict["params"]["es_min_delta"] < best_loss:
                        best_loss = loss
                        best_epoch = epoch

                        self.save_checkpoint(model, f'efficientdet-d{self.system_dict["params"]["compound_coef"]}_trained.pth')

                    model.train()

                    # Early stopping
                    if epoch - best_epoch > self.system_dict["params"]["es_patience"] > 0:
                        print('[Info] Stop training at epoch {}. The lowest loss achieved is {}'.format(epoch, best_loss))
                        break
        except KeyboardInterrupt:
            self.save_checkpoint(model, f'efficientdet-d{self.system_dict["params"]["compound_coef"]}_trained.pth')
            writer.close()
        writer.close()


        print("");
        print("");
        print("Training complete");
Пример #15
0
def train(model,
          train_dataloader,
          epochs,
          lr,
          steps_til_summary,
          epochs_til_checkpoint,
          model_dir,
          loss_fn,
          summary_fn=None,
          val_dataloader=None,
          double_precision=False,
          clip_grad=False,
          use_lbfgs=False,
          loss_schedules=None,
          validation_fn=None,
          start_epoch=0):

    optim = torch.optim.Adam(lr=lr, params=model.parameters())

    # copy settings from Raissi et al. (2019) and here
    # https://github.com/maziarraissi/PINNs
    if use_lbfgs:
        optim = torch.optim.LBFGS(lr=lr,
                                  params=model.parameters(),
                                  max_iter=50000,
                                  max_eval=50000,
                                  history_size=50,
                                  line_search_fn='strong_wolfe')

    # Load the checkpoint if required
    if start_epoch > 0:
        # Load the model and start training from that point onwards
        model_path = os.path.join(model_dir, 'checkpoints',
                                  'model_epoch_%04d.pth' % start_epoch)
        checkpoint = torch.load(model_path)
        model.load_state_dict(checkpoint['model'])
        model.train()
        optim.load_state_dict(checkpoint['optimizer'])
        optim.param_groups[0]['lr'] = lr
        assert (start_epoch == checkpoint['epoch'])
    else:
        # Start training from scratch
        if os.path.exists(model_dir):
            val = input("The model directory %s exists. Overwrite? (y/n)" %
                        model_dir)
            if val == 'y':
                shutil.rmtree(model_dir)
        os.makedirs(model_dir)

    summaries_dir = os.path.join(model_dir, 'summaries')
    utils.cond_mkdir(summaries_dir)

    checkpoints_dir = os.path.join(model_dir, 'checkpoints')
    utils.cond_mkdir(checkpoints_dir)

    writer = SummaryWriter(summaries_dir)

    total_steps = 0
    with tqdm(total=len(train_dataloader) * epochs) as pbar:
        train_losses = []
        for epoch in range(start_epoch, epochs):
            if not epoch % epochs_til_checkpoint and epoch:
                # Saving the optimizer state is important to produce consistent results
                checkpoint = {
                    'epoch': epoch,
                    'model': model.state_dict(),
                    'optimizer': optim.state_dict()
                }
                torch.save(
                    checkpoint,
                    os.path.join(checkpoints_dir,
                                 'model_epoch_%04d.pth' % epoch))
                # torch.save(model.state_dict(),
                #            os.path.join(checkpoints_dir, 'model_epoch_%04d.pth' % epoch))
                np.savetxt(
                    os.path.join(checkpoints_dir,
                                 'train_losses_epoch_%04d.txt' % epoch),
                    np.array(train_losses))
                if validation_fn is not None:
                    validation_fn(model, checkpoints_dir, epoch)

            for step, (model_input, gt) in enumerate(train_dataloader):
                start_time = time.time()

                if torch.cuda.is_available():
                    model_input = {
                        key: value.cuda()
                        for key, value in model_input.items()
                    }
                    gt = {key: value.cuda() for key, value in gt.items()}
                else:
                    model_input = {
                        key: value.cpu()
                        for key, value in model_input.items()
                    }
                    gt = {key: value.cpu() for key, value in gt.items()}

                if double_precision:
                    model_input = {
                        key: value.double()
                        for key, value in model_input.items()
                    }
                    gt = {key: value.double() for key, value in gt.items()}

                if use_lbfgs:

                    def closure():
                        optim.zero_grad()
                        model_output = model(model_input)
                        losses = loss_fn(model_output, gt)
                        train_loss = 0.
                        for loss_name, loss in losses.items():
                            train_loss += loss.mean()
                        train_loss.backward()
                        return train_loss

                    optim.step(closure)

                model_output = model(model_input)
                losses = loss_fn(model_output, gt)

                # import ipdb; ipdb.set_trace()

                train_loss = 0.
                for loss_name, loss in losses.items():
                    single_loss = loss.mean()

                    if loss_schedules is not None and loss_name in loss_schedules:
                        writer.add_scalar(
                            loss_name + "_weight",
                            loss_schedules[loss_name](total_steps),
                            total_steps)
                        single_loss *= loss_schedules[loss_name](total_steps)

                    writer.add_scalar(loss_name, single_loss, total_steps)
                    train_loss += single_loss

                train_losses.append(train_loss.item())
                writer.add_scalar("total_train_loss", train_loss, total_steps)

                if not total_steps % steps_til_summary:
                    torch.save(
                        model.state_dict(),
                        os.path.join(checkpoints_dir, 'model_current.pth'))
                    # summary_fn(model, model_input, gt, model_output, writer, total_steps)

                if not use_lbfgs:
                    optim.zero_grad()
                    train_loss.backward()

                    if clip_grad:
                        if isinstance(clip_grad, bool):
                            torch.nn.utils.clip_grad_norm_(model.parameters(),
                                                           max_norm=1.)
                        else:
                            torch.nn.utils.clip_grad_norm_(model.parameters(),
                                                           max_norm=clip_grad)

                    optim.step()

                pbar.update(1)

                if not total_steps % steps_til_summary:
                    tqdm.write(
                        "Epoch %d, Total loss %0.6f, iteration time %0.6f" %
                        (epoch, train_loss, time.time() - start_time))

                    if val_dataloader is not None:
                        print("Running validation set...")
                        model.eval()
                        with torch.no_grad():
                            val_losses = []
                            for (model_input, gt) in val_dataloader:
                                model_output = model(model_input)
                                val_loss = loss_fn(model_output, gt)
                                val_losses.append(val_loss)

                            writer.add_scalar("val_loss", np.mean(val_losses),
                                              total_steps)
                        model.train()

                total_steps += 1

        torch.save(model.state_dict(),
                   os.path.join(checkpoints_dir, 'model_final.pth'))
        np.savetxt(os.path.join(checkpoints_dir, 'train_losses_final.txt'),
                   np.array(train_losses))
def train(opt):
    params = Params(f'projects/{opt.project}.yml')

    if params.num_gpus == 0:
        os.environ['CUDA_VISIBLE_DEVICES'] = '-1'

    if torch.cuda.is_available():
        torch.cuda.manual_seed(42)
    else:
        torch.manual_seed(42)

    opt.saved_path = opt.saved_path + f'/{params.project_name}/'
    opt.log_path = opt.log_path + f'/{params.project_name}/tensorboard/'
    os.makedirs(opt.log_path, exist_ok=True)
    os.makedirs(opt.saved_path, exist_ok=True)

    training_params = {
        'batch_size': opt.batch_size,
        'shuffle': True,
        'drop_last': True,
        'collate_fn': collater,
        'num_workers': opt.num_workers
    }

    val_params = {
        'batch_size': opt.batch_size,
        'shuffle': False,
        'drop_last': True,
        'collate_fn': collater,
        'num_workers': opt.num_workers
    }

    input_sizes = [512, 640, 768, 896, 1024, 1280, 1280, 1536, 1536]
    training_set = CocoAlbumentationsDataset(
        root_dir=os.path.join(opt.data_path, params.project_name),
        set=params.train_set,
        transform=A.Compose(
            [
                eval(params.augmentation[i])
                for i in range(len(params.augmentation))
            ],
            bbox_params=A.BboxParams(format='coco',
                                     label_fields=['category_ids'],
                                     min_visibility=0.2),
        ),
        img_size=input_sizes[opt.compound_coef])

    training_generator = DataLoader(training_set, **training_params)

    val_set = CocoDataset(root_dir=os.path.join(opt.data_path,
                                                params.project_name),
                          set=params.val_set,
                          transform=transforms.Compose([
                              Normalizer(mean=params.mean, std=params.std),
                              Resizer(input_sizes[opt.compound_coef])
                          ]))
    val_generator = DataLoader(val_set, **val_params)

    model = EfficientDetBackbone(num_classes=len(params.obj_list),
                                 compound_coef=opt.compound_coef,
                                 ratios=eval(params.anchors_ratios),
                                 scales=eval(params.anchors_scales))

    # load last weights
    if opt.load_weights is not None:
        weights_path = opt.load_weights

        try:
            ret = model.load_state_dict(torch.load(weights_path), strict=False)
        except RuntimeError as e:
            print(f'[Warning] Ignoring {e}')
            print(
                '[Warning] Don\'t panic if you see this, this might be because you load a pretrained weights with different number of classes. The rest of the weights should be loaded already.'
            )

        print(f'[Info] loaded weights: {os.path.basename(weights_path)}')
    else:
        print('[Info] initializing weights...')
        init_weights(model)

    # freeze backbone if train head_only
    if opt.head_only:

        def freeze_backbone(m):
            classname = m.__class__.__name__
            for ntl in ['EfficientNet', 'BiFPN']:
                if ntl in classname:
                    for param in m.parameters():
                        param.requires_grad = False

        model.apply(freeze_backbone)
        print('[Info] freezed backbone')

    # https://github.com/vacancy/Synchronized-BatchNorm-PyTorch
    # apply sync_bn when using multiple gpu and batch_size per gpu is lower than 4
    #  useful when gpu memory is limited.
    # because when bn is disable, the training will be very unstable or slow to converge,
    # apply sync_bn can solve it,
    # by packing all mini-batch across all gpus as one batch and normalize, then send it back to all gpus.
    # but it would also slow down the training by a little bit.
    if params.num_gpus > 1 and opt.batch_size // params.num_gpus < 4:
        model.apply(replace_w_sync_bn)
        use_sync_bn = True
    else:
        use_sync_bn = False

    writer = SummaryWriter(
        opt.log_path +
        f'/{datetime.datetime.now().strftime("%Y%m%d-%H%M%S")}/')

    # warp the model with loss function, to reduce the memory usage on gpu0 and speedup
    model = ModelWithLoss(model, debug=opt.debug)

    if params.num_gpus > 0:
        model = model.cuda()
        if params.num_gpus > 1:
            model = CustomDataParallel(model, params.num_gpus)
            if use_sync_bn:
                patch_replication_callback(model)

    if opt.optim == 'adamw':
        optimizer = torch.optim.AdamW(model.parameters(),
                                      opt.lr,
                                      weight_decay=opt.weight_decay)
    else:
        optimizer = torch.optim.SGD(model.parameters(),
                                    opt.lr,
                                    momentum=0.9,
                                    weight_decay=opt.weight_decay,
                                    nesterov=True)

    epoch = 0
    best_loss = 1e5
    best_epoch = 0
    step = 0
    model.train()

    num_iter_per_epoch = len(training_generator)

    #####################################################################################
    # Check the validation loss for the first time
    # to determine the best_loss more accurately
    if opt.load_weights is not None:
        if opt.load_weights.endswith('.pth'):
            None
        else:
            model.eval()
            loss_regression_ls = []
            loss_classification_ls = []
            for iter, data in enumerate(val_generator):
                with torch.no_grad():
                    imgs = data['img']
                    annot = data['annot']

                    if params.num_gpus == 1:
                        imgs = imgs.cuda()
                        annot = annot.cuda()

                    cls_loss, reg_loss = model(imgs,
                                               annot,
                                               obj_list=params.obj_list)
                    cls_loss = cls_loss.mean()
                    reg_loss = reg_loss.mean()

                    loss = cls_loss + reg_loss
                    if loss == 0 or not torch.isfinite(loss):
                        continue

                    loss_classification_ls.append(cls_loss.item())
                    loss_regression_ls.append(reg_loss.item())

            cls_loss = np.mean(loss_classification_ls)
            reg_loss = np.mean(loss_regression_ls)
            best_loss = cls_loss + reg_loss
            model.train()
    ########################################################################################

    try:
        for epoch in range(opt.num_epochs):
            last_epoch = step // num_iter_per_epoch
            if epoch < last_epoch:
                continue

            epoch_loss = []
            progress_bar = tqdm(training_generator)
            for iter, data in enumerate(progress_bar):
                if iter < step - last_epoch * num_iter_per_epoch:
                    progress_bar.update()
                    continue
                try:
                    imgs = data['img']
                    annot = data['annot']

                    if params.num_gpus == 1:
                        # if only one gpu, just send it to cuda:0
                        # elif multiple gpus, send it to multiple gpus in CustomDataParallel, not here
                        imgs = imgs.cuda()
                        annot = annot.cuda()

                    optimizer.zero_grad()
                    cls_loss, reg_loss = model(imgs,
                                               annot,
                                               obj_list=params.obj_list)
                    cls_loss = cls_loss.mean()
                    reg_loss = reg_loss.mean()

                    loss = cls_loss + reg_loss
                    if loss == 0 or not torch.isfinite(loss):
                        continue

                    loss.backward()
                    # torch.nn.utils.clip_grad_norm_(model.parameters(), 0.1)
                    optimizer.step()

                    epoch_loss.append(float(loss))

                    progress_bar.set_description(
                        'Step: {}. Epoch: {}/{}. Iteration: {}/{}. Cls loss: {:.5f}. Reg loss: {:.5f}. Total loss: {:.5f}'
                        .format(step, epoch, opt.num_epochs, iter + 1,
                                num_iter_per_epoch, cls_loss.item(),
                                reg_loss.item(), loss.item()))
                    writer.add_scalars('Loss', {'train': loss}, step)
                    writer.add_scalars('Regression_loss', {'train': reg_loss},
                                       step)
                    writer.add_scalars('Classfication_loss',
                                       {'train': cls_loss}, step)

                    # log learning_rate
                    current_lr = optimizer.param_groups[0]['lr']
                    writer.add_scalar('learning_rate', current_lr, step)

                    step += 1

                    if step % opt.save_interval == 0 and step > 0:
                        save_checkpoint(
                            model,
                            f'efficientdet-d{opt.compound_coef}_last.pth')
                        print('checkpoint...')

                except Exception as e:
                    print('[Error]', traceback.format_exc())
                    print(e)
                    continue

            if epoch % opt.val_interval == 0:
                model.eval()
                loss_regression_ls = []
                loss_classification_ls = []
                for iter, data in enumerate(val_generator):
                    with torch.no_grad():
                        imgs = data['img']
                        annot = data['annot']

                        if params.num_gpus == 1:
                            imgs = imgs.cuda()
                            annot = annot.cuda()

                        cls_loss, reg_loss = model(imgs,
                                                   annot,
                                                   obj_list=params.obj_list)
                        cls_loss = cls_loss.mean()
                        reg_loss = reg_loss.mean()

                        loss = cls_loss + reg_loss
                        if loss == 0 or not torch.isfinite(loss):
                            continue

                        loss_classification_ls.append(cls_loss.item())
                        loss_regression_ls.append(reg_loss.item())

                cls_loss = np.mean(loss_classification_ls)
                reg_loss = np.mean(loss_regression_ls)
                loss = cls_loss + reg_loss

                print(
                    'Val. Epoch: {}/{}. Classification loss: {:1.5f}. Regression loss: {:1.5f}. Total loss: {:1.5f}'
                    .format(epoch, opt.num_epochs, cls_loss, reg_loss, loss))
                writer.add_scalars('Loss', {'val': loss}, step)
                writer.add_scalars('Regression_loss', {'val': reg_loss}, step)
                writer.add_scalars('Classfication_loss', {'val': cls_loss},
                                   step)

                if loss < best_loss:
                    best_loss = loss
                    best_epoch = epoch

                    save_checkpoint(
                        model, f'efficientdet-d{opt.compound_coef}_best.pth')

                model.train()

    except KeyboardInterrupt:
        save_checkpoint(model, f'efficientdet-d{opt.compound_coef}_last.pth')
        writer.close()
    writer.close()
Пример #17
0
def train_fn(data_loader, model, optimizer, device, scheduler=None):
    """
    Trains the bert model on the twitter data
    """
    # Set model to training mode (dropout + sampled batch norm is activated)
    model.train()
    losses = utils.AverageMeter()
    jaccards = utils.AverageMeter()

    # Set tqdm to add loading screen and set the length
    tk0 = tqdm(data_loader, total=len(data_loader), ncols=80)

    # Train the model on each batch
    for bi, d in enumerate(tk0):

        ids = d["ids"]
        token_type_ids = d["token_type_ids"]
        mask = d["mask"]
        targets_start = d["targets_start"]
        targets_end = d["targets_end"]
        sentiment = d["sentiment"]
        orig_selected = d["orig_selected"]
        orig_tweet = d["orig_tweet"]
        targets_start = d["targets_start"]
        targets_end = d["targets_end"]
        offsets = d["offsets"]

        # Move ids, masks, and targets to gpu while setting as torch.long
        ids = ids.to(device, dtype=torch.long)
        token_type_ids = token_type_ids.to(device, dtype=torch.long)
        mask = mask.to(device, dtype=torch.long)
        targets_start = targets_start.to(device, dtype=torch.long)
        targets_end = targets_end.to(device, dtype=torch.long)

        # Reset gradients
        model.zero_grad()
        # Use ids, masks, and token types as input to the model
        # Predict logits for each of the input tokens for each batch
        outputs_start, outputs_end = model(
            ids=ids,
            mask=mask,
            token_type_ids=token_type_ids,
        )  # (bs x SL), (bs x SL)
        # Calculate batch loss based on CrossEntropy
        loss = loss_fn(outputs_start, outputs_end, targets_start, targets_end)
        # Calculate gradients based on loss
        loss.backward()
        # Adjust weights based on calculated gradients
        optimizer.step()
        # Update scheduler
        scheduler.step()

        # Apply softmax to the start and end logits
        # This squeezes each of the logits in a sequence to a value between 0 and 1, while ensuring that they sum to 1
        # This is similar to the characteristics of "probabilities"
        outputs_start = torch.softmax(outputs_start,
                                      dim=1).cpu().detach().numpy()
        outputs_end = torch.softmax(outputs_end, dim=1).cpu().detach().numpy()

        # Calculate the jaccard score based on the predictions for this batch
        jaccard_scores = []
        for px, tweet in enumerate(orig_tweet):
            selected_tweet = orig_selected[px]
            tweet_sentiment = sentiment[px]
            jaccard_score, _ = calculate_jaccard_score(
                original_tweet=
                tweet,  # Full text of the px'th tweet in the batch
                target_string=selected_tweet,
                # Span containing the specified sentiment for the px'th tweet in the batch
                sentiment_val=
                tweet_sentiment,  # Sentiment of the px'th tweet in the batch
                idx_start=np.argmax(
                    outputs_start[px, :]
                ),  # Predicted start index for the px'th tweet in the batch
                idx_end=np.argmax(
                    outputs_end[px, :]
                ),  # Predicted end index for the px'th tweet in the batch
                offsets=offsets[
                    px]  # Offsets for each of the tokens for the px'th tweet in the batch
            )
            # if tweet_sentiment == 'neutral':
            #     continue
            jaccard_scores.append(jaccard_score)
        # Update the jaccard score and loss
        # For details, refer to `AverageMeter` in https://www.kaggle.com/abhishek/utils
        avg = np.mean(jaccard_scores) if len(jaccard_scores) else 0
        jaccards.update(avg, len(jaccard_scores))
        losses.update(loss.item(), ids.size(0))
        # Print the average loss and jaccard score at the end of each batch
        tk0.set_postfix(loss=losses.avg, jaccard=jaccards.avg)
Пример #18
0
def train(opt):
    params = Params(f'projects/{opt.project}.yml')
    global_validation_it = 0

    if params.num_gpus == 0:
        os.environ['CUDA_VISIBLE_DEVICES'] = '-1'

    if torch.cuda.is_available():
        torch.cuda.manual_seed(42)
    else:
        torch.manual_seed(42)

    opt.saved_path = opt.saved_path + f'/{params.project_name}/'
    opt.log_path = opt.log_path + f'/{params.project_name}/tensorboard/'
    os.makedirs(opt.log_path, exist_ok=True)
    os.makedirs(opt.saved_path, exist_ok=True)

    training_params = {
        'batch_size': opt.batch_size,
        'shuffle': True,
        'drop_last': True,
        'collate_fn': TUMuchTrafficDataset.collater,
        'num_workers': opt.num_workers
    }

    val_params = {
        'batch_size': opt.batch_size,
        'shuffle': False,
        'drop_last': True,
        'collate_fn': TUMuchTrafficDataset.collater,
        'num_workers': opt.num_workers
    }

    advprop = opt.advprop
    if advprop:  # for models using advprop pretrained weights
        normalize = transforms.Lambda(
            lambda mem: {
                "img": (mem["img"] * 2.0 - 1.0).astype(np.float32),
                "annot": mem["annot"]
            })
    else:  # for other models
        normalize = Normalizer(mean=[0.485, 0.456, 0.406],
                               std=[0.229, 0.224, 0.225])

    tfs = transforms.Compose([
        TopCutter(886),
        transforms.RandomApply([Negate()], p=0.1),
        transforms.RandomApply([ContrastEnhancementWithNoiseReduction()],
                               p=0.1),
        Resize(384),
        RandomCrop(384, 768), normalize,
        HorizontalFlip(prob=0.5),
        transforms.RandomApply([AddGaussianNoise(0, 2.55)], p=0.5),
        transforms.RandomApply([AddSaltAndPepperNoise(prob=0.0017)], p=0.5),
        ToTensor()
    ])
    tfrecord_paths = [opt.data_path
                      ] if opt.data_path.endswith(".tfrecord") else [
                          str(x.absolute())
                          for x in Path(opt.data_path).rglob('*.tfrecord')
                      ]
    training_set = TUMuchTrafficDataset(tfrecord_paths=tfrecord_paths,
                                        transform=tfs)
    training_generator = DataLoader(training_set, **training_params)

    tfrecord_paths = [opt.data_path
                      ] if opt.data_path.endswith(".tfrecord") else [
                          str(x.absolute())
                          for x in Path(opt.val_path).rglob('*.tfrecord')
                      ]
    val_set = TUMuchTrafficDataset(tfrecord_paths=tfrecord_paths,
                                   transform=tfs)
    val_generator = DataLoader(val_set, **val_params)

    if not opt.load_backbone:
        load_weights = False
    else:
        load_weights = True
    model = EfficientDetBackbone(num_classes=len(params.obj_list),
                                 compound_coef=opt.compound_coef,
                                 ratios=eval(params.anchors_ratios),
                                 scales=eval(params.anchors_scales),
                                 load_weights=load_weights)
    pytorch_total_params = sum(p.numel() for p in model.parameters())
    print("# Params: {:08d}".format(pytorch_total_params))

    # load last weights
    if opt.load_weights is not None:
        if opt.load_weights.endswith('.pth'):
            weights_path = opt.load_weights
        else:
            weights_path = get_last_weights(opt.saved_path)
        try:
            last_step = int(
                os.path.basename(weights_path).split('_')[-1].split('.')[0])
        except:
            last_step = 0

        try:
            ret = model.load_state_dict(torch.load(weights_path), strict=False)
        except RuntimeError as e:
            print(f'[Warning] Ignoring {e}')
            print(
                '[Warning] Don\'t panic if you see this, this might be because you load a pretrained weights with different number of classes. The rest of the weights should be loaded already.'
            )

        print(
            f'[Info] loaded weights: {os.path.basename(weights_path)}, resuming checkpoint from step: {last_step}'
        )
    else:
        last_step = 0
        print('[Info] initializing weights...')
        init_weights(model)

    # freeze backbone if train head_only
    if opt.head_only:

        def freeze_backbone(m):
            classname = m.__class__.__name__
            for ntl in ['EfficientNet', 'BiFPN']:
                if ntl in classname:
                    for param in m.parameters():
                        param.requires_grad = False

        model.apply(freeze_backbone)
        print('[Info] freezed backbone')

    # freeze backbone (only efficientnet) if train no_effnet
    if opt.no_effnet:

        def freeze_backbone(m):
            classname = m.__class__.__name__
            for ntl in ['EfficientNet']:
                if ntl in classname:
                    for param in m.parameters():
                        param.requires_grad = False

        model.apply(freeze_backbone)
        print('[Info] freezed backbone')
    pytorch_total_params = sum(p.numel() for p in model.parameters()
                               if p.requires_grad)
    print("# Training Parameters: {:06}".format(pytorch_total_params))

    # https://github.com/vacancy/Synchronized-BatchNorm-PyTorch
    # apply sync_bn when using multiple gpu and batch_size per gpu is lower than 4
    #  useful when gpu memory is limited.
    # because when bn is disable, the training will be very unstable or slow to converge,
    # apply sync_bn can solve it,
    # by packing all mini-batch across all gpus as one batch and normalize, then send it back to all gpus.
    # but it would also slow down the training by a little bit.
    if params.num_gpus > 1 and opt.batch_size // params.num_gpus < 4:
        model.apply(replace_w_sync_bn)
        use_sync_bn = True
    else:
        use_sync_bn = False

    writer = SummaryWriter(
        opt.log_path + f'/{datetime.datetime.now().strftime("%Y%m%d-%H%M")}/')

    # warp the model with loss function, to reduce the memory usage on gpu0 and speedup
    model = ModelWithLoss(model, debug=opt.debug)

    if params.num_gpus > 0:
        model = model.cuda()
        if params.num_gpus > 1:
            model = CustomDataParallel(model, params.num_gpus)
            if use_sync_bn:
                patch_replication_callback(model)

    if opt.optim == 'adamw':
        optimizer = torch.optim.AdamW(model.parameters(), opt.lr)
    else:
        optimizer = torch.optim.SGD(model.parameters(),
                                    opt.lr,
                                    momentum=0.9,
                                    nesterov=True)

    scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer,
                                                           patience=1e6,
                                                           verbose=True)

    # use apex for mixed precision training
    # model, optimizer = amp.initialize(model, optimizer)

    epoch = 0
    best_loss = 1e5
    best_epoch = 0
    step = max(0, last_step)
    model.train()

    num_iter_per_epoch = len(training_generator)

    try:
        for epoch in range(opt.num_epochs):
            last_epoch = step // num_iter_per_epoch
            if epoch < last_epoch:
                continue

            epoch_loss = []
            progress_bar = tqdm(training_generator)
            for it, data in enumerate(progress_bar):
                if it < step - last_epoch * num_iter_per_epoch:
                    progress_bar.update()
                    continue
                try:
                    imgs = data['img']
                    annot = data['annot']

                    if params.num_gpus == 1:
                        # if only one gpu, just send it to cuda:0
                        # elif multiple gpus, send it to multiple gpus in CustomDataParallel, not here
                        imgs = imgs.cuda()
                        annot = annot.cuda()

                    global_validation_it += 1
                    optimizer.zero_grad()

                    cls_loss, reg_loss = model(imgs, annot)
                    cls_loss = cls_loss.mean()
                    reg_loss = reg_loss.mean()

                    loss = cls_loss + reg_loss
                    if loss == 0 or not torch.isfinite(loss):
                        continue

                    loss.backward()
                    # torch.nn.utils.clip_grad_norm_(model.parameters(), 0.1)
                    optimizer.step()

                    epoch_loss.append(float(loss))

                    progress_bar.set_description(
                        'Step: {}. Epoch: {}/{}. Iteration: {}/{}. Cls loss: {:.5f}. Reg loss: {:.5f}. Total loss: {:.5f}'
                        .format(step, epoch, opt.num_epochs, it + 1,
                                num_iter_per_epoch, cls_loss.item(),
                                reg_loss.item(), loss.item()))
                    writer.add_scalars('Loss', {'train': loss}, step)
                    writer.add_scalars('Regression_loss', {'train': reg_loss},
                                       step)
                    writer.add_scalars('Classfication_loss',
                                       {'train': cls_loss}, step)

                    # log learning_rate
                    current_lr = optimizer.param_groups[0]['lr']
                    writer.add_scalar('learning_rate', current_lr, step)

                    step += 1

                    if step % opt.save_interval == 0 and step > 0:
                        save_checkpoint(
                            model,
                            f'efficientdet-d{opt.compound_coef}_{epoch}_{step}.pth'
                        )
                        print('checkpoint...')

                except Exception as e:
                    print('[Error]', traceback.format_exc())
                    print(e)
                    continue
            scheduler.step(np.mean(epoch_loss))

            # sleep for 30 seconds, to reduce overheating
            import time
            time.sleep(30)

            if epoch % opt.val_interval == 0:
                model.eval()
                loss_regression_ls = []
                loss_classification_ls = []
                for it, data in enumerate(val_generator):
                    with torch.no_grad():
                        imgs = data['img']
                        annot = data['annot']
                        if params.num_gpus == 1:
                            # if only one gpu, just send it to cuda:0
                            # elif multiple gpus, send it to multiple gpus in CustomDataParallel, not here
                            imgs = imgs.cuda()
                            annot = annot.cuda()
                        if it < 12:
                            plot_tensorboard(imgs, annot, model, writer,
                                             global_validation_it, it, "")
                            global_validation_it += 1

                        if params.num_gpus == 1:
                            imgs = imgs.cuda()
                            annot = annot.cuda()

                        cls_loss, reg_loss = model(imgs,
                                                   annot,
                                                   obj_list=params.obj_list)
                        cls_loss = cls_loss.mean()
                        reg_loss = reg_loss.mean()

                        loss = cls_loss + reg_loss
                        if loss == 0 or not torch.isfinite(loss):
                            continue

                        loss_classification_ls.append(cls_loss.item())
                        loss_regression_ls.append(reg_loss.item())

                cls_loss = np.mean(loss_classification_ls)
                reg_loss = np.mean(loss_regression_ls)
                loss = cls_loss + reg_loss

                print(
                    'Val. Epoch: {}/{}. Classification loss: {:1.5f}. Regression loss: {:1.5f}. Total loss: {:1.5f}'
                    .format(epoch, opt.num_epochs, cls_loss, reg_loss, loss))
                writer.add_scalars('Loss', {'val': loss}, step)
                writer.add_scalars('Regression_loss', {'val': reg_loss}, step)
                writer.add_scalars('Classfication_loss', {'val': cls_loss},
                                   step)

                if loss + opt.es_min_delta < best_loss:
                    best_loss = loss
                    best_epoch = epoch

                    save_checkpoint(
                        model,
                        f'efficientdet-d{opt.compound_coef}_{epoch}_{step}.pth'
                    )

                model.train()

                # Early stopping
                if epoch - best_epoch > opt.es_patience > 0:
                    print(
                        '[Info] Stop training at epoch {}. The lowest loss achieved is {}'
                        .format(epoch, best_loss))
                    break
    except KeyboardInterrupt:
        save_checkpoint(
            model, f'efficientdet-d{opt.compound_coef}_{epoch}_{step}.pth')
        writer.close()
    writer.close()
Пример #19
0
def eval_fn(data_loader, model, device):
    """
    Evaluation function to predict on the test set
    """
    # Set model to evaluation mode
    # I.e., turn off dropout and set batchnorm to use overall mean and variance (from training), rather than batch level mean and variance
    # Reference: https://github.com/pytorch/pytorch/issues/5406
    model.eval()
    losses = utils.AverageMeter()
    jaccards = utils.AverageMeter()

    # Turns off gradient calculations (https://datascience.stackexchange.com/questions/32651/what-is-the-use-of-torch-no-grad-in-pytorch)
    with torch.no_grad():
        tk0 = tqdm(data_loader, total=len(data_loader), ncols=80)
        # Make predictions and calculate loss / jaccard score for each batch
        for bi, d in enumerate(tk0):
            ids = d["ids"]
            token_type_ids = d["token_type_ids"]
            mask = d["mask"]
            sentiment = d["sentiment"]
            orig_selected = d["orig_selected"]
            orig_tweet = d["orig_tweet"]
            targets_start = d["targets_start"]
            targets_end = d["targets_end"]
            offsets = d["offsets"].numpy()

            # Move ids, masks, and targets to gpu while setting as torch.long
            ids = ids.to(device, dtype=torch.long)
            token_type_ids = token_type_ids.to(device, dtype=torch.long)
            mask = mask.to(device, dtype=torch.long)
            targets_start = targets_start.to(device, dtype=torch.long)
            targets_end = targets_end.to(device, dtype=torch.long)

            # Move tensors to GPU for faster matrix calculations
            ids = ids.to(device, dtype=torch.long)
            token_type_ids = token_type_ids.to(device, dtype=torch.long)
            mask = mask.to(device, dtype=torch.long)
            targets_start = targets_start.to(device, dtype=torch.long)
            targets_end = targets_end.to(device, dtype=torch.long)

            # Predict logits for start and end indexes
            outputs_start, outputs_end = model(ids=ids,
                                               mask=mask,
                                               token_type_ids=token_type_ids)
            # Calculate loss for the batch
            loss = loss_fn(outputs_start, outputs_end, targets_start,
                           targets_end)
            # Apply softmax to the predicted logits for the start and end indexes
            # This converts the "logits" to "probability-like" scores
            outputs_start = torch.softmax(outputs_start,
                                          dim=1).cpu().detach().numpy()
            outputs_end = torch.softmax(outputs_end,
                                        dim=1).cpu().detach().numpy()
            # Calculate jaccard scores for each tweet in the batch
            jaccard_scores = []
            for px, tweet in enumerate(orig_tweet):
                selected_tweet = orig_selected[px]
                tweet_sentiment = sentiment[px]
                jaccard_score, _ = calculate_jaccard_score(
                    original_tweet=tweet,
                    target_string=selected_tweet,
                    sentiment_val=tweet_sentiment,
                    idx_start=np.argmax(outputs_start[px, :]),
                    idx_end=np.argmax(outputs_end[px, :]),
                    offsets=offsets[px])
                jaccard_scores.append(jaccard_score)

            # Update running jaccard score and loss
            jaccards.update(np.mean(jaccard_scores), ids.size(0))
            losses.update(loss.item(), ids.size(0))
            # Print the running average loss and jaccard score
            tk0.set_postfix(loss=losses.avg, jaccard=jaccards.avg)

    print(f"Jaccard = {jaccards.avg}")
    return jaccards.avg
Пример #20
0
model = MnistResNet().to(device)
dataset = MnistDataset()
train_loader, val_loader, _ = dataset.data_loaders(batch_size=256)

losses = []
loss_function = nn.CrossEntropyLoss()
optimizer = optim.Adadelta(model.parameters())

batches = len(train_loader)
val_batches = len(val_loader)

# training loop + eval loop
for epoch in range(epochs):
    total_loss = 0
    progress = tqdm(enumerate(train_loader), desc="Loss: ", total=batches)
    model.train()

    for i, data in progress:
        X, y = data[0].to(device), data[1].to(device)

        model.zero_grad()
        outputs = model(X)
        loss = loss_function(outputs, y)

        loss.backward()
        optimizer.step()
        current_loss = loss.item()
        total_loss += current_loss
        progress.set_description("Loss: {:.4f}".format(total_loss / (i + 1)))
Пример #21
0
def keyword_categories(file_info: List[FileInfo], category_words):
    return [
        kw_cat(fi.text + " " + " ".join(news.simple_tokenize(fi.url)),
               category_words)
        for fi in tqdm(file_info, desc="assigning keyword-based labels")
    ]
Пример #22
0
    result = len(list(doc.sents))
    return result


# %%
from collections import Counter


number_of_sents, pos_adj_counter, neg_adj_counter, ent_counter = (
    [],
    Counter(),
    Counter(),
    Counter(),
)

for review, sentiment in tqdm(data.to_records(index=False)):
    doc = nlp(review)
    adjs = get_adjs(doc)
    ents = get_ents(doc)

    # what to do with number_of_sents

    number_of_sents.append(
        get_number_of_sents(doc)
        )

    # what to do with pos_adj_counter and neg_adj_counter
    for adj in adjs:
        counter = pos_adj_counter if sentiment == "positive" else neg_adj_counter
        counter[adj.lower()] += 1
Пример #23
0
    def fit(
        self,
        train_dataloader: DataLoader,
        evaluator: SentenceEvaluator = None,
        epochs: int = 1,
        loss_fct=None,
        acitvation_fct=nn.Identity(),
        scheduler: str = 'WarmupLinear',
        warmup_steps: int = 10000,
        optimizer_class: Type[Optimizer] = transformers.AdamW,
        optimizer_params: Dict[str, object] = {
            'lr': 2e-5,
            'eps': 1e-6,
            'correct_bias': False
        },
        weight_decay: float = 0.01,
        evaluation_steps: int = 0,
        output_path: str = None,
        save_best_model: bool = True,
        max_grad_norm: float = 1,
        use_amp: bool = False,
        callback: Callable[[float, int, int], None] = None,
    ):
        """
        Train the model with the given training objective
        Each training objective is sampled in turn for one batch.
        We sample only as many batches from each objective as there are in the smallest one
        to make sure of equal training with each dataset.

        :param train_dataloader: DataLoader with training InputExamples
        :param evaluator: An evaluator (sentence_transformers.evaluation) evaluates the model performance during training on held-out dev data. It is used to determine the best model that is saved to disc.
        :param epochs: Number of epochs for training
        :param loss_fct: Which loss function to use for training. If None, will use nn.BCEWithLogitsLoss() if self.config.num_labels == 1 else nn.CrossEntropyLoss()
        :param acitvation_fct: Activation function applied on top of logits output of model.
        :param scheduler: Learning rate scheduler. Available schedulers: constantlr, warmupconstant, warmuplinear, warmupcosine, warmupcosinewithhardrestarts
        :param warmup_steps: Behavior depends on the scheduler. For WarmupLinear (default), the learning rate is increased from o up to the maximal learning rate. After these many training steps, the learning rate is decreased linearly back to zero.
        :param optimizer_class: Optimizer
        :param optimizer_params: Optimizer parameters
        :param weight_decay: Weight decay for model parameters
        :param evaluation_steps: If > 0, evaluate the model using evaluator after each number of training steps
        :param output_path: Storage path for the model and evaluation files
        :param save_best_model: If true, the best model (according to evaluator) is stored at output_path
        :param max_grad_norm: Used for gradient normalization.
        :param use_amp: Use Automatic Mixed Precision (AMP). Only for Pytorch >= 1.6.0
        :param callback: Callback function that is invoked after each evaluation.
                It must accept the following three parameters in this order:
                `score`, `epoch`, `steps`
        """
        train_dataloader.collate_fn = self.smart_batching_collate

        if use_amp:
            from torch.cuda.amp import autocast
            scaler = torch.cuda.amp.GradScaler()

        self.model.to(self._target_device)

        if output_path is not None:
            os.makedirs(output_path, exist_ok=True)

        self.best_score = -9999999
        num_train_steps = int(len(train_dataloader) * epochs)

        # Prepare optimizers
        param_optimizer = list(self.model.named_parameters())

        no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']
        optimizer_grouped_parameters = [{
            'params': [
                p for n, p in param_optimizer
                if not any(nd in n for nd in no_decay)
            ],
            'weight_decay':
            weight_decay
        }, {
            'params':
            [p for n, p in param_optimizer if any(nd in n for nd in no_decay)],
            'weight_decay':
            0.0
        }]

        optimizer = optimizer_class(optimizer_grouped_parameters,
                                    **optimizer_params)

        if isinstance(scheduler, str):
            scheduler = SentenceTransformer._get_scheduler(
                optimizer,
                scheduler=scheduler,
                warmup_steps=warmup_steps,
                t_total=num_train_steps)

        if loss_fct is None:
            loss_fct = nn.BCEWithLogitsLoss(
            ) if self.config.num_labels == 1 else nn.CrossEntropyLoss()

        skip_scheduler = False
        for epoch in trange(epochs, desc="Epoch"):
            training_steps = 0
            self.model.zero_grad()
            self.model.train()

            for features, labels in tqdm(train_dataloader,
                                         desc="Iteration",
                                         smoothing=0.05):
                if use_amp:
                    with autocast():
                        model_predictions = self.model(**features,
                                                       return_dict=True)
                        logits = acitvation_fct(model_predictions.logits)
                        if self.config.num_labels == 1:
                            logits = logits.view(-1)
                        loss_value = loss_fct(logits, labels)

                    scale_before_step = scaler.get_scale()
                    scaler.scale(loss_value).backward()
                    scaler.unscale_(optimizer)
                    torch.nn.utils.clip_grad_norm_(self.model.parameters(),
                                                   max_grad_norm)
                    scaler.step(optimizer)
                    scaler.update()

                    skip_scheduler = scaler.get_scale() != scale_before_step
                else:
                    model_predictions = self.model(**features,
                                                   return_dict=True)
                    logits = acitvation_fct(model_predictions.logits)
                    if self.config.num_labels == 1:
                        logits = logits.view(-1)

                    loss_value = loss_fct(logits, labels)
                    loss_value.backward()
                    torch.nn.utils.clip_grad_norm_(self.model.parameters(),
                                                   max_grad_norm)
                    optimizer.step()

                optimizer.zero_grad()

                if not skip_scheduler:
                    scheduler.step()

                training_steps += 1

                if evaluator is not None and evaluation_steps > 0 and training_steps % evaluation_steps == 0:
                    self._eval_during_training(evaluator, output_path,
                                               save_best_model, epoch,
                                               training_steps, callback)

                    self.model.zero_grad()
                    self.model.train()

            if evaluator is not None:
                self._eval_during_training(evaluator, output_path,
                                           save_best_model, epoch, -1,
                                           callback)
Пример #24
0
    def _estimate_transforms(self, nsamples):
        """ Uses block matrix inversion identities to quickly estimate transforms.

        After a bit of matrix math we can isolate a transform matrix (# features x # features)
        that is independent of any sample we are explaining. It is the result of averaging over
        all feature permutations, but we just use a fixed number of samples to estimate the value.

        TODO: Do a brute force enumeration when # feature subsets is less than nsamples. This could
              happen through a recursive method that uses the same block matrix inversion as below.
        """
        M = len(self.coef)

        mean_transform = np.zeros((M, M))
        x_transform = np.zeros((M, M))
        inds = np.arange(M, dtype=np.int)
        for _ in tqdm(range(nsamples), "Estimating transforms"):
            np.random.shuffle(inds)
            cov_inv_SiSi = np.zeros((0, 0))
            cov_Si = np.zeros((M, 0))
            for j in range(M):
                i = inds[j]

                # use the last Si as the new S
                cov_S = cov_Si
                cov_inv_SS = cov_inv_SiSi

                # get the new cov_Si
                cov_Si = self.cov[:, inds[:j + 1]]

                # compute the new cov_inv_SiSi from cov_inv_SS
                d = cov_Si[i, :-1].T
                t = np.matmul(cov_inv_SS, d)
                Z = self.cov[i, i]
                u = Z - np.matmul(t.T, d)
                cov_inv_SiSi = np.zeros((j + 1, j + 1))
                if j > 0:
                    cov_inv_SiSi[:-1, :-1] = cov_inv_SS + np.outer(t, t) / u
                    cov_inv_SiSi[:-1, -1] = cov_inv_SiSi[-1, :-1] = -t / u
                cov_inv_SiSi[-1, -1] = 1 / u

                # + coef @ (Q(bar(Sui)) - Q(bar(S)))
                mean_transform[i, i] += self.coef[i]

                # + coef @ R(Sui)
                coef_R_Si = np.matmul(
                    self.coef[inds[j + 1:]],
                    np.matmul(cov_Si, cov_inv_SiSi)[inds[j + 1:]])
                mean_transform[i, inds[:j + 1]] += coef_R_Si

                # - coef @ R(S)
                coef_R_S = np.matmul(self.coef[inds[j:]],
                                     np.matmul(cov_S, cov_inv_SS)[inds[j:]])
                mean_transform[i, inds[:j]] -= coef_R_S

                # - coef @ (Q(Sui) - Q(S))
                x_transform[i, i] += self.coef[i]

                # + coef @ R(Sui)
                x_transform[i, inds[:j + 1]] += coef_R_Si

                # - coef @ R(S)
                x_transform[i, inds[:j]] -= coef_R_S

        mean_transform /= nsamples
        x_transform /= nsamples
        return mean_transform, x_transform
Пример #25
0
    def predict(self,
                sentences: List[List[str]],
                batch_size: int = 32,
                show_progress_bar: bool = None,
                num_workers: int = 0,
                activation_fct=None,
                apply_softmax=False,
                convert_to_numpy: bool = True,
                convert_to_tensor: bool = False):
        """
        Performs predicts with the CrossEncoder on the given sentence pairs.

        :param sentences: A list of sentence pairs [[Sent1, Sent2], [Sent3, Sent4]]
        :param batch_size: Batch size for encoding
        :param show_progress_bar: Output progress bar
        :param num_workers: Number of workers for tokenization
        :param activation_fct: Activation function applied on the logits output of the CrossEncoder. If None, nn.Sigmoid() will be used if num_labels=1, else nn.Identity
        :param convert_to_numpy: Convert the output to a numpy matrix.
        :param apply_softmax: If there are more than 2 dimensions and apply_softmax=True, applies softmax on the logits output
        :param convert_to_tensor:  Conver the output to a tensor.
        :return: Predictions for the passed sentence pairs
        """
        input_was_string = False
        if isinstance(
                sentences[0],
                str):  # Cast an individual sentence to a list with length 1
            sentences = [sentences]
            input_was_string = True

        inp_dataloader = DataLoader(
            sentences,
            batch_size=batch_size,
            collate_fn=self.smart_batching_collate_text_only,
            num_workers=num_workers,
            shuffle=False)

        if show_progress_bar is None:
            show_progress_bar = (logger.getEffectiveLevel() == logging.INFO or
                                 logger.getEffectiveLevel() == logging.DEBUG)

        iterator = inp_dataloader
        if show_progress_bar:
            iterator = tqdm(inp_dataloader, desc="Batches")

        if activation_fct is None:
            activation_fct = nn.Sigmoid(
            ) if self.config.num_labels == 1 else nn.Identity()

        pred_scores = []
        self.model.eval()
        self.model.to(self._target_device)
        with torch.no_grad():
            for features in iterator:
                model_predictions = self.model(**features, return_dict=True)
                logits = activation_fct(model_predictions.logits)

                if apply_softmax and len(logits[0]) > 1:
                    logits = torch.nn.functional.softmax(logits, dim=1)
                pred_scores.extend(logits)

        if self.config.num_labels == 1:
            pred_scores = [score[0] for score in pred_scores]

        if convert_to_tensor:
            pred_scores = torch.stack(pred_scores)
        elif convert_to_numpy:
            pred_scores = np.asarray(
                [score.cpu().detach().numpy() for score in pred_scores])

        if input_was_string:
            pred_scores = pred_scores[0]

        return pred_scores
Пример #26
0
    parser.add_argument('-o', type=Path)

    args = parser.parse_args()

    logging.basicConfig(format='%(levelname)s:%(message)s', level=logging.INFO)
    logging.info(f'Using {args.storage} storage 💾')

    if args.o is not None:
        save_dir = args.o
        save_dir.mkdir(exist_ok=True)

    storage = LocalStorage() if args.storage == 'local' else AWSSTorage()
    if args.storage == 'local':
        logging.info(f'Store root={storage.root}')

    override = True

    bar = tqdm(zoo_source.items())
    uploading_bar = tqdm()
    for key, src_def in bar:
        bar.set_description(key)
        if src_def is None:
            # it means I was lazy and I meant to use timm
            src_def = partial(timm.create_model, key, pretrained=True)
        if key not in storage or override:
            src, dst = src_def(), AutoModel.from_name(key)
            cloned = clone_model(src, dst)
            storage(key, cloned, uploading_bar)

    # uploading_bar.update(0)
Пример #27
0
    def range_test(
        self,
        train_loader,
        val_loader=None,
        start_lr=None,
        end_lr=10,
        num_iter=100,
        step_mode="exp",
        smooth_f=0.05,
        diverge_th=5,
        accumulation_steps=1,
        non_blocking_transfer=True,
    ):
        """Performs the learning rate range test.
        Arguments:
            train_loader (`torch.utils.data.DataLoader`
                or child of `TrainDataLoaderIter`, optional):
                the training set data loader.
                If your dataset (data loader) returns a tuple (inputs, labels,*) then
                Pytorch data loader object can be provided. However, if a dataset
                returns different outputs e.g. dicts, then you should inherit
                from `TrainDataLoaderIter` class and redefine `inputs_labels_from_batch`
                method so that it outputs (inputs, labels).
            val_loader (`torch.utils.data.DataLoader`
                or child of `ValDataLoaderIter`, optional): if `None` the range test
                will only use the training loss. When given a data loader, the model is
                evaluated after each iteration on that dataset and the evaluation loss
                is used. Note that in this mode the test takes significantly longer but
                generally produces more precise results.
                Similarly to `train_loader`, if your dataset outputs are not standard
                you should inherit from `ValDataLoaderIter` class and
                redefine method `inputs_labels_from_batch` so that
                it outputs (inputs, labels). Default: None.
            start_lr (float, optional): the starting learning rate for the range test.
                Default: None (uses the learning rate from the optimizer).
            end_lr (float, optional): the maximum learning rate to test. Default: 10.
            num_iter (int, optional): the number of iterations over which the test
                occurs. Default: 100.
            step_mode (str, optional): one of the available learning rate policies,
                linear or exponential ("linear", "exp"). Default: "exp".
            smooth_f (float, optional): the loss smoothing factor within the [0, 1[
                interval. Disabled if set to 0, otherwise the loss is smoothed using
                exponential smoothing. Default: 0.05.
            diverge_th (int, optional): the test is stopped when the loss surpasses the
                threshold:  diverge_th * best_loss. Default: 5.
            accumulation_steps (int, optional): steps for gradient accumulation. If it
                is 1, gradients are not accumulated. Default: 1.
            non_blocking_transfer (bool, optional): when non_blocking_transfer is set,
                tries to convert/move data to the device asynchronously if possible,
                e.g., moving CPU Tensors with pinned memory to CUDA devices. Default: True.
        Example (fastai approach):
            >>> lr_finder = LRFinder(net, optimizer, criterion, device="cuda")
            >>> lr_finder.range_test(dataloader, end_lr=100, num_iter=100)
        Example (Leslie Smith's approach):
            >>> lr_finder = LRFinder(net, optimizer, criterion, device="cuda")
            >>> lr_finder.range_test(trainloader, val_loader=val_loader, end_lr=1, num_iter=100, step_mode="linear")
        Gradient accumulation is supported; example:
            >>> train_data = ...    # prepared dataset
            >>> desired_bs, real_bs = 32, 4         # batch size
            >>> accumulation_steps = desired_bs // real_bs     # required steps for accumulation
            >>> dataloader = torch.utils.data.DataLoader(train_data, batch_size=real_bs, shuffle=True)
            >>> acc_lr_finder = LRFinder(net, optimizer, criterion, device="cuda")
            >>> acc_lr_finder.range_test(dataloader, end_lr=10, num_iter=100, accumulation_steps=accumulation_steps)
        If your DataLoader returns e.g. dict, or other non standard output, intehit from TrainDataLoaderIter,
        redefine method `inputs_labels_from_batch` so that it outputs (inputs, lables) data:
            >>> import torch_lr_finder
            >>> class TrainIter(torch_lr_finder.TrainDataLoaderIter):
            >>>     def inputs_labels_from_batch(self, batch_data):
            >>>         return (batch_data['user_features'], batch_data['user_history']), batch_data['y_labels']
            >>> train_data_iter = TrainIter(train_dl)
            >>> finder = torch_lr_finder.LRFinder(model, optimizer, partial(model._train_loss, need_one_hot=False))
            >>> finder.range_test(train_data_iter, end_lr=10, num_iter=300, diverge_th=10)
        Reference:
        [Training Neural Nets on Larger Batches: Practical Tips for 1-GPU, Multi-GPU & Distributed setups](
        https://medium.com/huggingface/ec88c3e51255)
        [thomwolf/gradient_accumulation](https://gist.github.com/thomwolf/ac7a7da6b1888c2eeac8ac8b9b05d3d3)
        """

        # Reset test results
        self.history = {"lr": [], "acc": []}
        self.best_loss = None
        self.best_acc = None

        # Move the model to the proper device
        self.model.to(self.device)

        # Check if the optimizer is already attached to a scheduler
        self._check_for_scheduler()

        # Set the starting learning rate
        if start_lr:
            self._set_learning_rate(start_lr)

        # Initialize the proper learning rate policy
        if step_mode.lower() == "exp":
            lr_schedule = ExponentialLR(self.optimizer, end_lr, num_iter)
        elif step_mode.lower() == "linear":
            lr_schedule = LinearLR(self.optimizer, end_lr, num_iter)
        else:
            raise ValueError(
                "expected one of (exp, linear), got {}".format(step_mode))

        if smooth_f < 0 or smooth_f >= 1:
            raise ValueError("smooth_f is outside the range [0, 1[")

        # Create an iterator to get data batch by batch
        if isinstance(train_loader, DataLoader):
            train_iter = TrainDataLoaderIter(train_loader)
        elif isinstance(train_loader, TrainDataLoaderIter):
            train_iter = train_loader
        else:
            raise ValueError("`train_loader` has unsupported type: {}."
                             "Expected types are `torch.utils.data.DataLoader`"
                             "or child of `TrainDataLoaderIter`.".format(
                                 type(train_loader)))

        if val_loader:
            if isinstance(val_loader, DataLoader):
                val_iter = ValDataLoaderIter(val_loader)
            elif isinstance(val_loader, ValDataLoaderIter):
                val_iter = val_loader
            else:
                raise ValueError(
                    "`val_loader` has unsupported type: {}."
                    "Expected types are `torch.utils.data.DataLoader`"
                    "or child of `ValDataLoaderIter`.".format(
                        type(val_loader)))

        for iteration in tqdm(range(num_iter)):
            # Train on batch and retrieve accuracy
            acc = self._train_batch(
                train_iter,
                accumulation_steps,
                non_blocking_transfer=non_blocking_transfer,
            )
            if val_loader:
                acc = self._validate(
                    val_iter, non_blocking_transfer=non_blocking_transfer)

            # Update the learning rate
            self.history["lr"].append(lr_schedule.get_lr()[0])
            lr_schedule.step()

            # Track the best accuracy and smooth it if smooth_f is specified
            if iteration == 0:
                self.best_acc = acc
            else:
                if smooth_f > 0:
                    acc = smooth_f * acc + (1 -
                                            smooth_f) * self.history["acc"][-1]
                if acc > self.best_acc:
                    self.best_acc = acc

            # Check if the loss has diverged; if it has, stop the test
            self.history["acc"].append(acc)
            #if loss > diverge_th * self.best_loss:
            #print("Stopping early, the loss has diverged")
            #break

        print(
            "Learning rate search finished. See the graph with {finder_name}.plot()"
        )
Пример #28
0
def train_encoder_model(encoder,
                        train_dl,
                        test_dl,
                        Y,
                        Y_test,
                        epochs=config.num_enc_epochs):
    epochs_losses = []
    criterion = nn.MSELoss()
    optimizer = optim.Adam(lr=1e-3, params=encoder.parameters())

    # scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=5, gamma=0.5)
    scaler = torch.cuda.amp.GradScaler(enabled=True)
    Y = torch.from_numpy(Y).float().cuda()
    Y_test = torch.from_numpy(Y_test).float().cuda()
    epochs_losses_test = []
    for epoch in tqdm(range(epochs)):
        losses = []
        test_losses = []
        encoder.train()

        for i, batch in enumerate(train_dl, 0):
            inputs, mris, idxs = batch
            inputs = einops.rearrange(inputs, 'b h w c -> b c h w')
            #             inputs.cuda()
            Y_batch = mris.float().cuda()
            with torch.cuda.amp.autocast():
                outputs = encoder(inputs.float().cuda()).float().cuda()
                #             print(len(inputs), len(idxs))
                #             print(outputs.size())
                #             print(outputs)
                #             print(Y_batch)
                loss = criterion(outputs, Y_batch)
            scaler.scale(loss).backward()
            scaler.step(optimizer)
            scaler.update()
            # scheduler.step()

            optimizer.zero_grad()
            losses.append(loss.item())
        encoder.eval()
        with torch.no_grad():
            for i, batch in enumerate(test_dl):
                inputs, mris, idxs = batch
                inputs = einops.rearrange(inputs, 'b h w c -> b c h w')
                Y_test_batch = mris.float().cuda()
                with torch.cuda.amp.autocast():
                    outputs = encoder(inputs.float().cuda()).float().cuda()
                    loss = criterion(outputs, Y_test_batch)

                test_losses.append(loss.item())
        print(
            f'[{epoch+1}, loss: {np.mean(losses):.4f} test loss: {np.mean(test_losses):.4f}'
        )
        epochs_losses.append(np.mean(losses))
        epochs_losses_test.append(np.mean(test_losses))
        losses = []
        test_losses = []
    # import matplotlib.pyplot as plt
    # plt.plot(range(epochs), epochs_losses, label='train')
    # plt.plot(range(epochs), epochs_losses_test, label='test')
    # plt.legend()
    # plt.show()
    return encoder
Пример #29
0
def train(model: nn.Module,
          scheduler,
          optimizer,
          images,
          datasets,
          n_epoches: int,
          batch_size: int,
          eval_valid_freq: int = 1,
          eval_test_freq: int = 3,
          device=None):

    log_dir = inc_folder_no(ROOT_DIR / "runs" / "s04_exp" / "run_")
    writer = SummaryWriter(log_dir=log_dir)
    global_step = 0
    model.train()

    valid_res = {'loss': float('nan'), 'accuracy': float('nan')}
    test_res = {'loss': float('nan'), 'accuracy': float('nan')}
    best_performance = 0.0

    train_examples, valid_examples, test_examples = datasets

    try:
        with tqdm(range(n_epoches),
                  desc='epoch') as epoches, tqdm(total=math.ceil(
                      sum([len(x)
                           for x in train_examples]) / batch_size) * n_epoches,
                                                 desc='training') as pbar:
            for i in epoches:
                scheduler.step()
                for bimgs, bx, bnts, bxlen, bnt, bnbtn, bnrow in iter_batch(
                        batch_size,
                        images,
                        train_examples,
                        shuffle=True,
                        device=device):
                    pbar.update()
                    global_step += 1

                    model.zero_grad()
                    bnts_pred = model(bimgs, bx, bxlen)

                    loss, mask, btokens = padded_aware_nllloss(bnts_pred, bnts)
                    accuracy = ((torch.argmax(bnts_pred, dim=1)
                                 == bnts.view(-1)).float() *
                                mask).sum().item() / btokens

                    loss.backward()
                    optimizer.step()

                    writer.add_scalar('train/loss', loss, global_step)
                    writer.add_scalar('train/accuracy', accuracy, global_step)

                    pbar.set_postfix(loss=f"{loss:.5f}",
                                     accuracy=f"{accuracy:.5f}")

                if (i + 1) % eval_valid_freq == 0:
                    valid_res = eval(model, images, valid_examples, device)
                    for k, v in valid_res.items():
                        writer.add_scalar(f'valid/{k}', v, global_step)

                    if valid_res['accuracy'] > best_performance:
                        best_performance = valid_res['accuracy']
                        torch.save(model, log_dir + f"/model.{i}.bin")

                if (i + 1) % eval_test_freq == 0:
                    test_res = eval(model, images, test_examples, device)
                    for k, v in test_res.items():
                        writer.add_scalar(f'test/{k}', v, global_step)

                epoches.set_postfix(
                    v_l=f'{valid_res["loss"]:.5f}',
                    v_a=f'{valid_res["accuracy"]:.5f}',
                    t_l=f'{test_res["loss"]:.5f}',
                    t_a=f'{test_res["accuracy"]:.5f}',
                )
    finally:
        writer.close()
Пример #30
0
    def encode(self,
               sentences: Union[str, List[str], List[int]],
               batch_size: int = 32,
               show_progress_bar: bool = None,
               output_value: str = 'sentence_embedding',
               convert_to_numpy: bool = True,
               convert_to_tensor: bool = False,
               is_pretokenized: bool = False,
               device: str = None,
               num_workers: int = 0) -> Union[List[Tensor], ndarray, Tensor]:
        """
        Computes sentence embeddings

        :param sentences: the sentences to embed
        :param batch_size: the batch size used for the computation
        :param show_progress_bar: Output a progress bar when encode sentences
        :param output_value:  Default sentence_embedding, to get sentence embeddings. Can be set to token_embeddings to get wordpiece token embeddings.
        :param convert_to_numpy: If true, the output is a list of numpy vectors. Else, it is a list of pytorch tensors.
        :param convert_to_tensor: If true, you get one large tensor as return. Overwrites any setting from convert_to_numpy
        :param is_pretokenized: DEPRECATED - No longer used
        :param device: Which torch.device to use for the computation
        :param num_workers: DEPRECATED - No longer used
        :return:
           By default, a list of tensors is returned. If convert_to_tensor, a stacked tensor is returned. If convert_to_numpy, a numpy matrix is returned.
        """
        self.eval()
        if show_progress_bar is None:
            show_progress_bar = (logger.getEffectiveLevel() == logging.INFO or
                                 logger.getEffectiveLevel() == logging.DEBUG)

        if convert_to_tensor:
            convert_to_numpy = False

        input_was_string = False
        if isinstance(
                sentences,
                str):  #Cast an individual sentence to a list with length 1
            sentences = [sentences]
            input_was_string = True

        if device is None:
            device = self._target_device

        self.to(device)

        all_embeddings = []
        length_sorted_idx = np.argsort(
            [self._text_length(sen) for sen in sentences])
        sentences_sorted = [sentences[idx] for idx in length_sorted_idx]

        iterator = range(0, len(sentences), batch_size)
        if show_progress_bar:
            iterator = tqdm(iterator, desc="Batches")

        for start_index in iterator:
            sentences_batch = sentences_sorted[start_index:start_index +
                                               batch_size]
            features = self.tokenize(sentences_batch)
            features = batch_to_device(features, device)

            with torch.no_grad():
                out_features = self.forward(features)
                embeddings = out_features[output_value]

                if output_value == 'token_embeddings':
                    #Set token embeddings to 0 for padding tokens
                    input_mask = out_features['attention_mask']
                    input_mask_expanded = input_mask.unsqueeze(-1).expand(
                        embeddings.size()).float()
                    embeddings = embeddings * input_mask_expanded

                embeddings = embeddings.detach()

                # fixes for #522 and #487 to avoid oom problems on gpu with large datasets
                if convert_to_numpy:
                    embeddings = embeddings.cpu()

                all_embeddings.extend(embeddings)

        all_embeddings = [
            all_embeddings[idx] for idx in np.argsort(length_sorted_idx)
        ]

        if convert_to_tensor:
            all_embeddings = torch.stack(all_embeddings)
        elif convert_to_numpy:
            all_embeddings = np.asarray(
                [emb.numpy() for emb in all_embeddings])

        if input_was_string:
            all_embeddings = all_embeddings[0]

        return all_embeddings
Пример #31
0
def do_work(collection_name,
            log_verbose,
            slurm=True,
            unobserved=False,
            post_mortem=False,
            num_exps=-1,
            filter_dict={},
            dry_run=False):
    """Pull queued experiments from the database and run them.

    Parameters
    ----------
    collection_name: str
        Name of the collection in the MongoDB.
    log_verbose: bool
        Print all the Python syscalls before running them.
    slurm: bool
        Use the Slurm cluster.
    unobserved: bool
        Disable all Sacred observers (nothing written to MongoDB).
    post_mortem: bool
        Activate post-mortem debugging.
    num_exps: int, default: -1
        If >0, will only submit the specified number of experiments to the cluster.
        This is useful when you only want to test your setup.
    filter_dict: dict
        Dictionary for filtering the entries in the collection.
    dry_run: bool
        Just return the executables and configurations instead of running them.

    Returns
    -------
    None
    """

    collection = db_utils.get_collection(collection_name)

    query_dict = {'status': {"$in": ['QUEUED']}}
    query_dict.update(filter_dict)

    if collection.count_documents(query_dict) <= 0:
        print("No queued experiments.")
        return

    exps_list = list(collection.find(query_dict))

    nexps = num_exps if num_exps > 0 else len(exps_list)
    exp_chunks = db_utils.chunk_list(exps_list[:nexps])
    njobs = len(exp_chunks)

    if dry_run:
        configs = []
        for exps in exp_chunks:
            for exp in exps:
                configs.append(
                    get_config_from_exp(exp,
                                        log_verbose=log_verbose,
                                        unobserved=unobserved,
                                        post_mortem=post_mortem))
        return configs
    elif slurm:
        print(f"Starting {nexps} experiment{s_if(nexps)} in "
              f"{njobs} Slurm job{s_if(njobs)}.")

        for exps in tqdm(exp_chunks):
            slurm_config = exps[0]['slurm']
            seml_config = exps[0]['seml']
            if 'output_dir' in slurm_config:
                warnings.warn(
                    "'output_dir' has moved from 'slurm' to 'seml'. Please adapt your YAML accordingly"
                    "by moving the 'output_dir' parameter from 'slurm' to 'seml'."
                )
            elif 'output_dir' in seml_config:
                slurm_config['output_dir'] = seml_config['output_dir']
            del slurm_config['experiments_per_job']
            start_slurm_job(collection, exps, log_verbose, unobserved,
                            post_mortem, **slurm_config)
    else:
        login_node_name = 'fs'
        if login_node_name in os.uname()[1]:
            raise ValueError(
                "Refusing to run a compute experiment on a login node. "
                "Please use Slurm or a compute node.")

        print(
            f'Starting local worker thread that will run up to {nexps} experiments, '
            f'until no queued experiments remain.')
        collection.update_many(query_dict, {"$set": {"status": "PENDING"}})
        num_exceptions = 0
        i_exp = 0

        tq = tqdm(exp_chunks)
        for exps in tq:
            for exp in exps:
                exe, config = get_config_from_exp(exp,
                                                  log_verbose=log_verbose,
                                                  unobserved=unobserved,
                                                  post_mortem=post_mortem)

                cmd = f"python {exe} with {' '.join(config)}"

                if not unobserved:
                    # check also whether PENDING experiments have their Slurm ID set, in this case they are waiting
                    # for Slurm execution and we don't start them locally.
                    db_entry = collection.find_one_and_update(
                        filter={
                            '_id': exp['_id'],
                            'status': 'PENDING',
                            'slurm.id': {
                                '$exists': False
                            }
                        },
                        update={
                            '$set': {
                                'seml.command': cmd,
                                'status': 'RUNNING'
                            }
                        },
                        upsert=False)
                    if db_entry is None:
                        # another worker already set this entry to PENDING (or at least, it's no longer QUEUED)
                        # so we ignore it.
                        continue

                if log_verbose:
                    print(f'Running the following command:\n {cmd}')
                try:
                    output_dir = "."
                    slurm_config = exps[0]['slurm']
                    seml_config = exps[0]['seml']
                    if 'output_dir' in slurm_config:
                        warnings.warn(
                            "'output_dir' has moved from 'slurm' to 'seml'. Please adapt your YAML accordingly"
                            "by moving the 'output_dir' parameter from 'slurm' to 'seml'."
                        )
                        output_dir = slurm_config['output_dir']
                    if 'output_dir' in seml_config:
                        output_dir = seml_config['output_dir']
                    output_dir_path = os.path.abspath(
                        os.path.expanduser(output_dir))
                    exp_name = slurm_config['name']

                    output_file = f"{output_dir_path}/{exp_name}_{exp['_id']}-out.txt"
                    collection.find_and_modify(
                        {'_id': exp['_id']},
                        {"$set": {
                            "seml.output_file": output_file
                        }})

                    with open(output_file, "w") as log_file:
                        # pdb works with check_call but not with check_output. Maybe because of stdout/stdin.
                        subprocess.check_call(
                            cmd,
                            shell=True,
                            stderr=log_file,
                            stdout=log_file,
                        )

                except subprocess.CalledProcessError as e:
                    num_exceptions += 1
                except IOError:
                    print(f"Log file {output_file} could not be written.")
                    # Since Sacred is never called in case of I/O error, we need to set the experiment state manually.
                    collection.find_one_and_update(
                        filter={'_id': exp['_id']},
                        update={'$set': {
                            'status': 'FAILED'
                        }},
                        upsert=False)
                finally:
                    i_exp += 1
                    tq.set_postfix(
                        failed=f"{num_exceptions}/{i_exp} experiments")