Exemple #1
0
def train():
    transform = transforms.Compose([
        transforms.Resize((356, 356)),
        transforms.RandomCrop((299, 299)),
        transforms.ToTensor(),
        transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
    ])

    train_loader = get_loader(
        root_folder= "flickt8k/images",
        annotation_file = "flickt8k/captions.txt",
        transform= transform
    )

    torch.backends.cudnn.benchmark = True 
    device = torch.device("cuda" if torch.cuda.is_available()  else "cpu")
    # hyper 
    embed_size = 256  
    hidden_size= 256 
    vocab_size = len(dataset.vocab)
    num_layers = 1 
    learning_rate = 3e-4 
    num_epochs=100 

    model = CNNtoRNN(embed_size, hidden_size, vocab_size, num_layers).to(device)
    criterion = nn.CrossEntropyLoss(ignore_index=dataset.voca.stoi["<PAD>"])
    optimizer = optim.Adam(model.parameters(), lr=learning_rate)

    model.train() 
    for epoch in range(num_epochs):
        if save_model:
            checkpoint = {
                "state_dict": model.state_dict(), 
                "optimzier": optimizer.state_dict(),
                "step": step
            }

        for idx, (imgs, captions ) in enumerate(train_loader):
            imgs = imgs.to(device)
            captions = captions.to(device)

            outputs = model(imgs, captions[:-1]) # not sending end tpoken
            loss = criterion(outputs.reshape(-1, output.shape[2]), captions.reshape(-1, output.shape[2]))
            optimizer.zero_grad()
            loss.backward(loss)
            optimizer.step()
def train():

    mean = [0.485, 0.456, 0.406]
    std = [0.229, 0.224, 0.225]
    transform = transforms.Compose([
        transforms.Resize((256, 256)),
        transforms.ToTensor(),
        transforms.Normalize(mean, std)
    ])
    data_location = './flickr8k'
    train_loader, dataset = get_loader(
        root_folder=data_location + "/Images",
        annotation_file=data_location + "/captions.txt",
        transform=transform,
        num_workers=4,
    )
    torch.backends.cudnn.benchmark = True  # Get some boost probaby
    # device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    device = 'cpu'
    load_model = False
    save_model = False
    train_CNN = False
    #Hyperparameters
    embed_size = 256
    hidden_size = 256
    vocab_size = len(dataset.vocab)
    num_layers = 2
    learning_rate = 3e-4
    num_epochs = 20

    step = 0
    # init model, loss
    model = CNNtoRNN(embed_size, hidden_size, vocab_size,
                     num_layers).to(device)
    criterion = nn.CrossEntropyLoss(ignore_index=dataset.vocab.stoi["<PAD>"])
    optimizer = optim.Adam(model.parameters(), lr=learning_rate)

    if load_model:
        step = load_checkpoint(
            torch.load("../input/checkpoint2-epoch20/my_checkpoint2.pth.tar",
                       map_location='cpu'), model, optimizer)

    model.train()
    wanna_print = 100

    for epoch in range(num_epochs):

        if save_model:
            checkpoint = {
                "state_dict": model.state_dict(),
                "optimizer": optimizer.state_dict(),
                "step": step
            }
            save_checkpoint(checkpoint)

        for idx, (imgs, captions) in enumerate(train_loader):

            imgs = imgs.to(device)
            captions = captions.to(device)

            # Don't pass the <EOS>
            outputs = model(imgs, captions[:-1])

            # loss accepts only 2 dimension
            # seq_len, N, vocabulary_size --> (seq_len, N) Each time as its own example

            print("Outputs shape ", outputs.shape)

            loss = criterion(outputs.reshape(-1, outputs.shape[2]),
                             captions.reshape(-1))

            print("Step", idx, loss.item())

            step += 1

            optimizer.zero_grad()

            loss.backward(loss)

            optimizer.step()

            if (idx + 1) % wanna_print == 0:
                print("Epoch: {} loss: {:.5f}".format(epoch, loss.item()))

                #generate the caption
                model.eval()
                with torch.no_grad():
                    dataiter = iter(train_loader)
                    img, _ = next(dataiter)
                    print(img[0].shape)
                    caps = model.caption_image(img[0:1].to(device),
                                               vocabulary=dataset.vocab)
                    caption = ' '.join(caps)
                    show_image(img[0], title=caption)
                model.train()
Exemple #3
0
def train():
    transform = transforms.Compose([
        transforms.Resize((356, 356)),
        transforms.RandomCrop((299, 299)),
        transforms.ToTensor(),
        transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),
    ])

    train_loader, dataset = get_loader("../data/flickr8k/images/",
                                       "../data/flickr8k/captions.txt",
                                       transform=transform)

    torch.backends.cudnn.benchmark = True
    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    load_model = False
    save_model = True
    train_CNN = False

    # Hyperparameters
    embed_size = 256
    hidden_size = 256
    vocab_size = len(dataset.vocab)
    num_layers = 1
    learning_rate = 3e-4
    num_epochs = 100

    # for tensorboard
    writer = SummaryWriter("runs/flickr")
    step = 0

    # initialize model, loss etc
    model = CNNtoRNN(embed_size, hidden_size, vocab_size,
                     num_layers).to(device)
    criterion = nn.CrossEntropyLoss(ignore_index=dataset.vocab.stoi["<PAD>"])
    optimizer = optim.Adam(model.parameters(), lr=learning_rate)

    # Only finetune the CNN
    for name, param in model.encoderCNN.inception.named_parameters():
        if "fc.weight" in name or "fc.bias" in name:
            param.requires_grad = True
        else:
            param.requires_grad = train_CNN

    if load_model:
        step = load_checkpoint(torch.load("my_checkpoint.pth.tar"), model,
                               optimizer)

    model.train()

    for epoch in range(num_epochs):
        # Uncomment the line below to see a couple of test cases
        # print_examples(model, device, dataset)

        if save_model:
            checkpoint = {
                "state_dict": model.state_dict(),
                "optimizer": optimizer.state_dict(),
                "step": step,
            }
            save_checkpoint(checkpoint)

        for idx, (imgs, captions) in tqdm(enumerate(train_loader),
                                          total=len(train_loader)):
            imgs = imgs.to(device)
            captions = captions.to(device)

            outputs = model(imgs, captions[:-1])
            loss = criterion(outputs.reshape(-1, outputs.shape[2]),
                             captions.reshape(-1))

            writer.add_scalar("Training loss", loss.item(), global_step=step)
            step += 1

            optimizer.zero_grad()
            loss.backward(loss)
            optimizer.step()
Exemple #4
0
def train():
    transform = transforms.Compose([
        transforms.Resize((240, 240)),
        transforms.RandomCrop(
            (224, 224)),  #the input size of inception network
        transforms.ToTensor(),
        transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225)),
    ])
    train_loader, dataset = get_loader(root_folder="archive/Images",
                                       annotation_file="archive/captions.txt",
                                       transform=transform,
                                       batch_size=128,
                                       num_workers=0)
    #Set some hyperparamters
    torch.backends.cudnn.benchmark = True  #Speed up the training process
    device = torch.device("cuda" if torch.cuda.is_available() else 'cpu')
    load_model = False
    save_model = False
    train_CNN = False
    embed_size = 256
    hidden_size = 256
    vocab_size = len(dataset.vocab)
    num_layers = 1
    learning_rate = 3e-4
    num_epochs = 100
    #for tensorboard
    writer = SummaryWriter("runs/flickr")
    step = 0
    #initialize model, loss etc
    model = CNNtoRNN(embed_size, hidden_size, vocab_size,
                     num_layers).to(device)

    # Only finetune the CNN
    for name, param in model.EncoderCNN.inception.named_parameters():
        if "fc.weight" in name or "fc.bias" in name:
            param.requires_grad = True
        else:
            param.requires_grad = train_CNN

    if load_model:
        step = load_checkpoint(torch.load("my_checkpoint.pth.tar"), model,
                               optimizer)

    criterion = nn.CrossEntropyLoss(
        ignore_index=dataset.vocab.stoi["<PAD>"])  #对于"<PAD>"的词语不需要计算损失
    optimizer = optim.Adam(filter(lambda p: p.requires_grad,
                                  model.parameters()),
                           lr=learning_rate)
    scheduler = optim.lr_scheduler.MultiStepLR(optimizer,
                                               milestones=[60, 120, 140])
    model.train()
    print('Begins')
    imgs, captions = next(iter(train_loader))
    for epoch in range(num_epochs):
        print_examples(model, device, dataset, save_path='result.txt')
        if save_model:
            checkpoint = {
                "state_dict": model.state_dict(),
                "optimizer": optimizer.state_dict(),
                "step": step
            }
            save_checkpoint(checkpoint)
        # loop = tqdm(enumerate(train_loader),total=len(train_loader),leave=False)
        total_loss = 0
        # for idx, (imgs,captions) in loop:
        imgs = imgs.to(device)
        captions = captions.to(device)

        outputs = model(imgs, captions[:-1])  #EOS标志不需要送进网络训练,我们希望他能自己训练出来
        # outputs :(seq_len, batch_size, vocabulary_size), 但是交叉熵损失接受二维的tensor
        loss = criterion(outputs.reshape(-1, outputs.shape[2]),
                         captions.reshape(-1))
        step += 1
        optimizer.zero_grad()
        loss.backward(loss)
        total_loss += loss.item()
        optimizer.step()
        print(total_loss)
def train():
    transform = transforms.Compose([
        transforms.Resize((356, 356)),
        transforms.RandomCrop((299, 299)),
        transforms.ToTensor(),
        transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
    ])

    train_loader, dataset = get_loader(root_folder='flickr8k/images/',
                                       annotation_file='flickr8k/captions.txt',
                                       transform=transform,
                                       num_workers=2)

    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
    load_model = False
    save_model = True

    embed_size = 256
    hidden_size = 256
    vocab_size = len(dataset.vocab)
    num_layers = 1
    learning_rate = 3e-4
    num_epochs = 100

    writer = SummaryWriter('logs/flickr')
    step = 0

    model = CNNtoRNN(embed_size, hidden_size, vocab_size,
                     num_layers).to(device)
    criterion = nn.CrossEntropyLoss(ignore_index=dataset.vocab.stoi['<PAD>'])
    optimizer = optim.Adam(model.parameters(), lr=learning_rate)

    if load_model:
        step = load_checkpoint(torch.load('my_ckpt.pth.tar'), model, optimizer)

    model.train()

    for epoch in range(num_epochs):
        print_examples(model, device, epoch)
        if save_model:
            checkpoint = {
                'state_dict': model.state_dict(),
                'optimizer': optimizer.state_dict(),
                'step': step,
            }
            save_checkpoint(checkpoint)

        for idx, (imgs, captions) in enumerate(train_loader):
            imgs = imgs.to(device)
            captions = captions.to(device)

            outputs = model(imgs, captions)[:-1]

            loss = criterion(outputs.reshape(-1, outputs.shape[2]),
                             captions.reshape(-1))

            writer.add_scalar('loss', loss.item(), global_step=step)
            step += 1

            optimizer.zero_grad()
            loss.backward()
            optimizer.step()
Exemple #6
0
def train():
    file_path_cap = os.path.join(Constants.data_folder_ann,
                                 Constants.captions_train_file)
    file_path_inst = os.path.join(Constants.data_folder_ann,
                                  Constants.instances_train_file)
    coco_dataloader_train, coco_data_train = get_dataloader(
        file_path_cap, file_path_inst, "train")
    file_path_cap = os.path.join(Constants.data_folder_ann,
                                 Constants.captions_val_file)
    file_path_inst = os.path.join(Constants.data_folder_ann,
                                  Constants.instances_val_file)
    coco_dataloader_val, coco_data_val = get_dataloader(
        file_path_cap, file_path_inst, "val")
    step = 0
    best_bleu4 = 0
    # initilze model, loss, etc
    model = CNNtoRNN(coco_data_train.vocab)
    model = model.to(Constants.device)
    criterion = nn.CrossEntropyLoss(
        ignore_index=coco_data_train.vocab.stoi[Constants.PAD])
    optimizer = optim.Adam(model.parameters(), lr=Hyper.learning_rate)
    #####################################################################
    if Constants.load_model:
        step = load_checkpoint(model, optimizer)

    for i in range(Hyper.total_epochs):
        model.train()  # Set model to training mode
        model.decoderRNN.train()
        model.encoderCNN.train()
        epoch = i + 1
        epochs_since_improvement = 0
        print(f"Epoch: {epoch}")
        if Constants.save_model:
            checkpoint = {
                "state_dict": model.state_dict(),
                "optimizer": optimizer.state_dict(),
                "step": step,
            }
            save_checkpoint(checkpoint)

        for _, (imgs, captions) in tqdm(enumerate(coco_dataloader_train),
                                        total=len(coco_dataloader_train),
                                        leave=False):
            imgs = imgs.to(Constants.device)
            captions = captions.to(Constants.device)
            outputs = model(imgs, captions[:-1])  # forward pass
            vocab_size = outputs.shape[2]
            outputs1 = outputs.reshape(-1, vocab_size)
            captions1 = captions.reshape(-1)
            loss = criterion(outputs1, captions1)
            optimizer.zero_grad()
            loss.backward(loss)
            optimizer.step()

        save_checkpoint_epoch(checkpoint, epoch)
        # One epoch's validation
        recent_bleu4 = validate(val_loader=coco_dataloader_val,
                                model=model,
                                criterion=criterion)

        # Check if there was an improvement
        is_best = recent_bleu4 > best_bleu4
        best_bleu4 = max(recent_bleu4, best_bleu4)
        if not is_best:
            epochs_since_improvement += 1
            print("\nEpochs since last improvement: %d\n" %
                  (epochs_since_improvement, ))
        else:
            epochs_since_improvement = 0
Exemple #7
0
def train():
    transform = transforms.Compose([
        transforms.Resize((356, 356)),
        transforms.RandomCrop((299, 299)),  # CNN takes input 299 x 299
        transforms.ToTensor(),
        transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),
    ])

    train_loader, dataset = get_loader(
        root_folder='flickr8k/images',
        annotation_file='flickr8k/captions.txt',
        transform=transform,
        num_workers=2,
    )

    # model configuration
    torch.backends.cudnn.benchmark = True
    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
    load_model = False
    save_model = False
    train_CNN = False

    # Hyperparameters
    ## We can increase capacity
    embed_size = 256
    hidden_size = 256
    vocab_size = len(dataset.vocab)
    num_layers = 1
    laerning_rate = 3e-4
    num_epochs = 100

    # for tensorboard
    writer = SummaryWriter('runs/flickr')
    step = 0

    # initialize model, loss etc
    model = CNNtoRNN(embed_size, hidden_size, vocab_size,
                     num_layers).to(device)
    criterion = nn.CrossEntropyLoss(ignore_index=dataset.vocab.stoi["<PAD>"])
    optimizer = optim.Adam(model.parameters(), lr=learning_rate)

    if load_model:
        step = load_checkpoint(
            torch.load('my_checkpoint.pth.tar'), model, optimizer
        )  # we're returning step here so that the loss fucntions continues where it ended

    model.train()

    for epoch in range(num_epochs):
        print_examples(model, device, dataset)
        if save_model:
            checkpoint = {
                "state_dict": model.state_dict(),
                "optimizer": optimizer.state_dict(),
                "step": step,
            }
            save_checkpoint(checkpoint)

        for idx, (imgs, captions) in enumerate(train_loader):
            imgs = imgs.to(device)
            captions = captions.to(device)

            outputs = model(
                imgs, captions[:-1]
            )  # we actually learn to predict the end token so we're not going to send in the end token
            loss = criterion(
                outputs.reshape(-1, outputs.shape[2]), captions.reshape(-1)
            )  #predicting for each example we're predicting for a bunch of different time steps
            # example , 20 words that it's predicting and then each word has its logit corresponding to each word in the vocabulary right here.
            ## so we have three dimensions here , but the criterion only 2 dimension
            ### output -> (seq_len, N, vocabulary_size) , target -> (seq_len , N)

            writer.add_scalar("Training loss", loss.item(), global_step=step)
            step += 1

            optimizer.zero_grad()
            loss.backward(loss)
            optimizer.step()
Exemple #8
0
def train():
    transform = transforms.Compose([
        transforms.Resize((356, 356)),
        transforms.RandomCrop((299, 299)),
        transforms.ToTensor(),
        transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5)),
    ])

    train_loader, dataset = get_loader(
        root_folder=
        "/mnt/liguanlin/DataSets/ImageCaptionDatasets/flickr8k/images",
        annotation_file=
        "/mnt/liguanlin/DataSets/ImageCaptionDatasets/flickr8k/captions.txt",
        transform=transform,
        num_workers=2,
    )

    torch.backends.cudnn.benchmark = True
    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    load_model = False
    save_model = True

    #Hyperparameters
    embed_size = 256
    hidden_size = 256
    vocab_size = len(dataset.vocab)
    num_layers = 1
    learning_rate = 3e-4
    num_epochs = 100

    #for tensorboard
    writer = SummaryWriter("runs/flickr")
    step = 0

    #initialize model, loss etc
    model = CNNtoRNN(embed_size, hidden_size, vocab_size,
                     num_layers).to(device)
    criterion = nn.CrossEntropyLoss(ignore_index=dataset.vocab.stoi["<PAD>"])
    optimizer = optim.Adam(model.parameters(), lr=learning_rate)

    if load_model:
        step = load_checkpoint(torch.load("my_checkpoint.pth.tar"), model,
                               optimizer)

    model.train()

    for epoch in range(num_epochs):
        print_examples(model, device, dataset)
        if save_model:
            checkpoint = {
                "state_dict": model.state_dict(),
                "optimizer": optimizer.state_dict(),
                "step": step,
            }
            save_checkpoint(checkpoint)

        for idx, (imgs, captions) in enumerate(train_loader):
            imgs = imgs.to(device)
            captions = captions.to(device)

            outputs = model(imgs, captions[:-1])
            loss = criterion(outputs.reshape(-1, outputs.shape[2]),
                             captions.reshape(-1))

            #record loss
            writer.add_scalar("Training loss", loss.item(), global_step=step)
            step += 1

            optimizer.zero_grad()
            loss.backward()
            optimizer.step()