def train(self):
data_manager = DataManager(self.batch_size, logger=self.logger,
is_many_to_one=self.is_many_to_one,
data_file_count=self.data_file_count,
pretrained_file=self.pre_train)
if self.is_many_to_one:
net = RNN_M2O(len(data_manager.word_list), self.embedding_len,
self.hidden_size, self.learning_rate, self.num_hidden_layer,
self.drop_rate, use_adam=True, use_cuda=self.use_cuda,
pretrained_emb=data_manager.pretrained_embeddings())
else:
net = RNN_M2M(len(data_manager.word_list), self.embedding_len,
self.hidden_size, self.learning_rate, self.num_hidden_layer,
self.drop_rate, use_adam=True, use_cuda=self.use_cuda,
pretrained_emb=data_manager.pretrained_embeddings())
self._train(net, data_manager)
def test(self, id):
_, lr, hs, nh = re.search(r'M2(M|O)_([0-9]+)_([0-9]+)_([0-9]+)_?', id).groups()
lr, hs, nh = float('0.'+lr[1:]), int(hs), int(nh)
data_manager = DataManager(self.batch_size, logger=self.logger,
is_many_to_one=self.is_many_to_one,
data_file_count=self.data_file_count,
pretrained_file=self.pre_train, is_test=True)
if self.is_many_to_one:
model = RNN_M2O
else:
model = RNN_M2M
net = model(len(data_manager.word_list), self.embedding_len,
hs, lr, nh, self.drop_rate, use_adam=True, use_cuda=self.use_cuda,
pretrained_emb=data_manager.pretrained_embeddings())
status, _epoch_index, _perplexity_history, _min_perplexity = self._load(net, id)
if status:
loss_fn = net.get_loss()
# Testing
test_losses = 0.
test_acc = 0.
test_counter = 0
net.eval()
for data, label in data_manager.test_loader():
data = T.autograd.Variable(T.LongTensor(data))
label = T.autograd.Variable(T.LongTensor(label))
if self.use_cuda:
data = data.cuda()
label = label.cuda()
output, predicted = net(data)
test_losses += loss_fn(output.view(-1, len(data_manager.word_list)), label.view(-1)) \
.data.cpu()[0] * data.size(0)
test_acc += (label.squeeze() == predicted).float().mean().data * data.size(0)
test_counter += data.size(0)
mean_test_loss = test_losses/test_counter
mean_test_acc = test_acc/test_counter
perplexity = np.exp(mean_test_loss)
self.logger.i('Loss: %.4f, Acc: %.4f, Perp: %.4f'%(mean_test_loss, mean_test_acc, perplexity))
return mean_test_loss, mean_test_acc, perplexity
else:
raise AssertionError('Model file not found!')
output_mu, output_logSigma2, latent_mu, latent_logSigma2 = CVAE(
test_minibatch)
loss = loss_function(output_mu, output_logSigma2, test_minibatch,
latent_mu, latent_logSigma2)
test_loss += loss.data[0]
#print("Test: batch_index: ", batch_index, " Loss: ", loss)
test_loss /= test_set.size()[0]
print('====> Test set loss: {:.4f} at Epoch: {:}'.format(test_loss, epoch))
return test_loss
image_path = 'data/processed'
mask_path = 'data/ISIC-2017_Training_Part1_GroundTruth'
label_file = 'data/ISIC-2017_Training_Part3_GroundTruth.csv'
mymanager = DataManager(image_path, mask_path, label_file)
#getting only sebor images
images = mymanager.get_melanoma(as_tensor=True, normalize=False)
#shuffle before splitting into test and train set
images = mymanager.shuffle_withoutLabels(images, replace=False)
#split into train and test set
train_set, test_set = mymanager.datasplit_withoutLabels(images, train_size=0.7)
seed = 1
batch_size = 5
epochs = 400
on_cuda = False
lr = 1e-4
#global parameters for early stopping ar indicated with ES_variablename
ES_Refepoches = 5 #if the test error does not decrease over this nomber of epoches, do not contiunue with training
ES_epocheCounter = 0
final_scores[m] = s
else:
final_scores[method] = score
return final_scores
if __name__ == '__main__':
# get configs
args = get_args()
print(args)
writer = SummaryWriter(log_dir=os.path.join(args.savedir, args.exp_name))
save_args(args,os.path.join(args.savedir,args.exp_name))
# load data
dm = DataManager(args)
# prepare model
model = init_model(args, dm)
model = model.cuda()
# prepare training
optimizer = Adam(lr=args.lr, params=model.parameters())
schedule = ExponentialLR(optimizer, args.lr_decay)
loss_fn = NLLLossWithLength(ignore_index=dm.word2idx['<pad>'])
# loss_fn = nn.NLLLoss(ignore_index=dm.word2idx['<pad>'])
# split data
train_data, val_data, test_data = dm.split()
train_loader = DataLoader(train_data, batch_size=args.batch_size, shuffle=True, num_workers=args.num_workers,
collate_fn=partial(collate_fn, split='train', padding_idx=dm.word2idx['<pad>']))
from pycocoevalcap.meteor.meteor import Meteor
from tensorboardX import SummaryWriter
import os
from models import init_model
from utils import NLLLossWithLength
from torch.optim.lr_scheduler import ExponentialLR
from opts import get_eval_args
from train import evaluate
if __name__ == '__main__':
# get configs
args = get_eval_args()
print(args)
# load data
dm = DataManager(args)
# prepare model
model = init_model(args, dm)
model = model.cuda()
model.load_state_dict(torch.load(args.model_path))
# split data
_, _, test_data = dm.split()
test_loader = DataLoader(test_data,
batch_size=args.batch_size,
num_workers=args.num_workers,
collate_fn=partial(
collate_fn,
split='test',
padding_idx=dm.word2idx['<pad>']))
from collections import Counter
import pickle
from dataloader import DataManager
import numpy as np
if __name__ == '__main__':
data_manager = DataManager()
data = data_manager.train_dataset.data
if data_manager.word_counter is None:
word_counter = pickle.load(open('data/word_counter', 'rb'))
else:
word_counter = data_manager.word_counter
print('Number of sentences: %d' % (len(data)))
print('Number of words: %d' % (sum(word_counter.values())))
print(
'Number of unique words: %d(w/ min freq 3) %d(w/o min freq)' %
(len([k
for k, v in word_counter.items() if v >= 3]), len(word_counter)))
unk_rate = word_counter['<unk>'] / sum(word_counter.values()) * 100.
print(
'Coverage of your limited vocabulary: %.2f%%, UNK token rate: %.2f%%' %
(100. - unk_rate, unk_rate))
print('Top 10 most frequent words: ', word_counter.most_common(10))
len_list = [len(line) for line in data]
print('Maximum sentence length: %d' % (np.max(len_list)))
print('Minimum sentence length: %d' % (np.min(len_list)))
print('Average sentence length: %.2f' % (np.mean(len_list)))
print('Sentence length variation: %.2f' % (np.std(len_list)))
# print('Distribution of classes: ', class_counter)
load = False
train_model = True
save = False
test_model = True
# Parameter
train_size = 0.9
validation_size = 0.1
# image_path = 'data/ISIC-2017_Training_Data'
image_path = 'data/processed'
mask_path = 'data/ISIC-2017_Training_Part1_GroundTruth'
label_file = 'data/ISIC-2017_Training_Part3_GroundTruth.csv'
# call class DataManger
mymanager = DataManager(image_path, mask_path, label_file)
# call class MainResNEt
r = MainResNet()
# Load images and labels
r.load_data()
# Generate dataloaders for train, test, validation
r.dataloader()
# Initialize the model
model = rs.resnet101(pretrained=False)
save_name = 'resNet101_1.pt'
# Load model
# Load / Safe
load = True
safe = True
# Parameter
train_size = 0.7
validation_size = 0.1
# image_path = 'data/ISIC-2017_Training_Data'
image_path = 'data/processed'
mask_path = 'data/ISIC-2017_Training_Part1_GroundTruth'
label_file = 'data/ISIC-2017_Training_Part3_GroundTruth.csv'
# call class DataManger
mymanager = DataManager(image_path, mask_path, label_file)
X_train, y_train, X_test, y_test, X_val, y_val = load_data()
# Initialize the variables
if torch.cuda.is_available():
X_train = X_train.cuda()
y_train = mymanager.convert_labels(y_train).cuda()
X_test = X_test.cuda()
y_test = mymanager.convert_labels(y_test).cuda()
else:
X_train = X_train
y_train = mymanager.convert_labels(y_train) #.type(torch.LongTensor)
X_test = X_test
y_test = mymanager.convert_labels(y_test) #.type(torch.LongTensor)
best_prec1 = 0.0
torch.cuda.device(1)
# Parameter
train_size = 0.7
validation_size = 0.1
start = time.time()
# image_path = 'data/ISIC-2017_Training_Data'
image_path = 'data/processed'
mask_path = 'data/ISIC-2017_Training_Part1_GroundTruth'
label_file = 'data/ISIC-2017_Training_Part3_GroundTruth.csv'
# call class DataManger
mymanager = DataManager(image_path, mask_path, label_file)
# load images and labels (actually, it is get_sebor because this set is smaller and therefore faster loaded, no time to load the whole set)
images = mymanager.get_images(as_tensor=True)
labels = mymanager.get_labels(as_tensor=True)
# check if images and labels have similar size
if len(images) != len(labels):
raise Exception('Error: Images and labels does not have equal length')
# shuffle images and labels
images, labels = mymanager.shuffle(images, labels)
# split data in train, test and validation subset
X_train, y_train, X_test, y_test, X_val, y_val = mymanager.datasplit(
images, labels, train_size=train_size, validation_size=0.1)
std = [x / 255 for x in [63.0, 62.1, 66.7]]
train_transform = transforms.Compose(
[transforms.RandomHorizontalFlip(), transforms.RandomCrop(32, padding=4), transforms.ToTensor(),
transforms.Normalize(mean, std)])
test_transform = transforms.Compose([transforms.ToTensor(), transforms.Normalize(mean, std)])
if args.dataset == 'melanoma':
# Path
image_path = './data/processed'
mask_path = './data/ISIC-2017_Training_Part1_GroundTruth'
label_file = './data/ISIC-2017_Training_Part3_GroundTruth.csv'
# Generate a DataManager
mymanager = DataManager(image_path, mask_path, label_file)
# Load the data
X_train, y_train, X_test, y_test, X_val, y_val = load_data()
y_train = mymanager.convert_labels(y_train)
y_test = mymanager.convert_labels(y_test)
train_data = torch.utils.data.TensorDataset(X_train, y_train)
test_data = torch.utils.data.TensorDataset(X_test, y_test)
# Number of labels
nlabels = 3
else:
train_data = dset.CIFAR100(args.data_path, train=True, transform=train_transform, download=True)
test_data = dset.CIFAR100(args.data_path, train=False, transform=test_transform, download=True)
nlabels = 100