def train_iters(model,
train_feature_path,
val_feature_path,
learning_rate=0.01,
batch_size=20,
epochs=50):
criterion = nn.MultiLabelSoftMarginLoss()
optimizer = torch.optim.Adam(model.parameters(),
lr=learning_rate,
weight_decay=1e-5)
# scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, 0.95)
scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, 'min')
train_loss, val_loss, avg_loss = 0, 0, 0
train_accuracy, val_accuracy = 0, 0
num_iter = len(train_feature_path) // batch_size
train_feature_vec, train_label_vec = load_features_labels(
train_feature_path)
val_feature_vec, val_label_vec = load_features_labels(val_feature_path)
print('*** loaded train and val features')
for i in range(epochs):
for j in range(num_iter):
x = Variable(
torch.FloatTensor(
np.array(train_feature_vec[j * batch_size:(j + 1) *
batch_size]))).cuda()
y = Variable(
torch.FloatTensor(
np.array(train_label_vec[j * batch_size:(j + 1) *
batch_size]))).cuda()
optimizer.zero_grad()
y_pred = model(x)
loss = criterion(y_pred, y)
avg_loss += float(loss)
loss.backward()
optimizer.step()
del x, y, y_pred
if (j + 1) % 1000 == 0:
train_loss = avg_loss / 1000
avg_loss = 0
val_loss = validate(model, val_feature_vec, val_label_vec)
scheduler.step(val_loss)
print('epoch %d: %d --- %f, %f' % (i, j, train_loss, val_loss))
writer.add_scalar('loss/train', train_loss, i)
writer.add_scalar('loss/validate', val_loss, i)
if (j + 1) % 5000 == 0:
torch.save(
model, 'output/trained_model_' + str(epochs) + '_' +
str(learning_rate) + '_' + str(j))
train_accuracy = find_accuracy(model, train_feature_vec,
train_label_vec)
val_accuracy = find_accuracy(model, val_feature_vec, val_label_vec)
print('----- validation accuracy: %f, train_accuracy: %f' %
(val_accuracy, train_accuracy))
writer.add_scalar('accuracy/train', train_accuracy)
writer.add_scalar('accuracy/val', val_accuracy)
torch.save(model,
'output/final_model_' + str(epochs) + '_' + str(learning_rate))
return train_accuracy, val_accuracy
def train():
transforms = Compose([ToTensor()])
train_dataset = CaptchaData('./data/train/', transform=transforms)
train_data_loader = DataLoader(train_dataset,
batch_size=batch_size,
num_workers=0,
shuffle=True,
drop_last=True)
test_data = CaptchaData('./data/test/', transform=transforms)
test_data_loader = DataLoader(test_data,
batch_size=batch_size,
num_workers=0,
shuffle=True,
drop_last=True)
cnn = CNN()
if torch.cuda.is_available():
cnn.cuda()
if restor:
cnn.load_state_dict(torch.load(model_path))
# freezing_layers = list(cnn.named_parameters())[:10]
# for param in freezing_layers:
# param[1].requires_grad = False
# print('freezing layer:', param[0])
optimizer = torch.optim.Adam(cnn.parameters(), lr=base_lr)
criterion = nn.MultiLabelSoftMarginLoss()
for epoch in range(max_epoch):
start_ = time.time()
loss_history = []
acc_history = []
cnn.train()
for img, target in train_data_loader:
img = Variable(img)
target = Variable(target)
if torch.cuda.is_available():
img = img.cuda()
target = target.cuda()
output = cnn(img)
loss = criterion(output, target)
optimizer.zero_grad()
loss.backward()
optimizer.step()
acc = calculat_acc(output, target)
acc_history.append(float(acc))
loss_history.append(float(loss))
print('train_loss: {:.4}|train_acc: {:.4}'.format(
torch.mean(torch.Tensor(loss_history)),
torch.mean(torch.Tensor(acc_history)),
))
loss_history = []
acc_history = []
cnn.eval()
for img, target in test_data_loader:
img = Variable(img)
target = Variable(target)
if torch.cuda.is_available():
img = img.cuda()
target = target.cuda()
output = cnn(img)
acc = calculat_acc(output, target)
acc_history.append(float(acc))
loss_history.append(float(loss))
print('test_loss: {:.4}|test_acc: {:.4}'.format(
torch.mean(torch.Tensor(loss_history)),
torch.mean(torch.Tensor(acc_history)),
))
print('epoch: {}|time: {:.4f}'.format(epoch, time.time() - start_))
torch.save(cnn.state_dict(), model_path)
def train(fold: int, verbose: int = 30):
# create folds
df = split_dataset('data/dirty_mnist_2nd_answer.csv')
df_train = df[df['kfold'] != fold].reset_index(drop=True)
df_valid = df[df['kfold'] == fold].reset_index(drop=True)
df_train.drop(['kfold'], axis=1).to_csv(f'data/train-kfold-{fold}.csv', index=False)
df_valid.drop(['kfold'], axis=1).to_csv(f'data/valid-kfold-{fold}.csv', index=False)
trainset = MnistDataset('data/dirty_mnist_2nd', f'data/train-kfold-{fold}.csv', transforms_train)
train_loader = DataLoader(trainset, batch_size=64, shuffle=True, num_workers=0)
validset = MnistDataset('data/dirty_mnist_2nd', f'data/valid-kfold-{fold}.csv', transforms_test)
valid_loader = DataLoader(validset, batch_size=16, shuffle=False, num_workers=0)
num_epochs = 10
device = 'cuda'
scaler = GradScaler()
model = EfficientNetModel().to(device)
if fold == 0:
model.load_state_dict(torch.load(f'models/effi_b4_kfold_e30/effinet_b4-f{fold}-48.pth'))
else:
model.load_state_dict(torch.load(f'models/effi_b4_kfold_e30/effinet_b4-f{fold}-28.pth'))
base_optimizer = torch.optim.SGD
optimizer = SAM(model.parameters(), base_optimizer, lr=0.001, momentum=0.9)
lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer,
step_size = 5,
gamma = 0.75)
criterion = nn.MultiLabelSoftMarginLoss()
for epoch in range(num_epochs):
model.train()
for i, (images, targets) in enumerate(train_loader):
optimizer.zero_grad()
images = images.to(device)
targets = targets.to(device)
with autocast():
outputs = model(images)
loss = criterion(outputs, targets)
loss.backward()
# sam optimizer first_steop
optimizer.first_step(zero_grad=True)
# sam optimizer second_steop
criterion(model(images), targets).backward()
optimizer.second_step(zero_grad=True)
if (i+1) % verbose == 0:
outputs = outputs > 0.0
acc = (outputs == targets).float().mean()
print(f'Fold {fold} | Epoch {epoch} | L: {loss.item():.7f} | A: {acc:.7f}')
model.eval()
valid_acc = 0.0
valid_loss = 0.0
for i, (images, targets) in enumerate(valid_loader):
images = images.to(device)
targets = targets.to(device)
with autocast():
outputs = model(images)
loss = criterion(outputs, targets)
valid_loss += loss.item()
outputs = outputs > 0.0
valid_acc += (outputs == targets).float().mean()
print(f'Fold {fold} | Epoch {epoch} | L: {valid_loss/(i+1):.7f} | A: {valid_acc/(i+1):.7f}\n')
# Learning rate 조절
lr_scheduler.step()
if epoch > num_epochs-20 and epoch < num_epochs-1:
torch.save(model.state_dict(), f'models/effinet_b4_SAM-f{fold}-{epoch}.pth')
resnet50(pretrained=True)
]
i = 0
model = models[0]
## set up paths
save_model_path = "{}/{}_{}_state_dict.pkl".format(root, root, model_names[i])
save_mat_path_fc = "{}/{}_{}_data_fc.mat".format(root, root, model_names[i])
save_mat_path_tune = "{}/{}_{}_data_tune.mat".format(root, root, model_names[i])
## resize last fully connected layer to match our problem
model.fc = nn.Linear(model.fc.in_features, 17)
model.type(dtype)
loss_fn = nn.MultiLabelSoftMarginLoss().type(dtype)
state_dict = torch.load(save_model_path,
map_location=lambda storage, loc: storage)
model.load_state_dict(state_dict)
print("{} model loaded from {}".format(model_names[i],
os.path.abspath(save_model_path)))
## first optize sigmoid thresholds
print("optimizing sigmoid cutoffs for each class")
sig_scores, y_array = get_scores(model, train_loaders[i], dtype)
sigmoid_threshold = optimize_F2(sig_scores, y_array)
print("optimal thresholds: ", sigmoid_threshold)
def __init__(self, label_embed):
super(SigmoidMLBCE, self).__init__()
self.criterion = nn.MultiLabelSoftMarginLoss()
self.num_class, self.embed_dim = label_embed.size()
self.embeddings_t = torch.t(label_embed)
def multi_criterion(logits, labels):
loss = nn.MultiLabelSoftMarginLoss()(logits, Variable(labels))
return loss
def main():
args = get_args()
if args.gpu is not None:
print("Use GPU: {} for training".format(args.gpu))
# number of classes for each dataset.
if args.dataset == 'PascalVOC':
num_classes = 20
else:
raise Exception("No dataset named {}.".format(args.dataset))
# Select Model & Method
model = models.__dict__[args.arch](pretrained=args.pretrained,
num_classes=num_classes)
if args.gpu is not None:
torch.cuda.set_device(args.gpu)
model = model.cuda(args.gpu)
# define loss function (criterion) and optimizer
criterion = nn.MultiLabelSoftMarginLoss().cuda(args.gpu)
# criterion = nn.BCEWithLogitsLoss().cuda(args.gpu)
# Take apart parameters to give different Learning Rate
param_features = []
param_classifiers = []
if args.arch.startswith('vgg'):
for name, parameter in model.named_parameters():
if 'features.' in name:
param_features.append(parameter)
else:
param_classifiers.append(parameter)
elif args.arch.startswith('resnet'):
for name, parameter in model.named_parameters():
if 'layer4.' in name or 'fc.' in name:
param_classifiers.append(parameter)
else:
param_features.append(parameter)
else:
raise Exception("Fail to recognize the architecture")
# Optimizer
optimizer = torch.optim.SGD([
{'params': param_features, 'lr': args.lr},
{'params': param_classifiers, 'lr': args.lr * args.lr_ratio}],
momentum=args.momentum,
weight_decay=args.weight_decay,
nesterov=args.nest)
# optionally resume from a checkpoint
if args.resume:
model, optimizer = load_model(model, optimizer, args)
train_loader, val_loader, test_loader = data_loader(args)
saving_dir = os.path.join(args.log_folder, args.name)
if args.evaluate:
# test_ap, test_loss = evaluate_cam(val_loader, model, criterion, args)
# test_ap, test_loss = evaluate_cam2(val_loader, model, criterion, args)
test_ap, test_loss = evaluate_cam3(val_loader, model, criterion, args)
print_progress(test_ap, test_loss, 0, 0, prefix='test')
return
# Training Phase
best_m_ap = 0
for epoch in range(args.start_epoch, args.epochs):
adjust_learning_rate(optimizer, epoch, args)
# Train for one epoch
train_ap, train_loss = \
train(train_loader, model, criterion, optimizer, epoch, args)
print_progress(train_ap, train_loss, epoch+1, args.epochs)
# Evaluate classification
val_ap, val_loss = validate(val_loader, model, criterion, epoch, args)
print_progress(val_ap, val_loss, epoch+1, args.epochs, prefix='validation')
# # Save checkpoint at best performance:
is_best = val_ap.mean() > best_m_ap
if is_best:
best_m_ap = val_ap.mean()
save_checkpoint({
'epoch': epoch + 1,
'arch': args.arch,
'state_dict': model.state_dict(),
'best_m_ap': best_m_ap,
'optimizer': optimizer.state_dict(),
}, is_best, saving_dir)
save_progress(saving_dir, train_ap, train_loss, val_ap, val_loss, args)
def loss(self, output, labels):
criterion = nn.MultiLabelSoftMarginLoss()
return criterion(output, labels)
def main(args):
# To reproduce training results
torch.manual_seed(args.seed)
if torch.cuda.is_available():
torch.cuda.manual_seed(args.seed)
# Create model directory
if not os.path.exists(args.model_path):
os.makedirs(args.model_path)
# Image Preprocessing
# For normalization, see https://github.com/pytorch/vision#models
transform = transforms.Compose([
transforms.RandomCrop(args.crop_size),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize((0.485, 0.456, 0.406), (0.229, 0.224, 0.225))
])
data_loader = get_loader(args.image_dir, args.file_list, transform,
args.batch_size, True)
model = EncoderCNN_singleimage_frontal(args.num_classes)
if args.pretrained:
model_pretrained = torch.load(args.pretrained)
model.load_state_dict(model_pretrained, strict=False)
# Get starting epoch #, note that model is named as '...your path to model/algoname-epoch#.pkl'
# A little messy here.
start_epoch = int(
args.pretrained.split('/')[-1].split('-')[1].split('.')[0]) + 1
else:
start_epoch = 1
# mult_label loss
#bce_loss = nn.BCELoss()
MIL_loss = nn.MultiLabelSoftMarginLoss()
# changed to GPU mode if available
if torch.cuda.is_available():
model.cuda()
#bce_loss.cuda()
MIL_loss.cuda()
#dp_model = torch.nn.DataParallel(model, device_ids=[0,1])
# Train the model
total_step = len(data_loader)
print('len(data_loader)', total_step)
learning_rate = args.learning_rate
# starting training
max_ap = 0
for epoch in range(start_epoch, args.num_epochs + 1):
# start decay learning rate
if epoch > args.lr_decay:
frac = (epoch - args.lr_decay) / args.learning_rate_decay_every
decay_rator = math.pow(0.8, frac)
# decay the learning rate
learning_rate = args.learning_rate * decay_rator
print('Learning rate for Epoch %d: %.6f' % (epoch, learning_rate))
# Constructing model parameters for optimization
optimizer = torch.optim.Adam(model.parameters(),
lr=learning_rate,
betas=(args.alpha, args.beta))
print('-------------Train for Epoch %d --------------' % epoch)
for i, (images, image_name, image_labels) in enumerate(data_loader):
images = to_var(images)
image_labels = torch.FloatTensor(image_labels)
image_labels = to_var(image_labels)
model.train()
#dp_model.train()
optimizer.zero_grad()
outputs = model(images)
loss = MIL_loss(outputs, image_labels)
loss.backward()
optimizer.step()
# Print log info
if i % args.log_step == 0:
print('Epoch [%d/%d], Step [%d/%d], BCE Loss: %.4f' %
(epoch, args.num_epochs, i, total_step, loss.item()))
gts = []
outputs = []
with torch.no_grad():
for i, (images, image_name,
image_labels) in enumerate(data_loader):
images = to_var(images)
model.eval()
#dp_model.train()
output = model(images)
output = sigmoid(output)
output = output.data.cpu().numpy()
gts.append(image_labels)
outputs.append(output)
if i == 96:
break
outputs = np.stack(outputs, 1).reshape(-1, 82)
gts = np.stack(gts, 1).reshape(-1, 82)
# outputs = outputs.data.cpu().numpy()
# image_labels = image_labels.data.cpu().numpy()
outs = []
for cls in range(82):
#print(roc_auc_score( gts[:,cls],predictions[:,cls]))
ap = voc_eval(cls, outputs, gts)
outs.append(ap)
print(np.mean(outs))
print(outs)
# if np.mean(outs) > 0.13:
# np.savetxt(args.predictions, outputs)
# np.savetxt(args.gts, gts)
# return
# if epoch % 1 == 0:
if np.mean(outs) > max_ap:
max_ap = np.mean(outs)
# Save the model after each epoch
torch.save(
model.state_dict(),
os.path.join(args.model_path,
'Resnet_single_image-%d.pkl' % (epoch)))
def __init__(self):
super(LogLoss, self).__init__()
self.classify_loss = nn.MultiLabelSoftMarginLoss()
self.__classes_num = NUM_CLASSES
def __init__(self, H_PARAM, embed_layer, bert=None):
super(KeyWordPredictor, self).__init__()
self.N_word = H_PARAM['N_WORD']
self.N_depth = H_PARAM['N_depth']
self.N_h = H_PARAM['N_h']
self.gpu = H_PARAM['gpu']
self.use_hs = H_PARAM['use_hs']
self.table_type = H_PARAM['table_type']
self.acc_num = 3
self.embed_layer = embed_layer
self.use_bert = True if bert else False
if bert:
self.q_bert = bert
encoded_num = 768
else:
self.q_lstm = nn.LSTM(input_size=self.N_word,
hidden_size=self.N_h // 2,
num_layers=self.N_depth,
batch_first=True,
dropout=0.3,
bidirectional=True)
encoded_num = self.N_h
self.hs_lstm = nn.LSTM(input_size=self.N_word,
hidden_size=self.N_h // 2,
num_layers=self.N_depth,
batch_first=True,
dropout=0.3,
bidirectional=True)
self.kw_lstm = nn.LSTM(input_size=self.N_word,
hidden_size=self.N_h // 2,
num_layers=self.N_depth,
batch_first=True,
dropout=0.3,
bidirectional=True)
self.q_num_att = nn.Linear(encoded_num, self.N_h)
self.hs_num_att = nn.Linear(self.N_h, self.N_h)
self.kw_num_out_q = nn.Linear(encoded_num, self.N_h)
self.kw_num_out_hs = nn.Linear(self.N_h, self.N_h)
self.kw_num_out = nn.Sequential(nn.Tanh(),
nn.Linear(self.N_h,
4)) # num of key words: 0-3
self.q_att = nn.Linear(encoded_num, self.N_h)
self.hs_att = nn.Linear(self.N_h, self.N_h)
self.kw_out_q = nn.Linear(encoded_num, self.N_h)
self.kw_out_hs = nn.Linear(self.N_h, self.N_h)
self.kw_out_kw = nn.Linear(self.N_h, self.N_h)
self.kw_out = nn.Sequential(nn.Tanh(), nn.Linear(self.N_h, 1))
self.softmax = nn.Softmax() #dim=1
self.CE = nn.CrossEntropyLoss()
self.log_softmax = nn.LogSoftmax()
self.mlsml = nn.MultiLabelSoftMarginLoss()
self.bce_logit = nn.BCEWithLogitsLoss()
self.sigm = nn.Sigmoid()
if self.gpu:
self.cuda()
def __init__(self, H_PARAM, embed_layer):
super(SuperModel, self).__init__()
self.train_component = H_PARAM['train_component']
self.gpu = H_PARAM['gpu']
self.B_word = H_PARAM['B_WORD']
self.N_word = H_PARAM['N_WORD']
self.N_h = H_PARAM['N_h']
self.N_depth = H_PARAM['N_depth']
self.trainable_emb = H_PARAM['trainable_emb']
self.table_type = H_PARAM['table_type']
self.use_hs = H_PARAM['use_hs']
self.with_from = H_PARAM['with_from']
self.SQL_TOK = [
'<UNK>', '<END>', 'WHERE', 'AND', 'EQL', 'GT', 'LT', '<BEG>'
]
self.use_bert = H_PARAM['use_bert'] #False
self.table_type = H_PARAM['table_type']
self.acc_num = 0
self.table_dict = None
self.test_db_ids = set()
self.save_dir = H_PARAM['save_dir']
# word embedding layer
self.embed_layer = embed_layer
# initial all modules
self.multi_sql = MultiSqlPredictor(H_PARAM)
self.multi_sql.eval()
self.key_word = KeyWordPredictor(H_PARAM)
self.key_word.eval()
self.col = ColPredictor(H_PARAM)
self.col.eval()
self.op = OpPredictor(H_PARAM)
self.op.eval()
self.agg = AggPredictor(H_PARAM)
self.agg.eval()
self.root_teminal = RootTeminalPredictor(H_PARAM)
self.root_teminal.eval()
self.des_asc = DesAscLimitPredictor(H_PARAM)
self.des_asc.eval()
self.having = HavingPredictor(H_PARAM)
self.having.eval()
self.andor = AndOrPredictor(H_PARAM)
self.andor.eval()
self.from_table = FromPredictor(H_PARAM)
self.from_table.eval()
self.softmax = nn.Softmax() #dim=1
self.CE = nn.CrossEntropyLoss()
self.log_softmax = nn.LogSoftmax()
self.mlsml = nn.MultiLabelSoftMarginLoss()
self.bce_logit = nn.BCEWithLogitsLoss()
self.sigm = nn.Sigmoid()
if self.gpu:
self.cuda()
self.path_not_found = 0
def main():
global args, best_prec1
args = parser.parse_args()
args.distributed = args.world_size > 1
# create model
print("=> creating model '{}'".format(args.arch))
if args.arch == 'localizer_alexnet':
model = localizer_alexnet(pretrained=args.pretrained)
elif args.arch == 'localizer_alexnet_robust':
model = localizer_alexnet_robust(pretrained=args.pretrained)
print(model)
model.features = torch.nn.DataParallel(model.features)
model.cuda()
# TODO:
# define loss function (criterion) and optimizer
criterion = nn.MultiLabelSoftMarginLoss().cuda()
optimizer = torch.optim.SGD(model.parameters(),
args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
#optimizer = torch.optim.Adam(model.parameters(), lr=args.lr,momentum=args.momentum,weight_decay=args.weight_decay)
# optionally resume from a checkpoint
if args.resume:
if os.path.isfile(args.resume):
print("=> loading checkpoint '{}'".format(args.resume))
checkpoint = torch.load(args.resume)
args.start_epoch = checkpoint['epoch']
best_prec1 = checkpoint['best_prec1']
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
print("=> loaded checkpoint '{}' (epoch {})".format(
args.resume, checkpoint['epoch']))
else:
print("=> no checkpoint found at '{}'".format(args.resume))
cudnn.benchmark = True
random.seed(
100
) #Fixing seed so that dataloader generates the same random batches always.
np.random.seed(100)
# Data loading code
# TODO: Write code for IMDBDataset in custom.py
trainval_imdb = get_imdb('voc_2007_trainval')
test_imdb = get_imdb('voc_2007_test')
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
train_dataset = IMDBDataset(
trainval_imdb,
transforms.Compose([
transforms.Resize((512, 512)),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
normalize,
]))
train_sampler = None
train_loader = torch.utils.data.DataLoader(train_dataset,
batch_size=args.batch_size,
shuffle=(train_sampler is None),
num_workers=args.workers,
pin_memory=True,
sampler=train_sampler)
val_loader = torch.utils.data.DataLoader(IMDBDataset(
test_imdb,
transforms.Compose([
transforms.Resize((384, 384)),
transforms.ToTensor(),
normalize,
])),
batch_size=args.batch_size,
shuffle=False,
num_workers=args.workers,
pin_memory=True)
if args.evaluate:
validate(val_loader, model, criterion)
return
# TODO: Create loggers for visdom and tboard
# TODO: You can pass the logger objects to train(), make appropriate
# modifications to train()
if args.vis:
import visdom
vis = visdom.Visdom(
server='http://ec2-3-21-126-90.us-east-2.compute.amazonaws.com/',
port='8097'
) #Change Address as per need. Can change to commandline arg
else:
vis = None
timestr = time.strftime("%Y%m%d-%H%M%S")
save_path = "runs/" + timestr + '/'
writer = SummaryWriter(save_path)
#Initializing model weights
pretrained_alexnet = models.alexnet(pretrained=True)
conv_layer_numbers = [0, 3, 6, 8, 10]
for elt in conv_layer_numbers:
#print(model.features.module[elt])
model.features.module[elt].weight.data.copy_(
pretrained_alexnet.features[elt].weight.data)
model.features.module[elt].bias.data.copy_(
pretrained_alexnet.features[elt].bias.data)
xavier_initialized_conv_layers = [0, 2, 4]
for elt in xavier_initialized_conv_layers:
torch.nn.init.xavier_uniform_(model.classifier[elt].weight)
print("Registering backward hook for final conv layer")
model.classifier[4].register_backward_hook(hook)
iter_cnt = 0
for epoch in range(args.start_epoch, args.epochs):
print("Epoch count is: ", epoch, "\t", "Iteration Count is: ",
iter_cnt)
adjust_learning_rate(optimizer, epoch)
# train for one epoch
iter_cnt = train(train_loader, model, criterion, optimizer, epoch,
iter_cnt, writer, vis)
#Obtaining histogram of weights and histogram of gradients of weights
'''for i,elt in enumerate(conv_layer_numbers):
weight_tag = "feature_conv_" + str(i) + "_weight"
grad_tag = "feature_conv_" + str(i) + "_grad"
pdb.set_trace()
writer.add_histogram(weight_tag, model.features.module[elt].weight.data.cpu(), epoch)
writer.add_histogram(grad_tag, model.features.module[elt].weight.grad.data.cpu(), epoch)
for i,elt in enumerate(xavier_initialized_conv_layers):
weight_tag = "classifier_conv_" + str(i+len(conv_layer_numbers)) + "_weight"
grad_tag = "classifier_conv_" + str(i+len(conv_layer_numbers)) + "_grad"
writer.add_histogram(weight_tag, model.classifier[elt].weight.data.cpu(), epoch)
writer.add_histogram(grad_tag, model.classifier[elt].weight.grad.data.cpu(), epoch)'''
# evaluate on validation set
if epoch % args.eval_freq == 0 or epoch == args.epochs - 1:
print("Evaluating on Validation Set. Epoch number is: ", epoch)
m1, m2 = validate(val_loader, model, criterion, epoch, writer)
# score = m1 #TODO: Delete this line and uncomment the line below later
score = m1 * m2
# remember best prec@1 and save checkpoint
is_best = score > best_prec1
best_prec1 = max(score, best_prec1)
save_checkpoint(
{
'epoch': epoch + 1,
'arch': args.arch,
'state_dict': model.state_dict(),
'best_prec1': best_prec1,
'optimizer': optimizer.state_dict(),
}, is_best)
writer.close()
model = Net()
model.cuda()
params = list(model.parameters())
print(model)
print('[init] number of parameters %d' % (len(params)))
for i in params:
print(i.size())
optimizer = optim.SGD(model.parameters(),
lr=learn_rate_init,
momentum=momentum_coeff)
spweight = loadmat_transpose(path_weight)
spweight = torch.from_numpy(spweight['ratio'][0] * 100).cuda()
criterion = nn.MultiLabelSoftMarginLoss(weight=spweight).cuda()
def dataset_dimension(index, select):
#note: select can be in any order! [7, 3, 9, 11, 4, ...]
var_samples = 0
var_width = index[1][1]
lab_samples = 0
lab_width = index[1][3]
for i in select:
var_samples += index[i][0]
lab_samples += index[i][2]
return var_samples, var_width, lab_samples, lab_width
def main(argv):
parser = argparse.ArgumentParser(description='CNN baseline for DSTC5 SAP Task')
parser.add_argument('--trainset', dest='trainset', action='store', metavar='TRAINSET', required=True, help='')
parser.add_argument('--testset', dest='testset', action='store', metavar='TESTSET', required=True, help='')
parser.add_argument('--dataroot', dest='dataroot', action='store', required=True, metavar='PATH', help='')
parser.add_argument('--roletype', dest='roletype', action='store', choices=['guide', 'tourist'], required=True, help='speaker')
args = parser.parse_args()
train_utters = []
trainset = dataset_walker.dataset_walker(args.trainset, dataroot=args.dataroot, labels=True, translations=True)
sys.stderr.write('Loading training instances ... ')
for call in trainset:
for (log_utter, translations, label_utter) in call:
if log_utter['speaker'].lower() != args.roletype:
continue
transcript = data_helpers.tokenize_and_lower(log_utter['transcript'])
speech_act = label_utter['speech_act']
sa_label_list = []
for sa in speech_act:
sa_label_list += ['%s_%s' % (sa['act'], attr) for attr in sa['attributes']]
sa_label_list = sorted(set(sa_label_list))
train_utters += [(transcript, log_utter['speaker'], sa_label_list)]
sys.stderr.write('Done\n')
test_utters = []
testset = dataset_walker.dataset_walker(args.testset, dataroot=args.dataroot, labels=True, translations=True)
sys.stderr.write('Loading testing instances ... ')
for call in testset:
for (log_utter, translations, label_utter) in call:
if log_utter['speaker'].lower() != args.roletype:
continue
try:
translation = data_helpers.tokenize_and_lower(translations['translated'][0]['hyp'])
except:
translation = ''
speech_act = label_utter['speech_act']
sa_label_list = []
for sa in speech_act:
sa_label_list += ['%s_%s' % (sa['act'], attr) for attr in sa['attributes']]
sa_label_list = sorted(set(sa_label_list))
test_utters += [(translation, log_utter['speaker'], sa_label_list)]
pprint(train_utters[:2])
pprint(test_utters[:2])
# load parameters
params = data_helpers.load_params("parameters/cnn.txt")
pprint(params)
num_epochs = int(params['num_epochs'])
validation_split = float(params['validation_split'])
batch_size = int(params['batch_size'])
multilabel = params['multilabel']=="true"
# build vocabulary
sents = [utter[0].split(' ') for utter in train_utters]
max_sent_len = int(params['max_sent_len'])
pad_sents = data_helpers.pad_sentences(sents, max_sent_len)
vocabulary, inv_vocabulary = data_helpers.build_vocab(pad_sents)
print("vocabulary size: %d" % len(vocabulary))
# params['max_sent_len'] = max_sent_len
# build inputs
train_inputs = data_helpers.build_input_data(pad_sents, vocabulary)
test_sents = [utter[0].split(' ') for utter in test_utters]
test_pad_sents = data_helpers.pad_sentences(test_sents, max_sent_len)
test_inputs = data_helpers.build_input_data(test_pad_sents, vocabulary)
# build labels
sa_train_labels = [utter[2] for utter in train_utters]
sa_test_labels = [utter[2] for utter in test_utters]
label_binarizer = preprocessing.MultiLabelBinarizer()
label_binarizer.fit(sa_train_labels+sa_test_labels)
train_labels = label_binarizer.transform(sa_train_labels)
test_labels = label_binarizer.transform(sa_test_labels)
# split and shuffle data
indices = np.arange(train_inputs.shape[0])
np.random.shuffle(indices)
train_inputs = train_inputs[indices]
train_labels = train_labels[indices]
num_validation = int(validation_split * train_inputs.shape[0])
# x_train = train_inputs[:-num_validation]
# y_train = train_labels[:-num_validation]
# x_val = train_inputs[-num_validation:]
# y_val = train_labels[-num_validation:]
x_train = train_inputs
y_train = train_labels
x_test = test_inputs
y_test_numpy = test_labels
# construct a pytorch data_loader
x_train = torch.from_numpy(x_train).long()
y_train = torch.from_numpy(y_train).float()
dataset_tensor = data_utils.TensorDataset(x_train, y_train)
train_loader = data_utils.DataLoader(dataset_tensor, batch_size=batch_size, shuffle=True, num_workers=4,
pin_memory=False)
x_test = torch.from_numpy(x_test).long()
y_test = torch.from_numpy(y_test_numpy).long()
dataset_tensor = data_utils.TensorDataset(x_test, y_test)
test_loader = data_utils.DataLoader(dataset_tensor, batch_size=batch_size, shuffle=False, num_workers=4,
pin_memory=False)
# load pre-trained word embeddings
embedding_dim = int(params['embedding_dim'])
embedding_matrix = data_helpers.load_embedding(vocabulary, embedding_dim=embedding_dim, embedding=params['embedding'])
# load model
model = SluConvNet(params, embedding_matrix, len(vocabulary), y_train.shape[1])
if torch.cuda.is_available():
model = model.cuda()
learning_rate = float(params['learning_rate'])
optimizer = optim.Adam(model.parameters(), lr=learning_rate)
# loss_fn = nn.BCEWithLogitsLoss()
loss_fn = nn.MultiLabelSoftMarginLoss()
for epoch in range(num_epochs):
model.train() # set the model to training mode (apply dropout etc)
for i, (inputs, labels) in enumerate(train_loader):
inputs, labels = autograd.Variable(inputs), autograd.Variable(labels)
if torch.cuda.is_available():
inputs, labels = inputs.cuda(), labels.cuda()
preds = model(inputs)
if torch.cuda.is_available():
preds = preds.cuda()
loss = loss_fn(preds, labels)
optimizer.zero_grad()
loss.backward()
optimizer.step()
if i % 100 == 0:
print("current loss: %.4f" % loss)
model.eval() # set the model to evaluation mode
true_acts, pred_acts, metrics = evaluate(model, label_binarizer, test_loader, y_test_numpy)
print("Precision: %.4f\tRecall: %.4f\tF1-score: %.4f\n" % (metrics[0], metrics[1], metrics[2]))
# end of training
true_acts, pred_acts, metrics = evaluate(model, label_binarizer, test_loader, y_test_numpy)
print("Precision: %.4f\tRecall: %.4f\tF1-score: %.4f\n" % (metrics[0], metrics[1], metrics[2]))
with open(("pred_result_%s.txt" % args.roletype), "w") as f:
for pred_act, true_act in zip(pred_acts, true_acts):
f.write("pred: %s\ntrue: %s\n\n" % (', '.join(pred_act), ', '.join(true_act)))
['glu', nn.GLU()],
])
loss = nn.ModuleDict([
['l1', nn.L1Loss()],
['nll', nn.NLLLoss()],
['kldiv', nn.KLDivLoss()],
['mse', nn.MSELoss()],
['bce', nn.BCELoss()],
['bce_with_logits', nn.BCEWithLogitsLoss()],
['cosine_embedding', nn.CosineEmbeddingLoss()],
['ctc', nn.CTCLoss()],
['hinge_embedding', nn.HingeEmbeddingLoss()],
['margin_ranking', nn.MarginRankingLoss()],
['multi_label_margin', nn.MultiLabelMarginLoss()],
['multi_label_soft_margin', nn.MultiLabelSoftMarginLoss()],
['multi_margin', nn.MultiMarginLoss()],
['smooth_l1', nn.SmoothL1Loss()],
['soft_margin', nn.SoftMarginLoss()],
['cross_entropy', nn.CrossEntropyLoss()],
['triplet_margin', nn.TripletMarginLoss()],
['poisson_nll', nn.PoissonNLLLoss()]
])
def _parse(
identifier: typing.Union[str, typing.Type[nn.Module], nn.Module],
dictionary: nn.ModuleDict,
target: str
) -> nn.Module:
"""
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
torch.backends.cudnn.enabled = False
model = MnistModel().to(device)
print(summary(model, input_size=(1, 3, 256, 256), verbose=0))
# 학습하기
# 무진님이 주신 에러 대응 코드 사용!
# 실행 하는 곳이 메인인 경우
if __name__ == '__main__':
# 옵티마이저와 멀티라벨소프트 마진 로스를 사용함
optimizer = optim.Adam(model.parameters(), lr=1e-3)
criterion = nn.MultiLabelSoftMarginLoss()
# 에포치 10주고 모델을 트레인으로 변환
num_epochs = 40
model.train()
# 에포치 만큼 반복
for epoch in range(num_epochs):
# 배치 사이즈 만큼 스탭을 진행함
for i, (images, targets) in enumerate(train_loader):
# 미분 값 초기화
optimizer.zero_grad()
# 데이터셋을 프로세스에 입력함
images = images.to(device)
targets = targets.to(device)
def main(opt):
# Set the random seed manually for reproducibility.
if torch.cuda.is_available():
torch.cuda.manual_seed(opt.seed)
else:
torch.manual_seed(opt.seed)
train_opt = {
'label_file': opt.train_label,
'imageinfo_file': opt.train_imageinfo,
'image_dir': opt.train_image_dir,
'batch_size': opt.batch_size,
'num_workers': opt.num_workers,
'train': True
}
val_opt = {
'label_file': opt.val_label,
'imageinfo_file': opt.val_imageinfo,
'image_dir': opt.val_image_dir,
'batch_size': opt.batch_size,
'num_workers': opt.num_workers,
'train': False
}
train_loader = get_data_loader(train_opt)
val_loader = get_data_loader(val_opt)
logger.info('Building model...')
num_labels = train_loader.dataset.get_num_labels()
model = DepNet(num_labels, finetune=opt.finetune, cnn_type=opt.cnn_type)
criterion = nn.MultiLabelSoftMarginLoss()
optimizer = optim.Adam(model.parameters(), lr=opt.learning_rate)
if torch.cuda.is_available():
model.cuda()
criterion.cuda()
logger.info("Start training...")
best_loss = sys.maxsize
best_epoch = 0
for epoch in range(opt.num_epochs):
learning_rate = adjust_learning_rate(opt, optimizer, epoch)
logger.info('===> Learning rate: %f: ', learning_rate)
# train for one epoch
train(opt, model, criterion, optimizer, train_loader, epoch)
# validate at every val_step epoch
if epoch % opt.val_step == 0:
logger.info("Start evaluating...")
val_loss, val_score = test(opt, model, criterion, val_loader)
logger.info('Val loss: \n%s', val_loss)
logger.info('Val score: \n%s', val_score)
loss = val_loss.avg
if loss < best_loss:
logger.info('Found new best loss: %.7f, previous loss: %.7f',
loss, best_loss)
best_loss = loss
best_epoch = epoch
logger.info('Saving new checkpoint to: %s', opt.output_file)
torch.save(
{
'epoch': epoch,
'model': model.state_dict(),
'best_loss': best_loss,
'best_epoch': best_epoch,
'opt': opt
}, opt.output_file)
else:
logger.info('Current loss: %.7f, best loss is %.7f @ epoch %d',
loss, best_loss, best_epoch)
if epoch - best_epoch > opt.max_patience:
logger.info('Terminated by early stopping!')
break
def __init__(self,
N_word,
N_h,
N_depth,
gpu,
use_hs,
dr=0.3,
temperature=False):
super(KeyWordPredictor, self).__init__()
self.N_h = N_h
self.gpu = gpu
self.use_hs = use_hs
self.q_lstm = nn.LSTM(input_size=N_word,
hidden_size=N_h / 2,
num_layers=N_depth,
batch_first=True,
dropout=dr,
bidirectional=True)
self.hs_lstm = nn.LSTM(input_size=N_word,
hidden_size=N_h / 2,
num_layers=N_depth,
batch_first=True,
dropout=dr,
bidirectional=True)
self.kw_lstm = nn.LSTM(input_size=N_word,
hidden_size=N_h / 2,
num_layers=N_depth,
batch_first=True,
dropout=dr,
bidirectional=True)
self.q_num_att = nn.Linear(N_h, N_h)
self.hs_num_att = nn.Linear(N_h, N_h)
self.kw_num_out_q = nn.Linear(N_h, N_h)
self.kw_num_out_hs = nn.Linear(N_h, N_h)
self.kw_num_out = nn.Sequential(nn.Tanh(),
nn.Linear(N_h,
4)) # num of key words: 0-3
self.q_att = nn.Linear(N_h, N_h)
self.hs_att = nn.Linear(N_h, N_h)
self.kw_out_q = nn.Linear(N_h, N_h)
self.kw_out_hs = nn.Linear(N_h, N_h)
self.kw_out_kw = nn.Linear(N_h, N_h)
self.kw_out = nn.Sequential(nn.Tanh(), nn.Linear(N_h, 1))
self.softmax = nn.Softmax() #dim=1
self.CE = nn.CrossEntropyLoss()
self.log_softmax = nn.LogSoftmax()
self.mlsml = nn.MultiLabelSoftMarginLoss()
self.bce_logit = nn.BCEWithLogitsLoss()
self.sigm = nn.Sigmoid()
if temperature:
self.T1 = nn.Parameter(torch.FloatTensor([1.])) # kw_num_out
self.T2 = nn.Parameter(torch.FloatTensor([1.])) # kw_out
else:
self.T1 = self.T2 = 1.0
if gpu:
self.cuda()
def main():
cnn = CNN()
#resume
resume = './model_cos_90_0.01.pkl'
checkpoint = torch.load(resume)
cnn.load_state_dict(checkpoint)
if use_cuda:
cnn = CNN().cuda()
cnn.train()
print('init net')
criterion = nn.MultiLabelSoftMarginLoss()
optimizer = torch.optim.Adam(cnn.parameters(), lr=learning_rate)
best_acc = 0
# Train the Model
train_dataloader = my_dataset.get_finetune_data_loader()
for epoch in range(num_epochs):
correct = 0
total = 0
for i, (images, labels) in enumerate(train_dataloader):
#lr_schedule
lr = cosine_anneal_schedule(epoch)
for param_group in optimizer.param_groups:
#print(param_group['lr'])
param_group['lr'] = lr
images = Variable(images).cuda()
labels = Variable(labels.float()).cuda()
predict_labels = cnn(images)
# print(predict_labels.type)
# print(labels.type)
loss = criterion(predict_labels, labels)
optimizer.zero_grad()
loss.backward()
optimizer.step()
predict_labels = predict_labels.cpu()
labels = labels.cpu()
c0 = captcha_setting.ALL_CHAR_SET[np.argmax(predict_labels[0, 0:captcha_setting.ALL_CHAR_SET_LEN].data.numpy())]
c1 = captcha_setting.ALL_CHAR_SET[np.argmax(predict_labels[0, captcha_setting.ALL_CHAR_SET_LEN:2 * captcha_setting.ALL_CHAR_SET_LEN].data.numpy())]
c2 = captcha_setting.ALL_CHAR_SET[np.argmax(predict_labels[0,
2 * captcha_setting.ALL_CHAR_SET_LEN:3 * captcha_setting.ALL_CHAR_SET_LEN].data.numpy())]
c3 = captcha_setting.ALL_CHAR_SET[np.argmax(predict_labels[0,
3 * captcha_setting.ALL_CHAR_SET_LEN:4 * captcha_setting.ALL_CHAR_SET_LEN].data.numpy())]
# c = '%s%s%s%s' % (c0, c1, c2, c3)
# print(c)
predict_labels = '%s%s%s%s' % (c0, c1, c2, c3)
# predict = '%s%s%s%s' % (c0, c1, c2, c3)
# predict_label = one_hot_encoding.decode(predict.numpy()[0])
true_label = one_hot_encoding.decode(labels.numpy()[0])
total += labels.size(0)
print(predict_labels, true_label)
if (predict_labels == true_label):
correct += 1
acc = 100 * correct / total
# if (total % 200 == 0):
# print('Test Accuracy of the model on the %d train images: %f %%' % (total, 100 * correct / total))
if (i+1) % 10 == 0:
print("epoch:", epoch, "step:", i, "loss:", loss.item(),"Accuracy:", acc)
if acc >= best_acc:
torch.save(cnn.state_dict(), "./model_cos_90_0.01_finetune_2.pkl") #current is model.pkl
print("save model")
#print("epoch:", epoch, "loss:", loss.item(),"Accuracy:", acc)
#torch.save(cnn.state_dict(), "./model_cos_90_0.01_finetune.pkl") #current is model.pkl
print("save last model")
nn.BatchNorm2d(512),
nn.ReLU(inplace=True),
nn.Conv2d(512, 1024, stride=1, kernel_size=3,
padding=1), # 1024,22,8
nn.BatchNorm2d(1024),
nn.ReLU(inplace=True),
nn.MaxPool2d(kernel_size=2, stride=2), # 1024,11,4
)
self.fc = nn.Sequential(nn.Linear(45056, 2048), nn.ReLU(inplace=True),
nn.Linear(2048, 1024), nn.ReLU(inplace=True),
nn.Linear(1024, 248))
def forward(self, x):
x = self.layer1(x)
x = self.layer2(x)
x = self.layer3(x)
x = self.layer4(x)
x = x.view(x.size(0), -1)
x = self.fc(x)
return x
net = CNN_Network()
net.to(DEVICE)
optimizer = torch.optim.Adam(net.parameters(), lr=lr)
loss = nn.MultiLabelSoftMarginLoss()
dummy_input = torch.rand(16, 1, 35, 90)
model = CNN_Network()
with SummaryWriter(comment='Net', log_dir='scalar') as w:
w.add_graph(model, (dummy_input, ))
def _compute_bce_discriminator_loss(netD, real_imgs, fake_imgs,
real_labels, fake_labels, real_catelabels,
conditions, gpus):
cate_criterion =nn.MultiLabelSoftMarginLoss()
criterion = nn.BCELoss()
ratio = 1.0
# gp_lambda = 0.00005
batch_size = real_imgs.size(0)
cond = conditions.detach()
fake = fake_imgs.detach()
real_features = nn.parallel.data_parallel(netD, (real_imgs), gpus)
fake_features = nn.parallel.data_parallel(netD, (fake), gpus)
# real pairs
inputs = (real_features, cond)
real_logits = nn.parallel.data_parallel(netD.get_cond_logits, inputs, gpus)
errD_real = criterion(real_logits, real_labels)
# wrong pairs
inputs = (real_features[:(batch_size-1)], cond[1:])
wrong_logits = \
nn.parallel.data_parallel(netD.get_cond_logits, inputs, gpus)
errD_wrong = criterion(wrong_logits, fake_labels[1:])
# fake pairs
inputs = (fake_features, cond)
fake_logits = nn.parallel.data_parallel(netD.get_cond_logits, inputs, gpus)
errD_fake = criterion(fake_logits, fake_labels)
# gradient penalty
# epsilon = torch.rand([len(real_imgs)] + [1 for _ in range(len(real_imgs.shape)-1)], \
# device='cuda', requires_grad=True)
# grad = _get_gradient(netD, real_imgs, fake_imgs, epsilon).contiguous()
# grad -= grad.min(1, keepdim=True)[0]
# grad /= grad.max(1, keepdim=True)[0]
# gp = _gradient_penalty(grad)
if netD.get_uncond_logits is not None:
real_logits = \
nn.parallel.data_parallel(netD.get_uncond_logits,
(real_features), gpus)
fake_logits = \
nn.parallel.data_parallel(netD.get_uncond_logits,
(fake_features), gpus)
uncond_errD_real = criterion(real_logits, real_labels)
uncond_errD_fake = criterion(fake_logits, fake_labels)
errD = ((errD_real + uncond_errD_real) / 2. +
(errD_fake + errD_wrong + uncond_errD_fake) / 3.)
errD_real = (errD_real + uncond_errD_real) / 2.
errD_fake = (errD_fake + uncond_errD_fake) / 2.
else:
errD = errD_real + (errD_fake + errD_wrong) * 0.5
acc = 0
if netD.cate_classify is not None:
cate_logits = nn.parallel.data_parallel(netD.cate_classify, real_features, gpus)
cate_logits = cate_logits.squeeze()
errD = errD + ratio * cate_criterion(cate_logits, real_catelabels)
acc = accuracy_score(real_catelabels.cpu().data.numpy().astype('int32'),
(cate_logits.cpu().data.numpy() > 0.5).astype('int32'))
return errD, errD_real.data, errD_wrong.data, errD_fake.data, acc
if the_number_of_gpu > 1:
log_func('[i] the number of gpu : {}'.format(the_number_of_gpu))
model = nn.DataParallel(model)
# for sync bn
# patch_replication_callback(model)
load_model_fn = lambda: load_model(
model, model_path, parallel=the_number_of_gpu > 1)
save_model_fn = lambda: save_model(
model, model_path, parallel=the_number_of_gpu > 1)
###################################################################################
# Loss, Optimizer
###################################################################################
class_loss_fn = nn.MultiLabelSoftMarginLoss(reduction='none').cuda()
if args.re_loss == 'L1_Loss':
re_loss_fn = L1_Loss
else:
re_loss_fn = L2_Loss
log_func('[i] The number of pretrained weights : {}'.format(
len(param_groups[0])))
log_func('[i] The number of pretrained bias : {}'.format(
len(param_groups[1])))
log_func('[i] The number of scratched weights : {}'.format(
len(param_groups[2])))
log_func('[i] The number of scratched bias : {}'.format(
len(param_groups[3])))
########################################################################################
if __name__ == '__main__':
print('%s: calling main function ... ' % os.path.basename(__file__))
# https://discuss.pytorch.org/t/print-autograd-graph/692/8
batch_size = 64
num_classes = 17
C, H, W = 3, 256, 256
inputs = torch.randn(batch_size, C, H, W)
labels = torch.randn(batch_size, num_classes)
in_shape = inputs.size()[1:]
if 1:
net = densenet121(in_shape=in_shape,
num_classes=num_classes).cuda().train()
x = Variable(inputs)
logits, probs = net.forward(x.cuda())
loss = nn.MultiLabelSoftMarginLoss()(logits, Variable(labels.cuda()))
loss.backward()
print(type(net))
print(net)
print('probs')
print(probs)
input('Press ENTER to continue.')
else:
print("=> no checkpoint found at '{}'".format(args.resume))
if torch.cuda.device_count() > 1:
print "Using %d gpus." % torch.cuda.device_count()
generator = nn.DataParallel(generator)
discriminator = nn.DataParallel(discriminator)
if torch.cuda.is_available():
generator.cuda()
discriminator.cuda()
if args.verbosity >= 1: print 'Done!'
if args.dual_discrim:
class_loss_fn = nn.MultiLabelSoftMarginLoss()
loss_fn = nn.BCELoss()
D_total_fake_loss = 0
D_total_fake_binary_loss = 0
D_total_fake_class_loss = 0
D_total_real_loss = 0
D_total_real_binary_loss = 0
D_total_real_class_loss = 0
G_total_loss = 0
G_total_binary_loss = 0
G_total_class_loss = 0
for epoch in xrange(0):
def main():
global args, best_prec1
torch.manual_seed(157)
torch.cuda.manual_seed(157)
#np.random.seed(0)
args = parser.parse_args()
args.distributed = args.world_size > 1
# create model
print("=> creating model '{}'".format(args.arch))
if args.arch == 'localizer_alexnet':
model = localizer_alexnet(pretrained=args.pretrained)
elif args.arch == 'localizer_alexnet_robust':
model = localizer_alexnet_robust(pretrained=args.pretrained)
args.lr = args.lr
print(model)
# torch.cuda.set_device(0)
# print(torch.cuda.current_device())
model.features = torch.nn.DataParallel(model.features)
model.cuda()
# TODO:
# define loss function (criterion) and optimizer
criterion = nn.MultiLabelSoftMarginLoss().cuda()
# criterion = nn.BCEWithLogitsLoss().cuda()
# optimizer = torch.optim.SGD(model.parameters(), lr=args.lr)
optimizer = torch.optim.SGD(model.classifier.parameters(),
args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
# optionally resume from a checkpoint
if args.resume:
if os.path.isfile(args.resume):
print("=> loading checkpoint '{}'".format(args.resume))
checkpoint = torch.load(args.resume)
args.start_epoch = checkpoint['epoch']
best_prec1 = checkpoint['best_prec1']
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
print("=> loaded checkpoint '{}' (epoch {})".format(
args.resume, checkpoint['epoch']))
else:
print("=> no checkpoint found at '{}'".format(args.resume))
cudnn.benchmark = True
# Data loading code
# TODO: Write code for IMDBDataset in custom.py
# trainval_imdb.num_images = 5011
trainval_imdb = get_imdb('voc_2007_trainval')
test_imdb = get_imdb('voc_2007_test')
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
train_dataset = IMDBDataset(
trainval_imdb,
transforms.Compose([
transforms.Resize((512, 512)),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
normalize,
]))
train_sampler = None
# len(train_loader.dataset.imgs) = 5011
train_loader = torch.utils.data.DataLoader(train_dataset,
batch_size=args.batch_size,
shuffle=False,
num_workers=args.workers,
pin_memory=True,
sampler=train_sampler)
# len(val_loader.dataset.imgs) = 4952, dir(val_loader) = 'class_to_idx', 'classes', 'imdb', 'imgs', 'loader', 'target_transform', 'transform'
val_loader = torch.utils.data.DataLoader(IMDBDataset(
test_imdb,
transforms.Compose([
transforms.Resize((384, 384)),
transforms.ToTensor(),
normalize,
])),
batch_size=args.batch_size,
shuffle=False,
num_workers=args.workers,
pin_memory=True)
# TODO: Create loggers for visdom and tboard
# TODO: You can pass the logger objects to train(), make appropriate
# modifications to train()
if args.arch == 'localizer_alexnet':
data_log = logger.Logger('./logs/task1', name='freeloc')
vis = visdom.Visdom(server='http://localhost', port='8087')
else:
data_log = logger.Logger('./logs_robust/task1', name='freeloc')
vis = visdom.Visdom(server='http://localhost', port='8087')
if args.evaluate:
validate(val_loader, model, criterion, vis, 0)
return
if args.arch == 'localizer_alexnet':
args.epochs = 30
else:
args.epochs = 45
for epoch in range(args.start_epoch, args.epochs):
adjust_learning_rate(optimizer, epoch)
# train for one epoch
train(train_loader, model, criterion, optimizer, epoch, data_log, vis,
args.arch)
# evaluate on validation set
if epoch % 10 == 0 or epoch == args.epochs - 1:
# if True: #temp
print("now evaluating!!!!!!")
m1, m2 = validate(val_loader, model, criterion, vis, epoch)
score = m1 * m2
# remember best prec@1 and save checkpoint
is_best = score > best_prec1
best_prec1 = max(score, best_prec1)
data_log.scalar_summary(tag='valid/metric1', value=m1, step=epoch)
data_log.scalar_summary(tag='valid/metric2', value=m2, step=epoch)
# import pdb;pdb.set_trace()
save_checkpoint(
{
'epoch': epoch + 1,
'arch': args.arch,
'state_dict': model.state_dict(),
'best_prec1': best_prec1,
'optimizer': optimizer.state_dict(),
}, is_best)
print('end of training')
def train():
imgModel = ImgModel()
loss_fn = nn.MultiLabelSoftMarginLoss(
) # MultiLabelSoftMarginLoss # CrossEntropyLoss
if torch.cuda.is_available(): # GPU
imgModel.to(DEVICE)
loss_fn.to(DEVICE)
loss_fn = nn.MultiLabelSoftMarginLoss(
) # MultiLabelSoftMarginLoss # CrossEntropyLoss
optimizer = optim.Adam(params=imgModel.parameters(), )
acces = []
epoch_flag = False
train_img_dir = os.path.join(imgModel.img_dir,
'trainImg_l{}'.format(imgModel.char_len))
train_dataloder = data.DataLoader(dataset=ImgDataset(
img_dir=train_img_dir, char_len=imgModel.char_len),
batch_size=imgModel.batch_size,
shuffle=True)
train_dataloder_dataset_len = len(train_dataloder.dataset)
if train_dataloder_dataset_len < imgModel.batch_size:
# 图片数量少于 批次batch_size
batch_acc_tatol = imgModel.char_len * train_dataloder_dataset_len
else:
# 图片数量 大于 批次batch_size
batch_acc_tatol = imgModel.char_len * imgModel.batch_size
for epoch in range(imgModel.epochs):
if epoch_flag: break
for batch_idx, (img, label) in enumerate(
train_dataloder
): # batch_idx == len(train_dataloder.dataset) /batch_size
if torch.cuda.is_available():
img, label = img.to(DEVICE), label.to(DEVICE)
optimizer.zero_grad()
output = imgModel(img)
# print(output.size()) # torch.Size([batch_size512, 144])
label = label.view_as(output)
loss = loss_fn(output, label)
loss.backward()
optimizer.step()
if batch_idx % 100 == 0: # 每100个批次统计一次
batch_acc = imgModel._acc_count(
output, label, batch_acc_tatol) # 当前批次batch准确度x%
acces.append(batch_acc)
per = sum(acces) / (len(acces) * 100) * 100. # 总准确度x%
if per > imgModel.per:
epoch_flag = True
break
print(
'time:{},第{}次训练,第{}批次:,loss_item:{},该批次内准确度:{}%,总准确度per:{}%'
.format(datetime.now(), epoch, batch_idx, loss.item(),
batch_acc, per))
model_file = 'l{}.pth'.format(imgModel.char_len)
model_path = os.path.join(imgModel.model_dir, model_file)
torch.save(imgModel.state_dict(), model_path)
def train():
x, y = load_data()
model = ResNet()
model = model.cuda()
# Loss and Optimizer
# criterion = nn.CrossEntropyLoss()
criterion = nn.MultiLabelSoftMarginLoss()
optimizer = torch.optim.Adam(model.parameters(), learning_rate)
x = np.asarray(x, dtype=np.int32)
y = np.asarray(y, dtype=np.int32)
x_torch = torch.from_numpy(x)
y_torch = torch.from_numpy(y)
# 先转换成 torch 能识别的 Dataset
torch_dataset = Data.TensorDataset(x_torch, y_torch)
# 把 dataset 放入 DataLoader
train_loader = Data.DataLoader(
dataset=torch_dataset, # torch TensorDataset format
batch_size=batch_size, # mini batch size
shuffle=False, # 要不要打乱数据 (打乱比较好)
num_workers=1, # 多线程来读数据
)
#traing the model
for epoch in range(15):
model.train()
print("epoch {}".format(epoch))
for i, (images, labels) in enumerate(train_loader):
images = Variable(images)
#print(images.shape)
labels = Variable(labels)
# print(images.shape)
# print(labels.shape)
# #forward + backward + optimizer
optimizer.zero_grad()
images = images.float()
labels = labels.long()
if i == break_batch_tra: #进行到最后一个batch_size时, reshape 无法
break # 使用，直接跳过
labels = labels.reshape(batch_size, 1)
labels = torch.zeros(batch_size,
class_num).scatter_(1, labels,
1) ##one_hot 编码
outputs = model(images.cuda())
loss = criterion(outputs, labels.cuda())
loss.backward()
optimizer.step()
if 1:
model.eval()
total = 0
correct = 0
for i, (images, labels) in enumerate(train_loader):
images = Variable(images)
labels = Variable(labels)
images = images.float()
labels = labels.long()
if i == break_batch_tra: #进行到最后一个batch_size时, reshape 无法
break # 使用，直接跳过
labels = labels.reshape(batch_size)
outputs = model(images.cuda())
pred_data, pred = torch.max(outputs.data, 1) ##取出经过网络计算后的标签
test = outputs.data
pred = pred.cpu().numpy()
labels = labels.numpy()
correct += (pred == labels).sum()
total += np.size(labels, 0)
accu = correct / total
print("train correct is {}".format(correct))
print("train total is {}".format(total))
print("train accuracy is {}\n".format(accu))
# torch.save(model, './model/'+str(epoch)+'test-model.pkl')
torch.save(model, './model/test-model.pkl')
print("model has been written")
return
########################################################################################
if __name__ == '__main__':
print('%s: calling main function ... ' % os.path.basename(__file__))
# https://discuss.pytorch.org/t/print-autograd-graph/692/8
batch_size = 1
num_classes = 17
C, H, W = 3, 256, 256
inputs = torch.randn(batch_size, C, H, W)
labels = torch.randn(batch_size, num_classes)
in_shape = inputs.size()[1:]
if 1:
net = PyResNet34(in_shape=in_shape,
num_classes=num_classes).cuda().train()
x = inputs
logits, probs = net.forward(x.cuda())
loss = nn.MultiLabelSoftMarginLoss()(logits, labels.cuda())
loss.backward()
print(type(net))
print(net)
print('probs')
print(probs)
#input('Press ENTER to continue.')
# 输出层 输出数量因设为想要的长度 此处是因为把验证码进行独热编码
self.fc2 = nn.Linear(1024, len(alphabet) * captcha_len)
def forward(self, x):
x = self.conv1(x)
x = self.conv2(x)
x = self.conv3(x)
x = x.view(x.size(0), -1)
x = self.fc1(x)
x = F.relu(x)
out = self.fc2(x)
return out
cnn = CNN()
loss_func = nn.MultiLabelSoftMarginLoss()
optimizer = torch.optim.Adam(cnn.parameters(), lr=0.001)
def train():
# 开始训练
for i in range(5):
for ind, (x, y, order) in enumerate(train_load):
x, y = Variable(x), Variable(y)
out_x = cnn(x)
loss = loss_func(out_x, y)
optimizer.zero_grad()
loss.backward()
optimizer.step()
if ind % 50 == 0:
print(f"Epoch: {i} | train los: {loss.data.numpy()}")
