def main():
source_train_loader = mnist.mnist_train_loader
target_train_loader = mnistm.mnistm_train_loader
if torch.cuda.is_available():
get_free_gpu()
print('Running GPU : {}'.format(torch.cuda.current_device()))
encoder = model.Extractor().cuda()
classifier = model.Classifier().cuda()
discriminator = model.Discriminator().cuda()
train.source_only(encoder, classifier, source_train_loader,
target_train_loader, save_name)
train.dann(encoder, classifier, discriminator, source_train_loader,
target_train_loader, save_name)
else:
print("There is no GPU -_-!")
def main():
print('Loading word embedding')
emb = KeyedVectors.load_word2vec_format(hp.word_embedding,
binary=hp.emb_binary)
print("Loading data")
stereotype_words = {}
gender_words = {}
no_gender_words = make_no_gender_words(open(hp.no_gender_words), emb)
stereotype_words['female'], stereotype_words['male'] = \
make_pair_words(hp.stereotype_words, emb)
gender_words['female'], gender_words['male'] = \
make_pair_words(hp.gender_words, emb)
all_words = no_gender_words \
+ stereotype_words['female'] \
+ stereotype_words['male'] \
+ gender_words['female'] \
+ gender_words['male']
train_words, dev_words = create_train_dev(gender_words, no_gender_words,
stereotype_words)
word2emb = {}
for word in all_words:
word2emb[word] = emb[word]
if hp.pre_train_autoencoder:
print('Pre-training autoencoder')
encoder = model.Encoder(hp.emb_size, hp.hidden_size,
hp.pta_dropout_rate)
decoder = model.Decoder(hp.hidden_size, hp.emb_size,
hp.pta_dropout_rate)
if hp.gpu >= 0:
encoder.cuda()
decoder.cuda()
encoder_optim = make_optim(encoder, hp.pta_optimizer,
hp.pta_learning_rate, hp.pta_lr_decay,
hp.pta_max_grad_norm)
decoder_optim = make_optim(decoder, hp.pta_optimizer,
hp.pta_learning_rate, hp.pta_lr_decay,
hp.pta_max_grad_norm)
if hp.pre_data == 'random':
checkpoint = pre_train_autoencoder(hp, encoder, encoder_optim,
decoder, decoder_optim, emb)
elif hp.pre_data == 'common':
checkpoint = pre_train_autoencoder(hp,
encoder,
encoder_optim,
decoder,
decoder_optim,
emb,
dev_words=dev_words)
encoder = model.Encoder(hp.emb_size, hp.hidden_size, hp.dropout_rate)
decoder = model.Decoder(hp.hidden_size, hp.emb_size, hp.dropout_rate)
if hp.gpu >= 0:
encoder.cuda()
decoder.cuda()
if hp.pre_train_autoencoder:
encoder.load_state_dict(checkpoint['encoder'])
decoder.load_state_dict(checkpoint['decoder'])
if hp.pre_train_classifier:
female_classifier = model.Classifier(hp.hidden_size)
male_classifier = model.Classifier(hp.hidden_size)
if hp.gpu >= 0:
female_classifier.cuda()
male_classifier.cuda()
female_classifier_optim = make_optim(female_classifier,
hp.cls_optimizer,
hp.cls_learning_rate,
hp.cls_lr_decay,
hp.cls_max_grad_norm)
male_classifier_optim = make_optim(male_classifier, hp.cls_optimizer,
hp.cls_learning_rate,
hp.cls_lr_decay,
hp.cls_max_grad_norm)
encoder.eval()
encoder.zero_grad()
train_females = []
train_males = []
dev_females = []
dev_males = []
train_female_embs = [
encoder(torch.FloatTensor(emb[word[0]]).cuda()).data
if hp.gpu >= 0 else encoder(torch.FloatTensor(emb[word[0]])).data
for word in train_words['female & male']
]
encoder.zero_grad()
train_male_embs = [
encoder(torch.FloatTensor(emb[word[1]]).cuda()).data
if hp.gpu >= 0 else encoder(torch.FloatTensor(emb[word[1]])).data
for word in train_words['female & male']
]
encoder.zero_grad()
train_stereotype_embs = [
encoder(torch.FloatTensor(emb[word]).cuda()).data
if hp.gpu >= 0 else encoder(torch.FloatTensor(emb[word])).data
for word in train_words['no gender']
]
encoder.zero_grad()
dev_female_embs = [
encoder(torch.FloatTensor(emb[word[0]]).cuda()).data
if hp.gpu >= 0 else encoder(torch.FloatTensor(emb[word[0]])).data
for word in dev_words['female & male']
]
encoder.zero_grad()
dev_male_embs = [
encoder(torch.FloatTensor(emb[word[1]]).cuda()).data
if hp.gpu >= 0 else encoder(torch.FloatTensor(emb[word[1]])).data
for word in dev_words['female & male']
]
encoder.zero_grad()
dev_stereotype_embs = [
encoder(torch.FloatTensor(emb[word]).cuda()).data
if hp.gpu >= 0 else encoder(torch.FloatTensor(emb[word])).data
for word in dev_words['no gender']
]
encoder.zero_grad()
print('Pre-training classifier')
female_checkpoint, male_checkpoint = pre_train_classifier(
hp, female_classifier, male_classifier, female_classifier_optim,
male_classifier_optim, train_female_embs, train_male_embs,
train_stereotype_embs, dev_female_embs, dev_male_embs,
dev_stereotype_embs)
print('Building female & male classifiers')
female_classifier = model.Classifier(hp.hidden_size)
male_classifier = model.Classifier(hp.hidden_size)
if hp.gpu >= 0:
female_classifier.cuda()
male_classifier.cuda()
if hp.pre_train_classifier:
female_classifier.load_state_dict(female_checkpoint['female'])
male_classifier.load_state_dict(male_checkpoint['male'])
print('Setting optimizer')
encoder_optim = make_optim(encoder, hp.optimizer, hp.learning_rate,
hp.lr_decay, hp.max_grad_norm)
female_classifier_optim = make_optim(female_classifier, hp.optimizer,
hp.learning_rate, hp.lr_decay,
hp.max_grad_norm)
male_classifier_optim = make_optim(male_classifier, hp.optimizer,
hp.learning_rate, hp.lr_decay,
hp.max_grad_norm)
decoder_optim = make_optim(decoder, hp.optimizer, hp.learning_rate,
hp.lr_decay, hp.max_grad_norm)
trainModel(encoder, encoder_optim, female_classifier,
female_classifier_optim, male_classifier, male_classifier_optim,
decoder, decoder_optim, train_words, dev_words, word2emb)
def cmd_train(context):
"""Main command do train the network.
:param context: this is a dictionary with all data from the
configuration file:
- 'command': run the specified command (e.g. train, test)
- 'gpu': ID of the used GPU
- 'bids_path_train': list of relative paths of the BIDS folders of each training center
- 'bids_path_validation': list of relative paths of the BIDS folders of each validation center
- 'bids_path_test': list of relative paths of the BIDS folders of each test center
- 'batch_size'
- 'dropout_rate'
- 'batch_norm_momentum'
- 'num_epochs'
- 'initial_lr': initial learning rate
- 'log_directory': folder name where log files are saved
"""
# Set the GPU
gpu_number = context["gpu"]
torch.cuda.set_device(gpu_number)
# These are the training transformations
train_transform = transforms.Compose([
mt_transforms.CenterCrop2D((128, 128)),
mt_transforms.ElasticTransform(alpha_range=(28.0, 30.0),
sigma_range=(3.5, 4.0),
p=0.3),
mt_transforms.RandomAffine(degrees=4.6,
scale=(0.98, 1.02),
translate=(0.03, 0.03)),
mt_transforms.RandomTensorChannelShift((-0.10, 0.10)),
mt_transforms.ToTensor(),
mt_transforms.NormalizeInstance(),
])
# These are the validation/testing transformations
val_transform = transforms.Compose([
mt_transforms.CenterCrop2D((128, 128)),
mt_transforms.ToTensor(),
mt_transforms.NormalizeInstance(),
])
# This code will iterate over the folders and load the data, filtering
# the slices without labels and then concatenating all the datasets together
train_datasets = []
for bids_ds in tqdm(context["bids_path_train"],
desc="Loading training set"):
ds_train = loader.BidsDataset(bids_ds,
transform=train_transform,
slice_filter_fn=loader.SliceFilter())
train_datasets.append(ds_train)
ds_train = ConcatDataset(train_datasets)
print(f"Loaded {len(ds_train)} axial slices for the training set.")
train_loader = DataLoader(ds_train,
batch_size=context["batch_size"],
shuffle=True,
pin_memory=True,
collate_fn=mt_datasets.mt_collate,
num_workers=1)
# Validation dataset ------------------------------------------------------
validation_datasets = []
for bids_ds in tqdm(context["bids_path_validation"],
desc="Loading validation set"):
ds_val = loader.BidsDataset(bids_ds,
transform=val_transform,
slice_filter_fn=loader.SliceFilter())
validation_datasets.append(ds_val)
ds_val = ConcatDataset(validation_datasets)
print(f"Loaded {len(ds_val)} axial slices for the validation set.")
val_loader = DataLoader(ds_val,
batch_size=context["batch_size"],
shuffle=True,
pin_memory=True,
collate_fn=mt_datasets.mt_collate,
num_workers=1)
model = M.Classifier(drop_rate=context["dropout_rate"],
bn_momentum=context["batch_norm_momentum"])
model.cuda()
num_epochs = context["num_epochs"]
initial_lr = context["initial_lr"]
# Using SGD with cosine annealing learning rate
optimizer = optim.SGD(model.parameters(), lr=initial_lr)
scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, num_epochs)
# Write the metrics, images, etc to TensorBoard format
writer = SummaryWriter(log_dir=context["log_directory"])
# Cross Entropy Loss
criterion = nn.CrossEntropyLoss()
# Training loop -----------------------------------------------------------
best_validation_loss = float("inf")
lst_train_loss = []
lst_val_loss = []
lst_accuracy = []
for epoch in tqdm(range(1, num_epochs + 1), desc="Training"):
start_time = time.time()
scheduler.step()
lr = scheduler.get_lr()[0]
writer.add_scalar('learning_rate', lr, epoch)
model.train()
train_loss_total = 0.0
num_steps = 0
for i, batch in enumerate(train_loader):
input_samples = batch["input"]
input_labels = get_modality(batch)
var_input = input_samples.cuda()
var_labels = torch.cuda.LongTensor(input_labels).cuda(
non_blocking=True)
outputs = model(var_input)
loss = criterion(outputs, var_labels)
train_loss_total += loss.item()
optimizer.zero_grad()
loss.backward()
optimizer.step()
num_steps += 1
train_loss_total_avg = train_loss_total / num_steps
lst_train_loss.append(train_loss_total_avg)
tqdm.write(f"Epoch {epoch} training loss: {train_loss_total_avg:.4f}.")
# Validation loop -----------------------------------------------------
model.eval()
val_loss_total = 0.0
num_steps = 0
val_accuracy = 0
num_samples = 0
for i, batch in enumerate(val_loader):
input_samples = batch["input"]
input_labels = get_modality(batch)
with torch.no_grad():
var_input = input_samples.cuda()
var_labels = torch.cuda.LongTensor(input_labels).cuda(
non_blocking=True)
outputs = model(var_input)
_, preds = torch.max(outputs, 1)
loss = criterion(outputs, var_labels)
val_loss_total += loss.item()
val_accuracy += int((var_labels == preds).sum())
num_steps += 1
num_samples += context['batch_size']
val_loss_total_avg = val_loss_total / num_steps
lst_val_loss.append(val_loss_total_avg)
tqdm.write(f"Epoch {epoch} validation loss: {val_loss_total_avg:.4f}.")
val_accuracy_avg = 100 * val_accuracy / num_samples
lst_accuracy.append(val_accuracy_avg)
tqdm.write(f"Epoch {epoch} accuracy : {val_accuracy_avg:.4f}.")
# add metrics for tensorboard
writer.add_scalars('validation metrics', {
'accuracy': val_accuracy_avg,
}, epoch)
writer.add_scalars('losses', {
'train_loss': train_loss_total_avg,
'val_loss': val_loss_total_avg,
}, epoch)
end_time = time.time()
total_time = end_time - start_time
tqdm.write("Epoch {} took {:.2f} seconds.".format(epoch, total_time))
if val_loss_total_avg < best_validation_loss:
best_validation_loss = val_loss_total_avg
torch.save(model,
"./" + context["log_directory"] + "/best_model.pt")
# save final model
torch.save(model, "./" + context["log_directory"] + "/final_model.pt")
# save the metrics
parameters = "CrossEntropyLoss/batchsize=" + str(context['batch_size'])
parameters += "/initial_lr=" + str(context['initial_lr'])
parameters += "/dropout=" + str(context['dropout_rate'])
plt.subplot(2, 1, 1)
plt.title(parameters)
plt.plot(lst_train_loss, color='red', label='Training')
plt.plot(lst_val_loss, color='blue', label='Validation')
plt.legend(loc='upper right')
plt.ylabel('Loss')
plt.subplot(2, 1, 2)
plt.plot(lst_accuracy)
plt.ylabel('Accuracy')
plt.xlabel('Epoch')
plt.savefig(parameters + '.png')
return
plot_loss_total = 0
return plot_losses
if __name__ == '__main__':
#device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
device = torch.device("cpu")
print('Loading data')
print(device)
print('Ignoring:', sys.argv[2:])
entries = data.load(sys.argv[1], exclude=sys.argv[2:])
n_iters = 30000
dataset = data.balanced(entries, n_iters)
#dataset=entries
print('Training')
embedded_size = 64
hidden_size = 128
encoder1 = model.Encoder(hidden_size, 3).to(device)
classifier1 = model.Classifier(hidden_size, 2).to(device)
train(dataset, encoder1, classifier1, device=device, print_every=2000)
print('Caching trained model')
torch.save(encoder1, 'encoder.pt')
torch.save(classifier1, 'classifier.pt')
# main.py # The main program; primary interface to the classification model import data import model (train_data, train_labels, test_data, test_labels) = data.import_data() classifier = model.Classifier() classifier.train(train_data, train_labels) classifier.evaluate(test_data, test_labels) classifier.cv(test_data, test_labels)
print('===> prepare dataloader ...')
train_loader = torch.utils.data.DataLoader(data.TrimmedVideoData(
args, mode='train', model_type='cnn'),
batch_size=args.train_batch,
num_workers=args.workers,
shuffle=True)
# collate_fn=data.collate_fn)
val_loader = torch.utils.data.DataLoader(data.TrimmedVideoData(
args, mode='val', model_type='cnn'),
batch_size=args.train_batch,
num_workers=args.workers,
shuffle=True)
''' load model '''
print('===> prepare model ...')
feature_extractor, classifier = model.Resnet50(), model.Classifier()
feature_extractor.cuda()
classifier.cuda()
clf_criterion = nn.CrossEntropyLoss()
clf_optimizer = torch.optim.Adam(classifier.parameters(),
lr=args.lr,
betas=(args.beta1, args.beta2))
''' setup tensorboard '''
writer = SummaryWriter(os.path.join(args.save_dir, 'train_info'))
print('===> start training ...')
iters = 0
best_acc = 0
def predict(input):
config = {'hidden_size': 512} # same config
model = m.Classifier(cnn.ConvolutionalModel,
cnn.Config(name_suffix='THE_BEAST', **config))
pred = model.predict(input)
return pred
'''
TODO: decoder mlp增加，encoder可以先用训练好的参数
'''
EPOCH = 11
BATCH_SIZE = 200
LATENT_CODE_g = 30 #40 general
LATENT_CODE_d = 20
log_interval = 10
last_checkpoint = 0
droupout_rate = 0
vae_encoder = model.VAE_encoder(droupout_rate, LATENT_CODE_g,
LATENT_CODE_d).cuda()
vae_decoder = model.VAE_decoder(LATENT_CODE_g, LATENT_CODE_d).cuda()
classifier = model.Classifier(LATENT_CODE_d).cuda()
'''训练'''
'''读取原来的训练文件'''
if last_checkpoint != 0:
vae_encoder.load_state_dict(
torch.load('save\\vae_encoder' + str(last_checkpoint) + '.pkl'))
vae_decoder.load_state_dict(
torch.load('save\\vae_decoder' + str(last_checkpoint) + '.pkl'))
classifier.load_state_dict(
torch.load('save\\classifier' + str(last_checkpoint) + '.pkl'))
z_prior_mu = np.load('save\\z_prior_mu' + str(last_checkpoint) + '.npy')
z_prior_logvar = np.load('save\\z_prior_logvar' + str(last_checkpoint) +
'.npy')
kkt_param = np.load('save\\kkt_param' + str(last_checkpoint) + '.npy')
vae_encoder.prior_mu = Variable(torch.tensor(z_prior_mu).type(
test_loader = DataLoader(test_data,
batch_size=config.batch,
shuffle=False,
num_workers=config.cpu_processor,
drop_last=True)
dev_loader = DataLoader(dev_data,
batch_size=config.batch,
shuffle=False,
num_workers=config.cpu_processor,
drop_last=True)
# 모델 설정
device = torch.device(config.gpu if torch.cuda.is_available() else 'cpu')
model = model.Classifier(dataset.vocab_list)
model.to(device)
optimizer = torch.optim.Adam(model.parameters(), lr=config.learning_rate)
loss_function = nn.CrossEntropyLoss()
print("--model set--")
# 훈련
step_list = []
loss_list = []
acc_test_list = []
acc_dev_list = []
step = 0
for i in range(config.epoch):
print("epoch = ", i)
for n, (label, sent1, sent2) in enumerate(train_loader):