def main():
import opts
import misc.utils as utils
opt = opts.parse_opt()
opt.caption_model ='topdown'
opt.batch_size=10
opt.id ='topdown'
opt.learning_rate= 5e-4
opt.learning_rate_decay_start= 0
opt.scheduled_sampling_start=0
opt.save_checkpoint_every=25#11500
opt.val_images_use=5000
opt.max_epochs=40
opt.start_from=None
opt.input_json='data/meta_coco_en.json'
opt.input_label_h5='data/label_coco_en.h5'
opt.input_image_h5 = 'data/coco_image_512.h5'
opt.use_att = utils.if_use_att(opt.caption_model)
opt.ccg = False
loader = DataLoader(opt)
opt.vocab_size = loader.vocab_size
opt.seq_length = loader.seq_length
data = loader.get_batch('train')
data = loader.get_batch('val')
def main():
opt = opts.parse_opt()
print(opt)
assert (opt.dataset == "imdb")
train_iter, dev_iter, test_iter, syn_data = imdb_make_synthesized_iter(opt)
train(opt, train_iter, dev_iter, test_iter, syn_data)
def main():
global total
global correct
opt = opts.parse_opt()
# load vocab
# get model
# opt.load_epoch = 3
bart = BART(opt)
bart.load_model(f'models/3_model.pt')
# generate(opt, model, SRC, TRG, opt.beam_size)
generate(opt, bart)
print(correct / total)
def main():
opt = opts.parse_opt()
logging.basicConfig(format='%(asctime)s %(message)s', level=logging.INFO)
tb_logger.configure(opt.logger_name, flush_secs=5)
# Load Vocabulary Wrapper
vocab = deserialize_vocab(
os.path.join(opt.vocab_path, '%s_vocab.json' % opt.data_name))
opt.vocab_size = len(vocab)
# Load data loaders
train_loader, val_loader = data.get_loaders(opt.data_name, vocab,
opt.batch_size, opt.workers,
opt)
# Construct the model
model = SGRAF(opt)
# Train the Model
best_rsum = 0
for epoch in range(opt.num_epochs):
print(opt.logger_name)
print(opt.model_name)
adjust_learning_rate(opt, model.optimizer, epoch)
# train for one epoch
train(opt, train_loader, model, epoch, val_loader)
# evaluate on validation set
r_sum = validate(opt, val_loader, model)
# remember best R@ sum and save checkpoint
is_best = r_sum > best_rsum
best_rsum = max(r_sum, best_rsum)
if not os.path.exists(opt.model_name):
os.mkdir(opt.model_name)
save_checkpoint(
{
'epoch': epoch + 1,
'model': model.state_dict(),
'best_rsum': best_rsum,
'opt': opt,
'Eiters': model.Eiters,
},
is_best,
filename='checkpoint_{}.pth.tar'.format(epoch),
prefix=opt.model_name + '/')
def main():
opt = opts.parse_opt()
opt.src_data = '../data/visualstorytelling/train.pkl'
opt.test_src_data = '../data/visualstorytelling/test.pkl'
dataset = get_loader(opt.src_data, opt.batch_size, train=True)
test_dataset = get_loader(opt.test_src_data, opt.batch_size, train=False, shuffle=False)
# get model
bart = BART(opt)
train_model(opt, bart, dataset, test_dataset, 10)
def get_en_opts():
opt = opts.parse_opt()
opt.caption_model = 'cross_topdown'
opt.batch_size = 10
#Pretrain
opt.id = 'topdown'
opt.learning_rate = 5e-4
opt.learning_rate_decay_start = 0
opt.scheduled_sampling_start = 0
opt.save_checkpoint_every = 1300 #11500
opt.val_images_use = 5000
opt.max_epochs = 40
opt.start_from = None
opt.input_json = '/home/andyweizhao/wabywang/010/data/dataset/coco_processed.json'
opt.input_label_h5 = 'data/dataset/coco_label.h5'
opt.input_fc_dir = '/media/andyweizhao/Elements/CVPR/cocotalk_fc'
opt.input_att_dir = 'data/cocotalk_att'
return opt
def main():
opt = opts.parse_opt()
opt.caption_model = 'topdown'
opt.batch_size = 10
opt.id = 'topdown'
opt.learning_rate = 5e-5
opt.learning_rate_decay_start = -1
opt.scheduled_sampling_start = -1
opt.save_checkpoint_every = 5000 #
opt.val_images_use = 5000
opt.max_epochs = 60
opt.start_from = 'save/multitask_pretrain' #"save" #None
opt.language_eval = 1
opt.input_json = 'data/meta_coco_en.json'
opt.input_label_h5 = 'data/label_coco_en.h5'
opt.self_critical_after = 25
opt.finetune_cnn_after = 0
opt.ccg = False
opt.input_image_h5 = 'data/coco_image_512.h5'
opt.checkpoint_path = 'save/multitask_pretrain_rl'
train(opt)
import datetime
import json
from dataset.dataset import COCO_Search18, COCO_Search18_evaluation, COCO_Search18_rl
from models.baseline_attention_multihead import baseline
from models.loss import CrossEntropyLoss, DurationSmoothL1Loss, MLPRayleighDistribution, MLPLogNormalDistribution, \
LogAction, LogDuration, NSS, CC, KLD, CC_MatchLoss, CC_terms
from utils.checkpointing import CheckpointManager
from utils.recording import RecordManager
from utils.evaluation import human_evaluation, evaluation, pairs_eval_scanmatch
from utils.logger import Logger
from opts import parse_opt
from utils.evaltools.scanmatch import ScanMatch
from models.sampling import Sampling
args = parse_opt()
# For reproducibility - refer https://pytorch.org/docs/stable/notes/randomness.html
# These five lines control all the major sources of randomness.
np.random.seed(args.seed)
torch.manual_seed(args.seed)
torch.cuda.manual_seed_all(args.seed)
torch.backends.cudnn.benchmark = False
torch.backends.cudnn.deterministic = True
transform = transforms.Compose([
transforms.Resize((args.height, args.width)),
transforms.ToTensor(),
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
])
# coding:utf8
import torch
import skimage.io
from opts import parse_opt
from models.decoder import Decoder
from models.encoder import Encoder
opt = parse_opt()
assert opt.test_model, 'please input test_model'
assert opt.image_file, 'please input image_file'
encoder = Encoder(opt.resnet101_file)
encoder.to(opt.device)
encoder.eval()
img = skimage.io.imread(opt.image_file)
with torch.no_grad():
img = encoder.preprocess(img)
img = img.to(opt.device)
fc_feat, att_feat = encoder(img)
print("====> loading checkpoint '{}'".format(opt.test_model))
chkpoint = torch.load(opt.test_model, map_location=lambda s, l: s)
decoder = Decoder(chkpoint['idx2word'], chkpoint['settings'])
decoder.load_state_dict(chkpoint['model'])
print("====> loaded checkpoint '{}', epoch: {}, train_mode: {}".format(
opt.test_model, chkpoint['epoch'], chkpoint['train_mode']))
decoder.to(opt.device)
decoder.eval()
def main():
# Hyper Parameters
opt = opts.parse_opt()
device_id = opt.gpuid
device_count = len(str(device_id).split(","))
#assert device_count == 1 or device_count == 2
print("use GPU:", device_id, "GPUs_count", device_count, flush=True)
os.environ['CUDA_VISIBLE_DEVICES']=str(device_id)
device_id = 0
torch.cuda.set_device(0)
# Load Vocabulary Wrapper
vocab = deserialize_vocab(os.path.join(opt.vocab_path, '%s_vocab.json' % opt.data_name))
opt.vocab_size = len(vocab)
# Load data loaders
train_loader, val_loader = data.get_loaders(
opt.data_name, vocab, opt.batch_size, opt.workers, opt)
# Construct the model
model = SCAN(opt)
model.cuda()
model = nn.DataParallel(model)
# Loss and Optimizer
criterion = ContrastiveLoss(opt=opt, margin=opt.margin, max_violation=opt.max_violation)
mse_criterion = nn.MSELoss(reduction="batchmean")
optimizer = torch.optim.Adam(model.parameters(), lr=opt.learning_rate)
# optionally resume from a checkpoint
if not os.path.exists(opt.model_name):
os.makedirs(opt.model_name)
start_epoch = 0
best_rsum = 0
if opt.resume:
if os.path.isfile(opt.resume):
print("=> loading checkpoint '{}'".format(opt.resume))
checkpoint = torch.load(opt.resume)
start_epoch = checkpoint['epoch']
best_rsum = checkpoint['best_rsum']
model.load_state_dict(checkpoint['model'])
print("=> loaded checkpoint '{}' (epoch {}, best_rsum {})"
.format(opt.resume, start_epoch, best_rsum))
else:
print("=> no checkpoint found at '{}'".format(opt.resume))
evalrank(model.module, val_loader, opt)
print(opt, flush=True)
# Train the Model
for epoch in range(start_epoch, opt.num_epochs):
message = "epoch: %d, model name: %s\n" % (epoch, opt.model_name)
log_file = os.path.join(opt.logger_name, "performance.log")
logging_func(log_file, message)
print("model name: ", opt.model_name, flush=True)
adjust_learning_rate(opt, optimizer, epoch)
run_time = 0
for i, (images, captions, lengths, masks, ids, _) in enumerate(train_loader):
start_time = time.time()
model.train()
optimizer.zero_grad()
if device_count != 1:
images = images.repeat(device_count,1,1)
score = model(images, captions, lengths, masks, ids)
loss = criterion(score)
loss.backward()
if opt.grad_clip > 0:
clip_grad_norm_(model.parameters(), opt.grad_clip)
optimizer.step()
run_time += time.time() - start_time
# validate at every val_step
if i % 100 == 0:
log = "epoch: %d; batch: %d/%d; loss: %.4f; time: %.4f" % (epoch,
i, len(train_loader), loss.data.item(), run_time / 100)
print(log, flush=True)
run_time = 0
if (i + 1) % opt.val_step == 0:
evalrank(model.module, val_loader, opt)
print("-------- performance at epoch: %d --------" % (epoch))
# evaluate on validation set
rsum = evalrank(model.module, val_loader, opt)
#rsum = -100
filename = 'model_' + str(epoch) + '.pth.tar'
# remember best R@ sum and save checkpoint
is_best = rsum > best_rsum
best_rsum = max(rsum, best_rsum)
save_checkpoint({
'epoch': epoch + 1,
'model': model.state_dict(),
'best_rsum': best_rsum,
'opt': opt,
}, is_best, filename=filename, prefix=opt.model_name + '/')
cPickle.dump(infos, f)
with open(
os.path.join(
opt['source'].checkpoint_path, 'histories_' +
opt['source'].id + str(iteration) + '.pkl'),
'wb') as f:
cPickle.dump(histories, f)
if best_flag:
checkpoint_path = os.path.join(
opt['source'].checkpoint_path, 'model-best.pth')
torch.save(model.state_dict(), checkpoint_path)
print("model saved to {}".format(checkpoint_path))
with open(
os.path.join(
opt['source'].checkpoint_path,
'infos_' + opt['source'].id + '-best.pkl'),
'wb') as f:
cPickle.dump(infos, f)
# Stop if reaching max epochs
if epoch >= opt['source'].max_epochs and opt['source'].max_epochs != -1:
break
opt = {}
opt['source'] = opts.parse_opt()
opt['target'] = opts.parse_opt('target')
train(opt)
def main():
mnist = input_data.read_data_sets('MNIST_data', one_hot=True)
train_data = mnist.train.images * 2.0 - 1.0
train_label = mnist.train.labels
test_data = mnist.test.images * 2.0 - 1.0
test_label = mnist.test.labels
x_dim = train_data.shape[1]
y_dim = train_label.shape[1]
opt = opts.parse_opt()
batch_size = opt.batch_size
# Changing the options here.
opt.input_data = "MNIST"
opt.input_c_dim = 1
opt.output_c_dim = 1
opt.input_dim = x_dim
opt.label_dim = y_dim
# Running arguments
opt.c = 1.
opt.ld = 500.
opt.H_lambda = 10.
opt.cgan_flag = True
opt.patch_flag = True
opt.G_lambda = 10.
opt.s_l = 0
opt.t_l = 1
# batch_size = opt.batch_size
# Runnign a session, to load the saved model.
with tf.Session() as sess:
model_store = opt.model_restore
print 'MNIST model is stored at %s' % model_store
whitebox_model = MNISTModel(model_store)
#initial ADVGAN
model = advGAN(whitebox_model, model_store, opt, sess)
best_model_path = './GAN/save/best.ckpt'
print 'advGAN is stored at %s' % best_model_path
model.load(best_model_path)
# tvars = tf.trainable_variables()
# tvars_vals = sess.run(tvars)
# for var, val in zip(tvars, tvars_vals):
# if 'generator' not in var.name:
# continue
# print(var.name, val.shape) # Prints the name of the variable alongside its value.
# We have to load a batch of images, then create the fake ones.
# They should be identical.
num_images = 10
images = train_data[:num_images]
fake_images = sess.run([model.fake_images_sample],
{model.source: images})
plt.imshow(np.reshape(fake_images[0], [28, 28]))
plt.show()
infos['iter'] = iteration
infos['epoch'] = epoch
infos['iterators'] = loader.iterators
infos['split_ix'] = loader.split_ix
infos['best_val_score'] = best_val_score
infos['opt'] = opt
infos['vocab'] = loader.get_vocab()
histories['val_result_history'] = val_result_history
histories['loss_history'] = loss_history
histories['lr_history'] = lr_history
histories['ss_prob_history'] = ss_prob_history
with open(os.path.join(opt.checkpoint_path, 'infos_'+opt.id+'.pkl'), 'wb') as f:
cPickle.dump(infos, f)
with open(os.path.join(opt.checkpoint_path, 'histories_'+opt.id+'.pkl'), 'wb') as f:
cPickle.dump(histories, f)
if best_flag:
checkpoint_path = os.path.join(opt.checkpoint_path, 'model-best.pth')
torch.save(model.state_dict(), checkpoint_path)
print("model saved to {}".format(checkpoint_path))
with open(os.path.join(opt.checkpoint_path, 'infos_'+opt.id+'-best.pkl'), 'wb') as f:
cPickle.dump(infos, f)
# Stop if reaching max epochs
if epoch >= opt.max_epochs and opt.max_epochs != -1:
break
opt = opts.parse_opt()
train(opt)
def test_point_wise():
train, dev, test = load_wiki(FLAGS.data, filter=FLAGS.clean) #wiki
# train, test, dev = load(FLAGS.data, filter=FLAGS.clean) #trec
q_max_sent_length = max(
map(lambda x: len(x), train['question'].str.split()))
a_max_sent_length = max(map(lambda x: len(x), train['answer'].str.split()))
print(q_max_sent_length)
print(a_max_sent_length)
print(len(train))
print('train question unique:{}'.format(len(train['question'].unique())))
print('train length', len(train))
print('test length', len(test))
print('dev length', len(dev))
alphabet, embeddings, embeddings_complex = prepare(
[train, test, dev],
max_sent_length=a_max_sent_length,
dim=FLAGS.embedding_dim,
is_embedding_needed=True,
fresh=True)
print(embeddings_complex)
print('alphabet:', len(alphabet))
opt = opts.parse_opt(q_max_sent_length, a_max_sent_length, alphabet,
embeddings, embeddings_complex)
with tf.Graph().as_default():
with tf.device("/gpu:0"):
session_conf = tf.ConfigProto()
session_conf.allow_soft_placement = FLAGS.allow_soft_placement
session_conf.log_device_placement = FLAGS.log_device_placement
session_conf.gpu_options.allow_growth = True
sess = tf.Session(config=session_conf)
with sess.as_default(), open(precision, "w") as log:
s = 'embedding_dim: ' + str(
FLAGS.embedding_dim) + '\n' + 'dropout_keep_prob: ' + str(
FLAGS.dropout_keep_prob) + '\n' + 'l2_reg_lambda: ' + str(
FLAGS.l2_reg_lambda) + '\n' + 'learning_rate: ' + str(
FLAGS.learning_rate
) + '\n' + 'batch_size: ' + str(
FLAGS.batch_size) + '\n' 'trainable: ' + str(
FLAGS.trainable
) + '\n' + 'num_epochs: ' + str(
FLAGS.num_epochs) + '\n' 'data: ' + str(
FLAGS.data) + '\n'
log.write(str(s) + '\n')
# train,test,dev = load("trec",filter=True)
# alphabet,embeddings = prepare([train,test,dev],is_embedding_needed = True)
# cnn = model(opt
# max_input_left=q_max_sent_length,
# max_input_right=a_max_sent_length,
# vocab_size=len(alphabet),
# embedding_size=FLAGS.embedding_dim,
# batch_size=FLAGS.batch_size,
# embeddings=embeddings,
# embeddings_complex=embeddings_complex,
# dropout_keep_prob=FLAGS.dropout_keep_prob,
# filter_sizes=list(map(int, FLAGS.filter_sizes.split(","))),
# num_filters=FLAGS.num_filters,
# l2_reg_lambda=FLAGS.l2_reg_lambda,
# is_Embedding_Needed=True,
# trainable=FLAGS.trainable,
# overlap_needed=FLAGS.overlap_needed,
# position_needed=FLAGS.position_needed,
# pooling=FLAGS.pooling,
# hidden_num=FLAGS.hidden_num,
# extend_feature_dim=FLAGS.extend_feature_dim)
cnn = setup(opt)
cnn.build_graph()
# Define Training procedure
global_step = tf.Variable(0, name="global_step", trainable=False)
starter_learning_rate = FLAGS.learning_rate
learning_rate = tf.train.exponential_decay(starter_learning_rate,
global_step, 100, 0.96)
optimizer = tf.train.AdamOptimizer(FLAGS.learning_rate)
# optimizer = tf.train.GradientDescentOptimizer(learning_rate)
grads_and_vars = optimizer.compute_gradients(cnn.loss)
train_op = optimizer.apply_gradients(grads_and_vars,
global_step=global_step)
saver = tf.train.Saver(tf.global_variables(), max_to_keep=20)
sess.run(tf.global_variables_initializer())
map_max = 0.65
now = int(time.time())
timeArray = time.localtime(now)
timeStamp = time.strftime("%Y%m%d%H%M%S", timeArray)
timeDay = time.strftime("%Y%m%d", timeArray)
print(timeStamp)
for i in range(FLAGS.num_epochs):
d = get_overlap_dict(train,
alphabet,
q_len=q_max_sent_length,
a_len=a_max_sent_length)
datas = batch_gen_with_point_wise(train,
alphabet,
FLAGS.batch_size,
q_len=q_max_sent_length,
a_len=a_max_sent_length)
for data in datas:
if opt.model == 'QA_quantum' or opt.model == 'CNNQLM_I' or opt.model == 'CNNQLM_I_Flat' or opt.model == 'CNNQLM_Vocab' or opt.model == 'CNNQLM_Dim':
feed_dict = {
cnn.question: data[0],
cnn.answer: data[1],
cnn.input_y: data[2],
cnn.q_position: data[3],
cnn.a_position: data[4],
cnn.overlap: data[5],
cnn.q_overlap: data[6],
cnn.a_overlap: data[7]
}
_, step, loss, accuracy, pred, scores, input_y = sess.run(
[
train_op, global_step, cnn.loss, cnn.accuracy,
cnn.predictions, cnn.scores, cnn.input_y
], feed_dict)
else:
feed_dict = {
cnn.question: data[0],
cnn.answer: data[1],
cnn.input_y: data[2],
cnn.q_overlap: data[3],
cnn.a_overlap: data[4],
cnn.q_position: data[5],
cnn.a_position: data[6]
}
_, step, loss, accuracy, pred, scores = sess.run([
train_op, global_step, cnn.loss, cnn.accuracy,
cnn.predictions, cnn.scores
], feed_dict)
time_str = datetime.datetime.now().isoformat()
print("{}: step {}, loss {:g}, acc {:g} ".format(
time_str, step, loss, accuracy))
now = int(time.time())
timeArray = time.localtime(now)
timeStamp = time.strftime("%Y%m%d%H%M%S", timeArray)
timeDay = time.strftime("%Y%m%d", timeArray)
print(timeStamp)
predicted = predict(sess, cnn, train, alphabet,
FLAGS.batch_size, q_max_sent_length,
a_max_sent_length)
predicted_label = np.argmax(predicted, 1)
map_mrr_train = evaluation.evaluationBypandas(
train, predicted[:, -1])
predicted_test = predict(sess, cnn, test, alphabet,
FLAGS.batch_size, q_max_sent_length,
a_max_sent_length)
predicted_label = np.argmax(predicted_test, 1)
map_mrr_test = evaluation.evaluationBypandas(
test, predicted_test[:, -1])
if map_mrr_test[0] > map_max:
map_max = map_mrr_test[0]
timeStamp = time.strftime("%Y%m%d%H%M%S",
time.localtime(int(time.time())))
folder = 'runs/' + timeDay
out_dir = folder + '/' + timeStamp + \
'__' + FLAGS.data + str(map_mrr_test[0])
if not os.path.exists(folder):
os.makedirs(folder)
#save_path = saver.save(sess, out_dir)
print("{}:train epoch:map mrr {}".format(i, map_mrr_train))
print("{}:test epoch:map mrr {}".format(i, map_mrr_test))
line2 = " {}:epoch: map_test{}".format(i, map_mrr_test)
log.write(line2 + '\n')
log.flush()
log.close()
def train():
opt = opts.parse_opt()
opt.input_data = "MNIST"
img_size = (opt.img_dim, opt.img_dim)
print 'Dimension of images:', img_size
train_data, train_label, id_gender = \
get_30_people_chunk(opt.image_path, 1, gender_meta=True, img_size=img_size)
test_data, test_label = get_30_people_chunk(opt.image_path,
2,
img_size=img_size)
names = get_people_names(opt.image_path, 30)
if opt.balance_data:
ratio = opt.balance_ratio
print 'Balancing dataset with ratio %f' % ratio
train_data, train_label = balance_dataset(train_data, train_label)
test_data, test_label = balance_dataset(test_data, test_label)
if opt.balance_gender:
print train_data.shape, train_label.shape
print test_data.shape, test_label.shape
print 'Balancing genders'
selected_people = []
for i in range(id_gender.shape[1]):
indices, = np.where(id_gender[:, i] == 1)
selected_people.append(np.random.choice(indices, 5, replace=False))
selected_people = np.concatenate(selected_people)
print 'Selected people are:'
print np.array(names)[selected_people]
selected_imgs = train_label[:, selected_people].sum(axis=1) != 0
train_data = train_data[selected_imgs, :]
train_label = train_label[selected_imgs, :]
selected_imgs = test_label[:, selected_people].sum(axis=1) != 0
test_data = test_data[selected_imgs, :]
test_label = test_label[selected_imgs, :]
print 'Shape of data:'
print '\tTraining data: ' + str(train_data.shape)
print '\tTraining label: ' + str(train_label.shape)
print '\tMax, Min Train: %.4f, %.4f' % (np.max(train_data),
np.min(train_data))
print '\tTest data: ' + str(test_data.shape)
print '\tTest label: ' + str(test_label.shape)
print '\tMax, Min Test: %.4f, %.4f' % (np.max(test_data),
np.min(test_data))
x_dim = train_data.shape[1]
y_dim = train_label.shape[1]
opt.input_c_dim = 3
opt.output_c_dim = 3
opt.input_dim = x_dim
opt.label_dim = y_dim
input_shape = (x_dim, x_dim, opt.input_c_dim)
batch_size = opt.batch_size
print 'Batch size: %d' % batch_size
NUM_REPR = 5
NUM_SAMPLES_EACH = int(batch_size / NUM_REPR / 2)
output_samples = get_output_samples(train_data, train_label, id_gender,
NUM_REPR, NUM_SAMPLES_EACH)
NUM_THREADS = 2
tf_config = tf.ConfigProto()
tf_config.intra_op_parallelism_threads = NUM_THREADS
tf_config.gpu_options.allow_growth = True
iteration_time = []
with tf.Session(config=tf_config) as sess:
id_model_path = '%s_%d_id_0' % (opt.lfw_base_path, x_dim)
print '\tRetrieving evil model from "%s"' % id_model_path
evil_model = FaceRecognizer(id_model_path, train_label.shape[1],
input_shape, opt.input_c_dim)
gender_model_path = '%s_%d_gender_0' % (opt.lfw_base_path, x_dim)
print '\tRetrieving good model from "%s"' % gender_model_path
good_model = FaceRecognizer(gender_model_path, 2, input_shape,
opt.input_c_dim)
model = advGAN(good_model, evil_model, opt, sess, mnist=False)
iteration = 0
if opt.resnet_gen:
generator_mode = 'ResNet'
else:
generator_mode = 'Regular'
summary_dir = "logs/LFW/g_%d_ld_%d_gl_%d_L2_%.2f_lr_%.4f_%s/" % (
opt.G_lambda, opt.ld, opt.good_loss_coeff, opt.L2_lambda,
opt.learning_rate, generator_mode)
if os.path.isdir(summary_dir) is False:
print 'Creating directory %s for logs.' % summary_dir
os.mkdir(summary_dir)
# else:
# print 'Removing all files in %s' % (summary_dir + '*')
# shutil.rmtree(summary_dir)
writer = tf.summary.FileWriter(summary_dir, sess.graph)
loader = Dataset2(train_data, train_label)
print 'Training data loaded.'
print 'Maximum iterations: %d' % opt.max_iteration
max_acc_diff = -1.0
while iteration < opt.max_iteration:
# this function returns (data, label, np.array(target)).
feed_data, evil_labels, real_data = loader.next_batch(
batch_size, negative=False)
good_labels = id_gender[np.argmax(evil_labels, axis=1)]
feed = {
model.source: feed_data,
model.target: real_data,
model.good_labels: good_labels,
model.evil_labels: evil_labels
}
# Training G once.
summary_str, G_loss, _ = sess.run(
[model.total_loss_merge_sum, model.g_loss, model.G_train_op],
feed)
writer.add_summary(summary_str, iteration)
# Training G twice.
summary_str, G_loss, gan_loss, hinge_loss, l1_loss, l2_loss, \
good_fn_loss, evil_fn_loss, adv_loss, total_loss, _ = sess.run([
model.total_loss_merge_sum,
model.g_loss,
model.gan_loss,
model.hinge_loss,
model.l1_loss,
model.l2_loss,
model.good_fn_loss,
model.evil_fn_loss,
model.adv_loss,
model.total_loss,
model.G_train_op], feed)
writer.add_summary(summary_str, iteration)
# Training D.
summary_str, D_loss, _ = \
sess.run([model.total_loss_merge_sum, model.d_loss, model.D_pre_train_op], feed)
writer.add_summary(summary_str, iteration)
if iteration % opt.losses_log_every == 0:
print "iteration: ", iteration
print '\tD: %.4f, G: %.4f\n\thinge(%.2f): %.4f, L1(%.2f): %.4f, L2(%.2f): %.4f' % (
D_loss, G_loss, opt.H_lambda, hinge_loss, opt.L1_lambda,
l1_loss, opt.L2_lambda, l2_loss)
print '\t\tGAN total loss: %.4f' % gan_loss
print '\tGood: %.4f, Evil: %.4f' % (good_fn_loss, evil_fn_loss)
print '\tAdv: %.4f, Total: %.4f' % (adv_loss, total_loss)
new_test_data = []
new_pred_data = []
head = 0
last_batch = False
while head < test_data.shape[0]:
if head + batch_size <= test_data.shape[0]:
tail = head + batch_size
else:
tail = test_data.shape[0]
head = test_data.shape[0] - batch_size
last_batch = True
cur_data, pred_data = sess.run(
[model.fake_images_output, model.prediction_ready],
{model.source: test_data[head:tail, :]})
if last_batch:
new_test_data.append(
cur_data[-(test_data.shape[0] % batch_size):, :])
new_pred_data.append(
pred_data[-(test_data.shape[0] % batch_size):, :])
else:
new_test_data.append(cur_data)
new_pred_data.append(pred_data)
head += batch_size
new_test_data = np.concatenate(new_test_data)
new_pred_data = np.concatenate(new_pred_data)
good_pred = np.argmax(
model.good_model.model.predict(new_pred_data), axis=1)
evil_pred = np.argmax(
model.evil_model.model.predict(new_pred_data), axis=1)
evil_true = np.argmax(test_label, axis=1)
good_true = np.argmax(id_gender[evil_true, :], axis=1)
good_accuracy = accuracy_score(good_true, good_pred)
evil_accuracy = accuracy_score(evil_true, evil_pred)
total_good_confusion = confusion_matrix(good_true, good_pred)
total_evil_confusion = confusion_matrix(
evil_true, evil_pred, labels=range(opt.evil_label_num))
print '\tGood Accuracy: %.4f, Evil Accuracy: %.4f' % (
good_accuracy, evil_accuracy)
print '\tAccuracy diff: %f' % (good_accuracy - evil_accuracy)
print 'Good confusion matrix:'
print total_good_confusion
evil_misclass = total_evil_confusion.sum(
axis=0) - np.diag(total_evil_confusion)
evil_idxs = np.argsort(-evil_misclass)
print 'Top 3 Misclassifications:'
print np.array(names)[evil_idxs][:3]
print evil_misclass[evil_idxs][:3]
evil_tp = np.diag(total_evil_confusion)
evil_idxs = np.argsort(-evil_tp)
print 'Top 3 True classifications:'
print np.array(names)[evil_idxs][:3]
print evil_tp[evil_idxs][:3]
# print 'Selected people are:'
# print names[evil_idxs].tolist()
# print evil_tp
# print total_evil_confusion
# print evil_idxs
fake_samples, fake_noise = sess.run(
[model.fake_images_output, model.fake_noise_output],
{model.source: output_samples})
fakes = merge(fake_samples, [2 * NUM_REPR, NUM_SAMPLES_EACH])
original = merge(output_samples,
[2 * NUM_REPR, NUM_SAMPLES_EACH])
noise = merge(fake_noise, [2 * NUM_REPR, NUM_SAMPLES_EACH])
final_image = np.concatenate([fakes, noise, original], axis=1)
scipy_imsave('snapshot_%d.png' % iteration, final_image)
if (good_accuracy - evil_accuracy) > max(0.5, max_acc_diff):
print '\tSaving new training data at accuracy diff: %.4f' % (
good_accuracy - evil_accuracy),
max_acc_diff = good_accuracy - evil_accuracy
# other_good = FaceRecognizer('%s_%d_gender_0' % (opt.lfw_base_path, x_dim),
# 2, input_shape, opt.input_c_dim)
# other_pred = np.argmax(other_good.model.predict(new_pred_data), axis=1)
# print 'Other Good accuracy: %.4f' % accuracy_score(good_true, other_pred)
# other_pred = np.argmax(other_good.model.predict(
# preprocess_images(new_test_data * 255.0)), axis=1)
# print '\tTest data processeced accuracy: %.4f' % \
# accuracy_score(good_true, other_pred)
# other_evil = FaceRecognizer('%s_%d_id_0' % (opt.lfw_base_path, x_dim),
# 34, input_shape, opt.input_c_dim)
# other_pred = np.argmax(other_evil.model.predict(new_pred_data), axis=1)
# print 'Other Evil accuracy: %.4f' % accuracy_score(evil_true, other_pred)
# other_pred = np.argmax(other_evil.model.predict(
# preprocess_images(new_test_data * 255.0)), axis=1)
# print '\tTest data processeced accuracy: %.4f' % \
# accuracy_score(evil_true, other_pred)
new_train_data = []
head = 0
last_batch = False
while head < train_data.shape[0]:
if head + batch_size <= train_data.shape[0]:
tail = head + batch_size
else:
tail = train_data.shape[0]
head = train_data.shape[0] - batch_size
last_batch = True
cur_data = sess.run(
model.fake_images_output,
{model.source: train_data[head:tail, :]})
if last_batch:
new_train_data.append(
cur_data[-(train_data.shape[0] %
batch_size):, :])
else:
new_train_data.append(cur_data)
head += batch_size
new_train_data = np.concatenate(new_train_data)
np.savez_compressed(opt.output_path,
train_data=new_train_data,
org_train_data=train_data,
train_label=train_label,
test_data=new_test_data,
org_test_data=test_data,
test_label=test_label,
id_gender=id_gender)
print '\t[DONE]'
iteration += 1
def parse_opt():
args = opts.parse_opt()
print('* parse_opt : ', vars(args))
def main():
opt = opts.parse_opt()
data_dir = opt.input_data_dir
# Data augmentation and normalization for training
# Just normalization for validation
data_transforms = {
'train':
transforms.Compose([
transforms.Resize((256, 256)),
transforms.RandomResizedCrop(224),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
]),
'val':
transforms.Compose([
transforms.Resize((224, 224)),
# transforms.CenterCrop(224),
transforms.ToTensor(),
transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
]),
}
batch_size = opt.batch_size
image_datasets = {
x: VisualConceptDataset(os.path.join(data_dir, x),
os.path.join(opt.input_label_dir, x),
data_transforms[x])
for x in ['train', 'val']
}
dataloaders = {
x: torch.utils.data.DataLoader(image_datasets[x],
batch_size=batch_size,
shuffle=True,
num_workers=4)
for x in ['train', 'val']
}
dataset_sizes = {x: len(image_datasets[x]) for x in ['train', 'val']}
# device = torch.device("cuda:2" if torch.cuda.is_available() else "cpu")
log_every = opt.losses_log_every
print_every = opt.print_every
checkpoint_path = opt.checkpoint_path
if not os.path.exists(checkpoint_path):
os.mkdir(checkpoint_path)
TIMESTAMP = "{0:%Y-%m-%dT%H-%M-%S/}".format(datetime.now())
tb_summary_writer = tb and tb.SummaryWriter(
os.path.join(checkpoint_path, TIMESTAMP))
def train_model(model, criterion, optimizer, scheduler, num_epochs=25):
since = time.time()
best_model_wts = copy.deepcopy(model.state_dict())
best_acc = 0.0
iteration = 0
for epoch in range(num_epochs):
print('Epoch {}/{}'.format(epoch, num_epochs - 1))
print('-' * 100)
# Each epoch has a training and validation phase
for phase in ['val', 'train']:
if phase == 'train':
scheduler.step()
model.train() # Set model to training mode
else:
model.eval() # Set model to evaluate mode
running_loss = 0.0
running_corrects = 0
running_prec = 0.0
running_recall = 0.0
f1score_sum = 0.0
# Iterate over data.
for inputs, labels in dataloaders[phase]:
# inputs = inputs.to(device)
inputs = inputs.cuda()
# labels = labels.float().to(device)
labels = labels.to(torch.float).cuda()
binary_labels = torch.gt(labels, 0).to(torch.int)
# print('in:', inputs.size(), labels.size())
# zero the parameter gradients
optimizer.zero_grad()
# forward
# track history if only in train
with torch.set_grad_enabled(phase == 'train'):
outputs = model(inputs)
preds = torch.gt(outputs, 0).to(torch.int)
# loss only consider label 1 predict and 0 predict ignore
if opt.label_smoothing:
# label smoothing loss
outputs.mul(labels)
else:
# without label smoothing
outputs.mul(binary_labels.to(torch.float))
loss = criterion(outputs, labels)
# backward + optimize only if in training phase
if phase == 'train':
loss.backward()
optimizer.step()
# statistics
running_loss += loss.item() * inputs.size(0)
running_corrects += torch.sum(preds == binary_labels.data)
'''
pred = [1, 1, 0, 0, 1]
label = [1, 0, 0, 1, 1]
pred == label -> [1, 0, 1, 0, 1]
tp = torch.sum([1, 0, 0, 0, 1])
prec = tp / torch.sum(pred)
recall = tp / torch.sum(label)
'''
tmp1 = (preds == binary_labels.data).to(torch.int)
tmp2 = binary_labels.mul(tmp1)
tp = torch.sum(tmp2).to(torch.float)
prec = torch.div(tp,
torch.sum(preds).to(torch.float) + 1e-8)
recall = torch.div(
tp,
torch.sum(binary_labels.to(torch.float)) + 1e-8)
f1score = torch.div(2 * prec * recall,
prec + recall + 1e-8)
running_prec += prec
running_recall += recall
f1score_sum += f1score
# print('pred:', preds.size(), labels.size())
iteration += 1
add_summary_value(tb_summary_writer, 'train_loss', loss,
iteration)
add_summary_value(tb_summary_writer, 'running_loss',
running_loss / iteration, iteration)
add_summary_value(tb_summary_writer, 'running_corrects',
running_corrects / iteration, iteration)
add_summary_value(tb_summary_writer, 'running_tp', tp,
iteration)
add_summary_value(tb_summary_writer, 'running_prec', prec,
iteration)
add_summary_value(tb_summary_writer, 'running_recall',
recall, iteration)
add_summary_value(tb_summary_writer, 'running_f1score',
f1score, iteration)
if (iteration % print_every == 0):
print(
'{} : Epoch {} Iteration {} Loss: {:.4f}/10000 running_loss: {:.4f}, Acc: {:.4f}'
.format(phase, epoch, iteration, loss * 10000,
running_loss / iteration,
running_corrects / batch_size))
print(
'TP: {}, Prec: {}, Recall: {} F1_score: {}'.format(
tp.data / batch_size, prec.data / batch_size,
recall.data / batch_size,
f1score.data / batch_size))
epoch_loss = running_loss / dataset_sizes[phase]
epoch_acc = running_corrects.double() / dataset_sizes[phase]
epoch_prec = running_prec / dataset_sizes[phase]
epoch_recall = running_recall / dataset_sizes[phase]
epoch_f1score = f1score_sum / dataset_sizes[phase]
if phase == 'train':
add_summary_value(tb_summary_writer, 'train_epoch_loss',
epoch_loss, epoch)
add_summary_value(tb_summary_writer,
'train_epoch_corrects', epoch_acc, epoch)
add_summary_value(tb_summary_writer, 'train_epoch_f1score',
epoch_f1score, epoch)
add_summary_value(tb_summary_writer, 'train_epoch_prec',
epoch_prec, epoch)
add_summary_value(tb_summary_writer, 'train_epoch_recall',
epoch_recall, epoch)
else:
add_summary_value(tb_summary_writer, 'val_epoch_loss',
epoch_loss, epoch)
add_summary_value(tb_summary_writer, 'val_epoch_corrects',
epoch_acc, epoch)
add_summary_value(tb_summary_writer, 'val_epoch_f1score',
epoch_f1score, epoch)
add_summary_value(tb_summary_writer, 'val_epoch_prec',
epoch_prec, epoch)
add_summary_value(tb_summary_writer, 'val_epoch_recall',
epoch_recall, epoch)
print()
print('{} : Loss: {:.4f} Acc: {:.4f} '.format(
phase, epoch_loss, epoch_acc))
print('prec: {:.4f} recall: {:.4f} f1score: {:.4f}\n'.format(
epoch_prec, epoch_recall, epoch_f1score))
# deep copy the model
# if phase == 'val' and epoch_acc > best_acc:
if phase == 'val' and epoch_f1score > best_acc:
print('epoch_f1score: %f , history_best_score: %f' %
(epoch_f1score, best_acc))
# best_acc = epoch_acc
best_acc = epoch_f1score
best_model_wts = copy.deepcopy(model.state_dict())
torch.save(
model.state_dict(),
os.path.join(checkpoint_path,
'%s-model-best.pth' % opt.id))
torch.save(
optimizer.state_dict(),
os.path.join(checkpoint_path,
'%s-info-best.path' % opt.id))
print("model save to %s/%s-model-best.pth" %
(checkpoint_path, opt.id))
print()
print()
time_elapsed = time.time() - since
print('Training complete in {:.0f}m {:.0f}s'.format(
time_elapsed // 60, time_elapsed % 60))
print('Best val Acc: {:4f}'.format(best_acc))
# load best model weights
model.load_state_dict(best_model_wts)
return model
# 调用模型
model_ft = models.resnet101(pretrained=True)
# 提取fc层中固定的参数
fc_features = model_ft.fc.in_features
# 修改类别为 vocab_size
num_classes = opt.num_classes
model_ft.fc = nn.Linear(fc_features, num_classes)
# model_ft = myResnet(model, num_classes)
# model_ft = model_ft.to(device)
model_ft = model_ft.cuda()
'''
For example, if a dataset contains 100 positive and 300 negative examples of a single class,
then pos_weight for the class should be equal to 300.
The loss would act as if the dataset contains 3×100=300 positive examples.
'''
criterion = nn.BCEWithLogitsLoss(pos_weight=torch.tensor(
opt.pos_weight).cuda()) # average caption length is 9.5
# criterion = nn.BCEWithLogitsLoss(reduction='sum')
# criterion = nn.MultiLabelMarginLoss()
# criterion = NEG_loss(num_classes, )
# Observe that all parameters are being optimized
if opt.optim == 'sgd':
optimizer_ft = optim.SGD(model_ft.parameters(),
lr=opt.learning_rate,
momentum=opt.momentum)
elif opt.optim == 'adam':
optimizer_ft = optim.Adam(model_ft.parameters(), lr=opt.learning_rate)
else:
print('%s is not supported yet' % opt.optim)
# Decay LR by a factor of 0.1 every 7 epochs
exp_lr_scheduler = lr_scheduler.StepLR(
optimizer_ft,
step_size=opt.learning_rate_decay_every,
gamma=opt.learning_rate_decay_factor)
if opt.label_smoothing:
print('Using Label Smoothing.')
model_ft = train_model(model_ft,
criterion,
optimizer_ft,
exp_lr_scheduler,
num_epochs=opt.max_epochs)
torch.save(model_ft.state_dict(),
os.path.join(checkpoint_path, '%s-model-best.pth' % opt.id))
print("model save to %s/%s-model-best.pth" % (checkpoint_path, opt.id))
import numpy as np
import pandas as pd
from data_augmentation import random_transform
from random import randint
from collections import Counter
from sklearn.utils import shuffle
from PIL import Image
import opts
args = opts.parse_opt()
if args.input_size == 128:
resize_shape = (128, 128, 3)
else:
resize_shape = (256, 256, 3)
def get_image(file,
shape=(resize_shape[0], resize_shape[1]),
location='data/train/'):
image = Image.open(location + file)
image = image.resize(shape)
image = np.array(image)
if len(image.shape) == 2:
image = np.stack([image] * 3, axis=2)
return image
def get_data():
data = pd.read_csv("data/train.csv")
data = shuffle(data)
def train():
flatten_flag = True # flatten output of G or not?
opt = opts.parse_opt()
opt.input_data = "MNIST"
# mapping [0,1] -> [-1,1]
# load data
# mnist = input_data.read_data_sets('MNIST_data', one_hot=True)
# train_data = mnist.train.images * 2.0 - 1.0
# train_label = mnist.train.labels
# test_data = mnist.test.images * 2.0 - 1.0
# test_label = mnist.test.labels
loaded = np.load('MNIST_data/B.npz')
train_data, train_label, test_data, test_label = \
loaded['train_data'], loaded['train_label'], \
loaded['test_data'], loaded['test_label']
# We create the label clues here.
if opt.cgan_gen is True:
label_clue = np.zeros((train_label.shape[1], opt.img_dim, opt.img_dim,
train_label.shape[1]))
for lbl in range(train_label.shape[1]):
label_clue[lbl, :, :, lbl] = 1
if opt.cgan_gen:
output_samples, output_labels = output_sample(test_data, test_label,
True)
else:
output_samples = output_sample(test_data, test_label)
print output_samples.shape
print 'Shape of data:'
print '\tTraining data: ' + str(train_data.shape)
print '\tTraining label: ' + str(train_label.shape)
print '\tTest data: ' + str(test_data.shape)
print '\tTest label: ' + str(test_label.shape)
x_dim = train_data.shape[1]
y_dim = train_label.shape[1]
opt.input_c_dim = 1
opt.output_c_dim = 1
opt.input_dim = x_dim
opt.label_dim = y_dim
batch_size = opt.batch_size
NUM_THREADS = 2
tf_config = tf.ConfigProto()
tf_config.intra_op_parallelism_threads = NUM_THREADS
tf_config.gpu_options.allow_growth = True
with tf.Session(config=tf_config) as sess:
# Initialize the variables, and restore the variables form checkpoint if there is.
# and initialize the writer
global_step = 0
print '\tRetrieving evil model from "%s"' % opt.evil_model_path
evil_model = MNISTModel(opt.evil_model_path)
print '\tRetrieving good model from "%s"' % opt.good_model_path
good_model = OddEvenMNIST(opt.good_model_path)
# model = advGAN(whitebox_model, model_store, opt, sess)
model = advGAN(good_model, evil_model, opt, sess)
min_adv_accuracy = 10e10
max_accuracy_diff = -np.inf
# summary_dir = "logs/MNIST/g_%d_ld_%d_gl_%d_L2_%.2f_dn_%d" % (
# opt.G_lambda, opt.ld, opt.good_loss_coeff,
# opt.L2_lambda, opt.d_train_num)
summary_dir = "logs/MNIST/dn_%d_gn_%d" % (opt.d_train_num,
opt.g_train_num)
duplicate_num = 0
while os.path.isdir(summary_dir + '_' + str(duplicate_num) + '/'):
duplicate_num += 1
summary_dir += '_' + str(duplicate_num) + '/'
print 'Creating directory %s for logs.' % summary_dir
os.mkdir(summary_dir)
writer = tf.summary.FileWriter(summary_dir, sess.graph)
loader = Dataset2(train_data, train_label)
print 'Training data loaded.'
best_evil_accuracy = -1.0
best_res_epoch = -1
best_res = None
for epoch_num in range(opt.max_epoch):
print 'Epoch %d' % epoch_num
# Randomly shuffle the data.
random_indices = np.arange(train_data.shape[0])
np.random.shuffle(random_indices)
train_data = train_data[random_indices, :]
train_label = train_label[random_indices, :]
real_buckets = []
for lbl in range(train_label.shape[1]):
real_buckets.append(np.where(train_label[:, lbl] == 1)[0])
# Mini-batch Gradient Descent.
batch_no = 0
while (batch_no * batch_size) < train_data.shape[0]:
head = batch_no * batch_size
if head + batch_size <= train_data.shape[0]:
tail = head + batch_size
else:
tail = train_data.shape[0]
head = train_data.shape[0] - batch_size
feed_data = train_data[head:tail, :]
evil_labels = train_label[head:tail, :]
good_labels = odd_even_labels(evil_labels)
# Finding randomly sampled real data.
real_data = np.zeros_like(feed_data)
# Indices of training batch with specific label.
# label_indices[i] = indices of feed data, that have evil_label[i] == 1.
label_indices = [np.where(evil_labels[:, lbl] == 1)[0] \
for lbl in range(evil_labels.shape[1])]
for lbl in range(evil_labels.shape[1]):
# We take a random sample of size |label_indices[lbl]|
# from the real bucket of `lbl`.
selected_real_data = np.random.choice(
real_buckets[lbl], label_indices[lbl].shape[0])
# We put this random sample in the same index of their
# corresponding batch training data.
real_data[label_indices[lbl], :] = train_data[
selected_real_data, :]
feed = {
model.source: feed_data,
model.target: real_data,
model.good_labels: good_labels,
model.evil_labels: evil_labels
}
# Train G.
for _ in range(opt.g_train_num):
summary_str, G_loss, gan_loss, hinge_loss, l1_loss, l2_loss, \
good_fn_loss, evil_fn_loss, adv_loss, total_loss, _ = sess.run([
model.total_loss_merge_sum,
model.g_loss,
model.gan_loss,
model.hinge_loss,
model.l1_loss,
model.l2_loss,
model.good_fn_loss,
model.evil_fn_loss,
model.adv_loss,
model.total_loss,
model.G_train_op], feed)
writer.add_summary(summary_str, global_step)
# Train D.
for _ in range(opt.d_train_num):
summary_str, D_loss, _ = sess.run([
model.total_loss_merge_sum, model.d_loss,
model.D_pre_train_op
], feed)
writer.add_summary(summary_str, global_step)
global_step += 1
batch_no += 1
# Validation after each trainig epoch.
print '\tD: %.4f, G: %.4f\n\thinge(%.1f): %.4f, L1(%.1f): %.4f, L2(%.1f): %.4f' % (
D_loss, G_loss, opt.H_lambda, hinge_loss, opt.L1_lambda,
l1_loss, opt.L2_lambda, l2_loss)
print '\t\tGAN total loss: %.4f' % gan_loss
print '\tGood: %.4f, Evil: %.4f' % (good_fn_loss, evil_fn_loss)
print '\tAdv: %.4f, Total: %.4f' % (adv_loss, total_loss)
new_pred_data = []
head = 0
last_batch = False
while head < test_data.shape[0]:
if head + batch_size <= test_data.shape[0]:
tail = head + batch_size
else:
tail = test_data.shape[0]
head = test_data.shape[0] - batch_size
last_batch = True
if opt.cgan_gen:
cur_data = sess.run(
model.fake_images_sample,
{model.evil_labels: test_label[head:tail, :]})
else:
cur_data = sess.run(
model.fake_images_sample,
{model.source: test_data[head:tail, :]})
if last_batch:
new_pred_data.append(
cur_data[-(test_data.shape[0] % batch_size):, :])
else:
new_pred_data.append(cur_data)
head += batch_size
new_pred_data = np.concatenate(new_pred_data)
good_pred = np.argmax(
model.good_model.model.predict(new_pred_data), axis=1)
evil_pred = np.argmax(
model.evil_model.model.predict(new_pred_data), axis=1)
evil_true = np.argmax(test_label, axis=1)
good_true = np.argmax(odd_even_labels(test_label), axis=1)
good_accuracy = accuracy_score(good_true, good_pred)
evil_accuracy = accuracy_score(evil_true, evil_pred)
total_good_confusion = confusion_matrix(good_true, good_pred)
total_evil_confusion = confusion_matrix(evil_true,
evil_pred,
labels=range(
opt.evil_label_num))
print '\tGood Accuracy: %.4f, Evil Accuracy: %.4f' % (
good_accuracy, evil_accuracy)
print '\tAccuracy diff: %f' % (good_accuracy - evil_accuracy)
print 'Good confusion matrix:'
print total_good_confusion
print 'Evil confusion matrix:'
print total_evil_confusion
# Creating snapshots to save.
if opt.cgan_gen:
fake_samples = sess.run(model.fake_images_sample,
{model.evil_labels: output_labels})
else:
fake_samples, fake_noise = sess.run(
[model.fake_images_sample, model.sample_noise],
{model.source: output_samples})
max_accuracy_diff = good_accuracy - evil_accuracy
fakes = merge(fake_samples[:100, :], [10, 10])
separator = np.ones((280, 2))
original = merge(output_samples[:100].reshape(-1, 28, 28, 1),
[10, 10])
if opt.cgan_gen:
scipy.misc.imsave(
'snapshot_%d.png' % epoch_num,
np.concatenate([fakes, separator, original], axis=1))
else:
noise = merge(fake_noise[:100], [10, 10])
scipy.misc.imsave(
'snapshot_%d.png' % epoch_num,
np.concatenate([fakes, noise, original], axis=1))
# Only for the purpose of finding best D and G training times.
if evil_accuracy > best_evil_accuracy:
best_evil_accuracy = evil_accuracy
best_res_epoch = epoch_num
if opt.cgan_gen:
best_res = np.concatenate([fakes, separator, original],
axis=1)
else:
best_res = np.concatenate([fakes, noise, original], axis=1)
best_image_path = 'best_dn_%d_gn_%d_%d_epoch_%d.png' % \
(opt.d_train_num, opt.g_train_num, duplicate_num, best_res_epoch)
scipy.misc.imsave(best_image_path, best_res)
# print 'Maximum iterations: %d' % opt.max_iteration
# while iteration < opt.max_iteration:
# # this function returns (data, label, np.array(target)).
# # data = loader.next_batch(batch_size, negative=False)
# feed_data, evil_labels, real_data = loader.next_batch(
# batch_size, negative=False)
# good_labels = odd_even_labels(evil_labels)
# feed = {
# model.source: feed_data,
# model.target: real_data,
# model.good_labels: good_labels,
# model.evil_labels: evil_labels
# }
# # if opt.cgan_gen:
# # feed[model.label_clue] = label_clue[evil_labels.argmax(axis=1)]
# # Training G once.
# # summary_str, G_loss, _ = sess.run(
# # [model.total_loss_merge_sum, model.g_loss, model.G_train_op], feed)
# # writer.add_summary(summary_str, iteration)
# # Training G twice.
# summary_str, G_loss, gan_loss, hinge_loss, l1_loss, l2_loss, \
# good_fn_loss, evil_fn_loss, adv_loss, total_loss, _ = sess.run([
# model.total_loss_merge_sum,
# model.g_loss,
# model.gan_loss,
# model.hinge_loss,
# model.l1_loss,
# model.l2_loss,
# model.good_fn_loss,
# model.evil_fn_loss,
# model.adv_loss,
# model.total_loss,
# model.G_train_op], feed)
# writer.add_summary(summary_str, iteration)
# # Training D.
# for _ in range(opt.d_train_num):
# summary_str, D_loss, _ = sess.run(
# [model.total_loss_merge_sum, model.d_loss, model.D_pre_train_op], feed)
# writer.add_summary(summary_str, iteration)
# if iteration % opt.losses_log_every == 0:
# # if iteration != 0 and iteration % opt.save_checkpoint_every == 0:
# # checkpoint_path = os.path.join(opt.checkpoint_path, 'checkpoint.ckpt')
# # print 'Saving the model in "%s"' % checkpoint_path
# # model.saver.save(sess, checkpoint_path, global_step=iteration)
# # test_loader = Dataset2(test_data, test_label)
# # test_num = test_loader._num_examples
# # test_iter_num = (test_num - batch_size) / batch_size
# # total_evil_accuracy = 0.0
# # total_good_accuracy = 0.0
# # fake_samples = [[] for _ in range(test_loader._num_labels)]
# # fake_noise = [[] for _ in range(test_loader._num_labels)]
# # original_samples = [[] for _ in range(test_loader._num_labels)]
# # for _ in range(test_iter_num):
# # # Loading the next batch of test images
# # test_input_data, test_evil_labels, _ = \
# # test_loader.next_batch(batch_size)
# # evil_categorical_labels = np.argmax(test_evil_labels, axis=1)
# # test_good_labels = odd_even_labels(test_evil_labels)
# # feed = {
# # model.source: test_input_data,
# # model.evil_labels: test_evil_labels,
# # model.good_labels: test_good_labels
# # }
# # # if opt.cgan_gen:
# # # feed[model.label_clue] = label_clue[test_evil_labels.argmax(axis=1)]
# # evil_accuracy, good_accuracy = sess.run(
# # [model.evil_accuracy, model.good_accuracy], feed)
# # # We divide the total accuracy by the number of test iterations.
# # total_good_accuracy += good_accuracy
# # total_evil_accuracy += evil_accuracy
# # # print 'Evil accuracy: %.6f\tGood accuracy: %.6f' % (
# # # evil_accuracy, good_accuracy)
# # # test_accuracy, test_adv_accuracy = sess.run(
# # # [model.accuracy, model.adv_accuracy], feed)
# # # test_acc += test_accuracy
# # # test_adv_acc += test_adv_accuracy
# # # fake_images, g_x = sess.run(
# # # [model.fake_images_sample, model.sample_noise],
# # # {model.source: test_input_data})
# # # for lbl in range(test_loader._num_labels):
# # # if len(fake_samples[lbl]) < 10:
# # # idx = np.where(evil_categorical_labels == lbl)[0]
# # # if idx.shape[0] >= 10:
# # # fake_samples[lbl] = fake_images[idx[:10]]
# # # fake_noise[lbl] = g_x[idx[:10]]
# # # original_samples[lbl] = test_input_data[idx[:10]]
# # # for lbl, sample, noise in zip(test_evil_labels, fake_images, fake_noise):
# # # if len(fake_samples[lbl]) > 10:
# # # continue
# # # fake_samples[lbl].append(sample)
# # # fake_noise[lbl].append(noise)
# # # pdb.set_trace()
# # # print fake_images.shape
# # # Finding those predicted labels that are equal to the target label
# # # idxs = np.where(out_predict_labels == target_label)[0]
# # # save_images(samples[:100], [10, 10], 'CIFAR10/result2/test_' + str(source_idx) + str(target_idx)+ '_.png')
# # # pdb.set_trace()
# # # show_samples.append(samples)
# # # input_samples.append(s_imgs)
# # # save_samples.append(samples)
# # # if opt.is_advGAN:
# # # save_samples.append(samples[idxs])
# # # else:
# # # We add all samples.
# # # show_samples = np.concatenate(show_samples, axis=0)
# # # save_samples = np.concatenate(save_samples, axis=0)
# # good_accuracy = total_good_accuracy / float(test_iter_num)
# # evil_accuracy = total_evil_accuracy / float(test_iter_num)
# # print '\tAccuracy diff: %f' % (good_accuracy - evil_accuracy)
# # print '\tGood accuracy %f, Evil accuracy %f' % (
# # good_accuracy, evil_accuracy)
# # Resizing the samples to save them later on.
# # fake_samples = np.reshape(np.array(fake_samples), [100, -1])
# # original_samples = np.reshape(np.array(original_samples), [100, -1])
# # fake_noise = np.reshape(np.array(fake_noise), [100, -1])
# # if (good_accuracy - evil_accuracy) > max_accuracy_diff:
# # test_accuracy = test_acc / float(test_iter_num)
# # test_adv_accuracy = test_adv_acc / float(test_iter_num)
# # if (good_accuracy - evil_accuracy) > max_accuracy_diff:
# # max_accuracy_diff = good_accuracy - evil_accuracy
# # if min_adv_accuracy > test_adv_accuracy:
# # min_adv_accuracy = test_adv_accuracy
# # save_images(fake_images[:100], [10, 10], 'fake.png')
# # save_images(test_input_data[:100], [10, 10], 'real.png')
# # all_idx = np.arange(100)
# # odds = np.where((all_idx / 10) % 2 == 1)[0]
# # evens = np.where((all_idx / 10) % 2 == 0)[0]
# # order = np.concatenate((odds, evens))
# # save_images(fake_samples[order], [10, 10], 'best_images.png')
# # save_images(fake_noise[order], [10, 10], 'best_noise.png')
# # save_images(original_samples[order], [10, 10], 'best_original.png')
# # save_anything = True
# # Saving the best yet model.
# # best_model_path = os.path.join(opt.checkpoint_path, 'best.ckpt')
# # print 'Saving the best model yet at "%s"' % best_model_path
# # model.saver.save(sess, best_model_path)
# # if save_anything is False:
# # # Nothing is saved. We save a version here.
# # save_images(fake_samples[:100], [10, 10], 'last_images.png')
# # save_images(fake_noise[:100], [10, 10], 'last_noise.png')
# # save_anything = True
# iteration += 1
# We can transform the training and test data given in the beginning here.
# This is only half the actual data.
if opt.save_data:
# if opt.cgan_gen:
raise NotImplementedError(
'Saving data for CGAN_GEN is not yet implemented.')
def main():
import opts
import misc.utils as utils
opt = opts.parse_opt()
opt.caption_model = 'topdown'
opt.batch_size = 10 #512#32*4*4
opt.id = 'topdown'
opt.learning_rate = 5e-4
opt.learning_rate_decay_start = 0
opt.scheduled_sampling_start = 0
opt.save_checkpoint_every = 5000 #450#5000#11500
opt.val_images_use = 5000
opt.max_epochs = 50 #30
opt.start_from = 'save/rt' #"save" #None
opt.language_eval = 1
opt.input_json = 'data/meta_coco_en.json'
opt.input_label_h5 = 'data/label_coco_en.h5'
# opt.input_json='data/coco_ccg.json' #'data/meta_coco_en.json'
# opt.input_label_h5='data/coco_ccg_label.h5' #'data/label_coco_en.h5'
# opt.input_fc_dir='/nlp/andyweizhao/self-critical.pytorch-master/data/cocotalk_fc'
# opt.input_att_dir='/nlp/andyweizhao/self-critical.pytorch-master/data/cocotalk_att'
opt.finetune_cnn_after = 0
opt.ccg = False
opt.input_image_h5 = 'data/coco_image_512.h5'
opt.use_att = utils.if_use_att(opt.caption_model)
from dataloader import DataLoader # just-in-time generated features
loader = DataLoader(opt)
# from dataloader_fixcnn import DataLoader # load pre-processed features
# loader = DataLoader(opt)
opt.vocab_size = loader.vocab_size
opt.vocab_ccg_size = loader.vocab_ccg_size
opt.seq_length = loader.seq_length
import models
model = models.setup(opt)
cnn_model = utils.build_cnn(opt)
cnn_model.cuda()
model.cuda()
data = loader.get_batch('train')
images = data['images']
# _fc_feats_2048 = []
# _fc_feats_81 = []
# _att_feats = []
# for i in range(loader.batch_size):
# x = Variable(torch.from_numpy(images[i]), volatile=True).cuda()
# x = x.unsqueeze(0)
# att_feats, fc_feats_81 = cnn_model(x)
# fc_feats_2048 = att_feats.mean(3).mean(2).squeeze()
# att_feats = F.adaptive_avg_pool2d(att_feats,[14,14]).squeeze().permute(1, 2, 0)#(0, 2, 3, 1)
# _fc_feats_2048.append(fc_feats_2048)
# _fc_feats_81.append(fc_feats_81)
# _att_feats.append(att_feats)
# _fc_feats_2048 = torch.stack(_fc_feats_2048)
# _fc_feats_81 = torch.stack(_fc_feats_81)
# _att_feats = torch.stack(_att_feats)
# att_feats = _att_feats.unsqueeze(1).expand(*((_att_feats.size(0), loader.seq_per_img,) + \
# _att_feats.size()[1:])).contiguous().view(*((_att_feats.size(0) * loader.seq_per_img,) + \
# _att_feats.size()[1:]))
# fc_feats_2048 = _fc_feats_2048.unsqueeze(1).expand(*((_fc_feats_2048.size(0), loader.seq_per_img,) + \
# _fc_feats_2048.size()[1:])).contiguous().view(*((_fc_feats_2048.size(0) * loader.seq_per_img,) + \
# _fc_feats_2048.size()[1:]))
# fc_feats_81 = _fc_feats_81
#
# att_feats = Variable(att_feats, requires_grad=False).cuda()
# Variable(fc_feats_81)
crit = utils.LanguageModelCriterion()
eval_kwargs = {'split': 'val', 'dataset': opt.input_json, 'verbose': True}
eval_kwargs.update(vars(opt))
val_loss, predictions, lang_stats = eval_split(cnn_model, model, crit,
loader, eval_kwargs, True)
def main():
opt = opts.parse_opt()
if opt.path_opt is not None:
with open(opt.path_opt, 'r') as handle:
options_yaml = yaml.load(handle, Loader=yaml.FullLoader)
utils.update_values(options_yaml, vars(opt))
opt.checkpoint_path = opt.checkpoint_path + opt.exp_name
print('=============')
print(opt.exp_name)
print('=============')
opt.input_json = opt.data_path + opt.input_json
opt.input_dic = opt.data_path + opt.input_dic
opt.seg_feature_root = opt.data_path + opt.seg_feature_root
opt.feature_root = opt.data_path + opt.feature_root
opt.proposal_h5 = opt.data_path + opt.proposal_h5
opt.densecap_references = [
opt.data_path + reference for reference in opt.densecap_references
]
opt.test_mode = (opt.val_split == 'testing')
if opt.enable_BUTD:
assert opt.att_input_mode == 'region', 'region attention only under the BUTD mode'
cudnn.benchmark = True
torch.manual_seed(opt.seed)
np.random.seed(opt.seed)
random.seed(opt.seed)
if opt.cuda:
torch.cuda.manual_seed_all(opt.seed)
if opt.dataset == 'anet':
from misc.dataloader_anet import DataLoader
else:
raise Exception('only support anet!')
if not os.path.exists(opt.checkpoint_path):
os.makedirs(opt.checkpoint_path)
# open the detection json file.
print('DataLoader loading proposal file: ', opt.proposal_h5)
h5_proposal_file = h5py.File(opt.proposal_h5, 'r', driver='core')
num_proposals = h5_proposal_file['dets_num'][:]
label_proposals = h5_proposal_file['dets_labels'][:]
h5_proposal_file.close()
# Data Loader
dataset = DataLoader(opt,
split=opt.train_split,
seq_per_img=opt.seq_per_img,
num_proposals=num_proposals,
label_proposals=label_proposals)
dataloader = torch.utils.data.DataLoader(dataset,
batch_size=opt.batch_size,
shuffle=True,
num_workers=opt.num_workers)
dataset_val = DataLoader(opt,
split=opt.val_split,
seq_per_img=opt.seq_per_img,
num_proposals=num_proposals,
label_proposals=label_proposals)
dataloader_val = torch.utils.data.DataLoader(dataset_val,
batch_size=opt.batch_size,
shuffle=False,
num_workers=opt.num_workers)
# ======================================================================
# Build the Model
opt.vocab_size = dataset.vocab_size
opt.detect_size = dataset.detect_size
opt.seq_length = opt.seq_length
opt.glove_w = torch.from_numpy(dataset.glove_w).float()
opt.glove_vg_cls = torch.from_numpy(dataset.glove_vg_cls).float()
opt.glove_clss = torch.from_numpy(dataset.glove_clss).float()
opt.wtoi = dataset.wtoi
opt.itow = dataset.itow
opt.itod = dataset.itod
opt.ltow = dataset.ltow
opt.itoc = dataset.itoc
opt.wtol = dataset.wtol
opt.wtod = dataset.wtod
opt.vg_cls = dataset.vg_cls
if opt.att_model == 'cyclical':
model = build_model(opt, device)
else:
raise ValueError('Unknown captioning model: {}'.format(opt.att_model))
infos = {}
histories = {}
# if opt.start_from is not None:
if opt.resume:
if opt.load_best_score == 1:
model_path = os.path.join(opt.checkpoint_path, 'model-best.pth')
info_path = os.path.join(opt.checkpoint_path,
'infos_' + opt.id + '-best.pkl')
else:
model_path = os.path.join(opt.checkpoint_path, 'model.pth')
info_path = os.path.join(opt.checkpoint_path,
'infos_' + opt.id + '.pkl')
# open old infos and check if models are compatible
with open(info_path, 'rb') as f:
infos = pickle.load(f)
saved_model_opt = infos['opt']
# opt.learning_rate = saved_model_opt.learning_rate
print('========================================')
print('Loading the model %s...' % (model_path))
if opt.inference_only:
print('Running Inference only ...')
print('========================================')
# model.load_state_dict(torch.load(model_path))
if not is_code_development():
model.load_state_dict(torch.load(model_path))
else:
model.load_state_dict(
torch.load(model_path,
map_location=lambda storage, loc: storage))
if os.path.isfile(
os.path.join(opt.checkpoint_path,
'histories_' + opt.id + '.pkl')):
with open(
os.path.join(opt.checkpoint_path,
'histories_' + opt.id + '.pkl'), 'rb') as f:
histories = pickle.load(f)
best_val_score = infos.get('best_val_score', None)
iteration = infos.get('iter', 0)
if opt.resume_decoder_exp_name != '' and not opt.resume:
start_epoch = opt.start_epoch
else:
start_epoch = infos.get('epoch', 0)
val_result_history = histories.get('val_result_history', {})
loss_history = histories.get('loss_history', {})
lr_history = histories.get('lr_history', {})
ss_prob_history = histories.get('ss_prob_history', {})
model = nn.DataParallel(model).to(device)
params = []
for key, value in dict(model.named_parameters()).items():
if value.requires_grad:
if ('ctx2pool_grd' in key) or ('vis_embed' in key):
print('Finetune param: {}'.format(key))
params += [{
'params': [value],
'lr': opt.learning_rate * 0.1, # finetune the fc7 layer
'weight_decay': opt.weight_decay,
'betas': (opt.optim_alpha, opt.optim_beta)
}]
else:
params += [{
'params': [value],
'lr': opt.learning_rate,
'weight_decay': opt.weight_decay,
'betas': (opt.optim_alpha, opt.optim_beta)
}]
print("Use %s as optmization method" % (opt.optim))
optimizer = None
if opt.optim == 'sgd':
optimizer = optim.SGD(params, lr=opt.learning_rate, momentum=0.9)
elif opt.optim == 'adam':
optimizer = optim.Adam(params)
elif opt.optim == 'adamax':
optimizer = optim.Adamax(params)
else:
raise ValueError('Unknown optimizer: {}'.format(opt.optim))
# set up tensorboard logger
tb_logger = utils.set_tb_logger(
opt.tb_log_dir, opt.exp_name,
opt.resume) if not opt.inference_only else None
# set up trainer
trainer = Trainer(opt, dataset, model, optimizer, dataloader,
dataloader_val)
# set up LR scheduler
scheduler = ReduceLROnPlateau(optimizer,
'max',
patience=opt.patience,
min_lr=opt.min_lr)
best_score = {
"Bleu_1": 0.0,
"Bleu_2": 0.0,
"Bleu_3": 0.0,
"Bleu_4": 0.0,
"METEOR": 0.0,
"ROUGE_L": 0.0,
"CIDEr": 0.0,
"SPICE": 0.0
}
for epoch in range(start_epoch, opt.max_epochs):
if not opt.inference_only:
trainer.train(epoch, tb_logger=tb_logger)
if epoch % opt.val_every_epoch == 0:
with torch.no_grad():
lang_stats = trainer.eval(epoch, tb_logger=tb_logger)
if opt.inference_only:
break
# update learning rate by monitoring CIDEr score
scheduler.step(lang_stats['CIDEr'], epoch)
# Save model if is improving on validation result
current_score = lang_stats['CIDEr']
best_flag = False
if best_val_score is None or current_score > best_val_score:
best_val_score = current_score
best_flag = True
checkpoint_path = os.path.join(opt.checkpoint_path, 'model.pth')
if opt.mGPUs:
torch.save(model.module.state_dict(), checkpoint_path)
else:
torch.save(model.state_dict(), checkpoint_path)
print("model saved to {}".format(checkpoint_path))
# Dump miscalleous informations
infos['iter'] = iteration
infos['epoch'] = epoch
infos['best_val_score'] = best_val_score
infos['opt'] = opt
infos['vocab'] = dataset.itow
histories['val_result_history'] = val_result_history
histories['loss_history'] = loss_history
histories['lr_history'] = lr_history
histories['ss_prob_history'] = ss_prob_history
with open(
os.path.join(opt.checkpoint_path,
'infos_' + opt.id + '.pkl'), 'wb') as f:
pickle.dump(infos, f)
with open(
os.path.join(opt.checkpoint_path,
'histories_' + opt.id + '.pkl'), 'wb') as f:
pickle.dump(histories, f)
if best_flag:
checkpoint_path = os.path.join(opt.checkpoint_path,
'model-best.pth')
if opt.mGPUs:
torch.save(model.module.state_dict(), checkpoint_path)
else:
torch.save(model.state_dict(), checkpoint_path)
print("model saved to {} with best cider score {:.3f}".format(
checkpoint_path, best_val_score))
with open(
os.path.join(opt.checkpoint_path,
'infos_' + opt.id + '-best.pkl'),
'wb') as f:
pickle.dump(infos, f)
# update best scores
for metric, _ in best_score.items():
best_score[metric] = lang_stats[metric]
print("===================================")
print("--> Highest scores on {} set at epoch {}".format(
opt.val_split, epoch))
for metric, score in sorted(best_score.items()):
print('{}: {:.4f}'.format(metric, score))
def train():
opt = parse_opt()
train_mode = opt.train_mode
idx2word = json.load(open(opt.idx2word, 'r'))
captions = json.load(open(opt.captions, 'r'))
# 模型
decoder = Decoder(idx2word, opt.settings)
decoder.to(opt.device)
lr = opt.learning_rate
optimizer, xe_criterion = decoder.get_optim_and_crit(lr)
if opt.resume:
print("====> loading checkpoint '{}'".format(opt.resume))
chkpoint = torch.load(opt.resume, map_location=lambda s, l: s)
assert opt.settings == chkpoint['settings'], \
'opt.settings and resume model settings are different'
assert idx2word == chkpoint['idx2word'], \
'idx2word and resume model idx2word are different'
decoder.load_state_dict(chkpoint['model'])
if chkpoint['train_mode'] == train_mode:
optimizer.load_state_dict(chkpoint['optimizer'])
lr = optimizer.param_groups[0]['lr']
print("====> loaded checkpoint '{}', epoch: {}, train_mode: {}".format(
opt.resume, chkpoint['epoch'], chkpoint['train_mode']))
elif train_mode == 'rl':
raise Exception('"rl" mode need resume model')
print('====> process image captions begin')
word2idx = {}
for i, w in enumerate(idx2word):
word2idx[w] = i
captions_id = {}
for split, caps in captions.items():
print('convert %s captions to index' % split)
captions_id[split] = {}
for fn, seqs in tqdm.tqdm(caps.items(), ncols=100):
tmp = []
for seq in seqs:
tmp.append(
[decoder.sos_id] +
[word2idx.get(w, None) or word2idx['<UNK>']
for w in seq] + [decoder.eos_id])
captions_id[split][fn] = tmp
captions = captions_id
print('====> process image captions end')
train_data = get_dataloader(opt.img_feats, captions['train'],
decoder.pad_id, opt.max_seq_len,
opt.batch_size, opt.num_workers)
val_data = get_dataloader(opt.img_feats,
captions['val'],
decoder.pad_id,
opt.max_seq_len,
opt.batch_size,
opt.num_workers,
shuffle=False)
test_captions = {}
for fn in captions['test']:
test_captions[fn] = [[]]
test_data = get_dataloader(opt.img_feats,
test_captions,
decoder.pad_id,
opt.max_seq_len,
opt.batch_size,
opt.num_workers,
shuffle=False)
if train_mode == 'rl':
rl_criterion = RewardCriterion()
ciderd_scorer = get_ciderd_scorer(captions, decoder.sos_id,
decoder.eos_id)
def forward(data, training=True, ss_prob=0.0):
decoder.train(training)
loss_val = 0.0
reward_val = 0.0
for fns, fc_feats, (caps_tensor,
lengths), ground_truth in tqdm.tqdm(data,
ncols=100):
fc_feats = fc_feats.to(opt.device)
caps_tensor = caps_tensor.to(opt.device)
if training and train_mode == 'rl':
sample_captions, sample_logprobs, seq_masks = decoder(
fc_feats,
sample_max=0,
max_seq_len=opt.max_seq_len,
mode=train_mode)
decoder.eval()
with torch.no_grad():
greedy_captions, _, _ = decoder(
fc_feats,
sample_max=1,
max_seq_len=opt.max_seq_len,
mode=train_mode)
decoder.train(training)
reward = get_self_critical_reward(sample_captions,
greedy_captions, fns,
ground_truth, decoder.sos_id,
decoder.eos_id,
ciderd_scorer)
loss = rl_criterion(
sample_logprobs, seq_masks,
torch.from_numpy(reward).float().to(opt.device))
reward_val += float(np.mean(reward[:, 0]))
else:
pred = decoder(fc_feats, caps_tensor, ss_prob=ss_prob)
loss = xe_criterion(pred, caps_tensor[:, 1:], lengths)
loss_val += float(loss)
if training:
optimizer.zero_grad()
loss.backward()
clip_gradient(optimizer, opt.grad_clip)
optimizer.step()
return loss_val / len(data), reward_val / len(data)
checkpoint_dir = os.path.join(opt.checkpoint, train_mode)
if not os.path.exists(checkpoint_dir):
os.makedirs(checkpoint_dir)
result_dir = os.path.join(opt.result, train_mode)
if not os.path.exists(result_dir):
os.makedirs(result_dir)
previous_loss = None
for epoch in range(opt.max_epochs):
print('--------------------epoch: %d' % epoch)
ss_prob = 0.0
if epoch > opt.scheduled_sampling_start >= 0:
frac = (epoch - opt.scheduled_sampling_start
) // opt.scheduled_sampling_increase_every
ss_prob = min(opt.scheduled_sampling_increase_prob * frac,
opt.scheduled_sampling_max_prob)
train_loss, train_reward = forward(train_data, ss_prob=ss_prob)
with torch.no_grad():
val_loss, _ = forward(val_data, training=False)
if train_mode == 'xe' and previous_loss is not None and val_loss >= previous_loss:
lr = lr * 0.5
for param_group in optimizer.param_groups:
param_group['lr'] = lr
previous_loss = val_loss
if epoch in [0, 5, 10, 15, 20, 25, 29, 30, 35, 39, 40, 45, 49]:
# test
results = []
for fns, fc_feats, _, _ in tqdm.tqdm(test_data, ncols=100):
fc_feats = fc_feats.to(opt.device)
for i, fn in enumerate(fns):
fc_feat = fc_feats[i]
with torch.no_grad():
rest, _ = decoder.sample(fc_feat,
beam_size=opt.beam_size,
max_seq_len=opt.max_seq_len)
results.append({'image_id': fn, 'caption': rest[0]})
json.dump(
results,
open(os.path.join(result_dir, 'result_%d.json' % epoch), 'w'))
chkpoint = {
'epoch': epoch,
'model': decoder.state_dict(),
'optimizer': optimizer.state_dict(),
'settings': opt.settings,
'idx2word': idx2word,
'train_mode': train_mode,
}
checkpoint_path = os.path.join(
checkpoint_dir, 'model_%d_%.4f_%s.pth' %
(epoch, val_loss, time.strftime('%m%d-%H%M')))
torch.save(chkpoint, checkpoint_path)
print('train_loss: %.4f, train_reward: %.4f, val_loss: %.4f' %
(train_loss, train_reward, val_loss))
import torch from torch.utils.data import Dataset import pickle import json import os import opts import numpy as np opt = opts.parse_opt() opt = vars(opt) class NCEDataset(Dataset): def __init__(self,opt,mode): super(NCEDataset, self).__init__() self.video_feats = pickle.load(open(opt["s3d_feat_path"],"rb")) self.cap_feats = pickle.load(open(opt["bert_feats_path"],"rb")) info = json.load(open(opt["info_json"])) self.ix_to_word = info['ix_to_word'] self.word_to_ix = info['word_to_ix'] self.splits = info['videos'] self.splits['train'] = list(filter(lambda x: x in self.video_feats.keys(),self.splits['train'])) self.splits['val'] = list(filter(lambda x: x in self.video_feats.keys(),self.splits['val'])) self.splits['test'] = list(filter(lambda x: x in self.video_feats.keys(),self.splits['test'])) self.train = self.splits['train'] self.test = self.splits['test']
def main(args):
cfg = setup(args)
if args.eval_only:
model = Trainer.build_model(cfg)
DetectionCheckpointer(model, save_dir=cfg.OUTPUT_DIR).resume_or_load(
cfg.MODEL.WEIGHTS, resume=args.resume)
res = Trainer.test(cfg, model)
if comm.is_main_process():
verify_results(cfg, res)
return res
trainer = Trainer(cfg)
trainer.resume_or_load(resume=args.resume)
return trainer.train()
if __name__ == "__main__":
args = parse_opt().parse_args()
print("Command Line Args:", args)
launch(
main,
args.num_gpus,
num_machines=args.num_machines,
machine_rank=args.machine_rank,
dist_url=args.dist_url,
args=(args, ),
)
