def main():
# Argument parsing
args = parse_arguments()
log_filename = os.path.join('report/version_4/',
'dualpath_v4_stage_1_eval.log')
if not os.path.exists('report/version_4/'):
os.mkdir('report/version_4/')
images_names = list(dataset_flickr30k.keys())
index_flickr30k = [i for i in range(len(dataset_flickr30k))]
index_train, index_validate, index_test = mylib.generate_3_subsets(
index_flickr30k, ratio=[0.93, 0.035, 0.035])
print("Number of samples in 3 subsets are {}, {} and {}".format(
len(index_train), len(index_validate), len(index_test)))
images_names_train = [images_names[i] for i in index_train]
images_names_val = [images_names[i] for i in index_validate]
images_names_test = [images_names[i] for i in index_test]
list_dataset = []
all_class = []
for idx, img_name in enumerate(images_names_val):
img_class = img_name[0:-4] # remove '.jpg'
all_class += [img_class]
for des in dataset_flickr30k[img_name]:
temp = [img_name, des, img_class]
list_dataset.append(temp)
start_epoch = args.start_epoch
if args.end_epoch == 0 or args.end_epoch < start_epoch:
end_epoch = start_epoch
else:
end_epoch = args.end_epoch
for idx in range(start_epoch, end_epoch + 1):
print("Evaluating Epoch {}".format(idx))
pretrained_model = 'checkpoints_v4_stage_1/checkpoints_v4_' + str(
idx) + '/my_checkpoint'
model = create_model(dict_length=len(my_dictionary),
pretrained_model=pretrained_model,
print_summary=False)
accuracy, class_loss, rank_loss = eval_model(model, list_dataset)
info = "Epoch: {}\nLoss_Classify: {:.6f}\nLoss_Ranking: {:.6f}\nAccuracy: {:.4f}\n-----".format(
idx, class_loss.numpy(), rank_loss.numpy(), accuracy)
print("Writing to report file")
with open(log_filename, 'a') as f:
f.write(info)
def main():
save_path = 'checkpoints/' # directory path to save checkpoint
if config.stage_2:
log_filename = os.path.join('report/dualpath/', 'dualpath_stage_2.log')
else:
log_filename = os.path.join('report/dualpath/', 'dualpath_stage_1.log')
if not os.path.exists('report/dualpath/'):
os.mkdir('report/')
os.mkdir('report/dualpath/')
images_names = list(dataset_flickr30k.keys())
index_flickr30k = [i for i in range(len(dataset_flickr30k))]
index_train, index_validate, index_test = mylib.generate_3_subsets(
index_flickr30k, ratio=[0.93, 0.035, 0.035])
print("Number of samples in 3 subsets are {}, {} and {}".format(
len(index_train), len(index_validate), len(index_test)))
images_names_train = [images_names[i] for i in index_train]
images_names_val = [images_names[i] for i in index_validate]
images_names_test = [images_names[i] for i in index_test]
list_dataset = []
all_class = [x[0:-4] for x in images_names_train]
all_class_sort = sorted(all_class)
for idx, img_name in enumerate(images_names_train):
img_class = all_class_sort.index(img_name[0:-4])
for des in dataset_flickr30k[img_name]:
temp = [img_name, des, img_class]
list_dataset.append(temp)
seeds = [x for x in range(config.numb_epochs)]
if config.stage_2: # parameter for ranking loss (stage 2)
lamb_0 = config.lamb_0
else:
lamb_0 = 0
model = create_model(nclass=len(images_names_train),
nword=len(my_dictionary),
ft_resnet=False)
optimizer = keras.optimizers.Adadelta()
ckpt = tf.train.Checkpoint(iters=tf.Variable(0),
epoch=tf.Variable(0),
optimizer=optimizer,
model=model)
manager = tf.train.CheckpointManager(ckpt, save_path, max_to_keep=1)
ckpt.restore(manager.latest_checkpoint)
if manager.latest_checkpoint:
print("Restored from {}".format(manager.latest_checkpoint))
else:
print("Initializing from scratch.")
last_epoch = int(ckpt.epoch)
last_index = int(ckpt.iters)
for current_epoch in range(last_epoch, config.numb_epochs):
epoch_loss_total_avg = tf.keras.metrics.Mean(
) # track mean loss in current epoch
epoch_loss_visual_avg = tf.keras.metrics.Mean(
) # track mean loss in current epoch
epoch_loss_text_avg = tf.keras.metrics.Mean(
) # track mean loss in current epoch
if config.stage_2:
epoch_loss_ranking_avg = tf.keras.metrics.Mean(
) # track mean loss in current epoch
print("Generate Batch")
batch_dataset = mylib.generate_batch_dataset(list_dataset,
config.batch_size,
seed=seeds[current_epoch])
print("Start Training")
for index in tqdm(range(last_index, len(batch_dataset))):
batch_data = batch_dataset[index]
img_ft, txt_ft, lbl = mylib.get_feature_from_batch(
batch_data,
image_folders=config.image_folders,
dictionary=my_dictionary,
resize_img=224,
max_len=32)
inputs = [
tf.convert_to_tensor(img_ft, dtype=tf.float32),
tf.convert_to_tensor(txt_ft, dtype=tf.float32)
]
loss_value, loss_vis, loss_txt, loss_rank, grads = grad(
model,
inputs,
lbl,
alpha=config.alpha,
lamb_0=lamb_0,
lamb_1=config.lamb_1,
lamb_2=config.lamb_2)
optimizer.apply_gradients(zip(grads, model.trainable_variables))
# Track mean loss in current epoch
epoch_loss_total_avg(loss_value)
epoch_loss_visual_avg(loss_vis)
epoch_loss_text_avg(loss_txt)
if config.stage_2:
epoch_loss_ranking_avg(loss_rank)
info = "Epoch {}/{}: Iter batch {}/{}\nLoss_Visual: {:.6f}\nLoss_Text: {:.6f}\nLoss_Ranking: {:.6f}\nLoss_Total: {:.6f}\n-----".format(
current_epoch + 1, config.numb_epochs, index + 1,
len(batch_dataset), epoch_loss_visual_avg.result(),
epoch_loss_text_avg.result(),
epoch_loss_ranking_avg.result(),
epoch_loss_total_avg.result())
else:
info = "Epoch {}/{}: Iter batch {}/{}\nLoss_Visual: {:.6f}\nLoss_Text: {:.6f}\nLoss_Total: {:.6f}\n-----".format(
current_epoch + 1, config.numb_epochs, index + 1,
len(batch_dataset), epoch_loss_visual_avg.result(),
epoch_loss_text_avg.result(),
epoch_loss_total_avg.result())
ckpt.iters.assign_add(1)
if (index + 1) % 20 == 0 or index <= 9:
print(info)
if (index + 1) % 20 == 0:
save_path = manager.save()
print("Saved checkpoint for epoch {} - iter {}: {}".format(
int(ckpt.epoch) + 1, int(ckpt.iters), save_path))
with open(log_filename, 'a') as f:
f.write('{}\n'.format(datetime.datetime.now().strftime(
'%Y-%m-%d %H:%M:%S.%f')))
f.write(info + '\n')
print(info)
save_path = manager.save()
print("Saved checkpoint for epoch {} - iter {}: {}".format(
int(ckpt.epoch), int(ckpt.iters), save_path))
last_index = 0
ckpt.iters = tf.Variable(0)
ckpt.epoch.assign_add(1)
def main():
# Argument parsing
# args = parse_arguments()
have_cuda = torch.cuda.is_available()
if have_cuda:
#cudnn.benchmark = True
device = torch.device('cuda:0')
else:
device = torch.device('cpu')
print(device)
if config.stage_2:
log_filename = os.path.join('report/dualpath_pytorch/',
'dualpath_pytorch_stage_2.log')
else:
log_filename = os.path.join('report/dualpath_pytorch/',
'dualpath_pytorch_stage_1.log')
if not os.path.exists('report/dualpath_pytorch/'):
os.mkdir('report/')
os.mkdir('report/dualpath_pytorch/')
images_names = list(dataset_flickr30k.keys())
index_flickr30k = [i for i in range(len(dataset_flickr30k))]
index_train, index_validate, index_test = mylib.generate_3_subsets(
index_flickr30k, ratio=[0.93, 0.035, 0.035])
print("Number of samples in 3 subsets are {}, {} and {}".format(
len(index_train), len(index_validate), len(index_test)))
images_names_train = [images_names[i] for i in index_train]
images_names_val = [images_names[i] for i in index_validate]
images_names_test = [images_names[i] for i in index_test]
list_dataset = []
img_classes = [int(x[0:-4]) for x in images_names_train]
img_classes = sorted(img_classes)
for idx, img_name in enumerate(images_names_train):
img_class = int(img_name[0:-4]) # remove '.jpg'
for des in dataset_flickr30k[img_name]:
temp = [img_name, des, img_classes.index(img_class)]
list_dataset.append(temp)
model = DualPath(nimages=len(images_names_train),
nwords=len(my_dictionary),
backbone_trainable=False,
initial_word2vec=True)
if have_cuda:
model = model.to(device)
pretrained_path = config.save_path + 'checkpoint_{}.pth'.format(
config.last_epoch)
optimizer = torch.optim.Adam(model.parameters())
if os.path.exists(pretrained_path):
print("Train from {}".format(pretrained_path))
checkpoint = torch.load(pretrained_path)
model.load_state_dict(checkpoint['model_state_dict'])
optimizer.load_state_dict(checkpoint['optimizer_state_dict'])
else:
print("Train form sratch")
model.train()
last_index = config.last_index
# Training
for current_epoch in range(config.last_epoch, config.numb_epochs):
print("Generate Batch")
batch_dataset = mylib.generate_batch_dataset(
list_dataset,
config.batch_size,
seed=config.seeds[current_epoch],
fill_remain=True)
image_loss = np.zeros(len(batch_dataset))
text_loss = np.zeros(len(batch_dataset))
total_loss = np.zeros(len(batch_dataset))
if config.stage_2:
rank_loss = np.zeros(len(batch_dataset))
lamb_2 = config.lamb_2
else:
lamb_2 = 0
print("Start Training")
for index in tqdm(range(last_index, len(batch_dataset))):
batch_data = batch_dataset[index]
img_ft, txt_ft, lbl = mylib.get_feature_from_batch(
batch_data,
image_folders=config.image_folders,
dictionary=my_dictionary,
resize=224,
max_len=32)
img_ft = torch.tensor(img_ft) # in the correct axis from transform
txt_ft = torch.tensor(txt_ft).permute(0, 3, 1, 2)
lbl = torch.tensor(lbl, dtype=torch.long)
if have_cuda:
img_ft = img_ft.to(device)
txt_ft = txt_ft.to(device)
lbl = lbl.to(device)
optimizer.zero_grad()
img_class, txt_class, fimg, ftxt = model(img_ft, txt_ft)
loss_img, loss_txt, loss_rank = loss_func(img_class,
txt_class,
lbl,
fimg,
ftxt,
lamb_0=config.lamb_0,
lamb_1=config.lamb_1,
lamb_2=lamb_2)
loss = config.lamb_0 * loss_img + config.lamb_1 * loss_txt + lamb_2 * loss_rank
loss.backward()
optimizer.step()
# Track mean loss in current epoch
image_loss[index] = loss_img
text_loss[index] = loss_txt
total_loss[index] = loss_img + loss_txt
if config.stage_2:
rank_loss[index] = loss_rank
total_loss[index] += loss_rank
info = "Epoch {}/{}: Iter batch {}/{}\nLoss_Visual: {:.6f}\nLoss_Text: {:.6f}\nLoss_Ranking: {:.6f}\nLoss_Total: {:.6f}\n-----".format(
current_epoch + 1, config.numb_epochs, index + 1,
len(batch_dataset),
np.mean(image_loss[last_index:index + 1]),
np.mean(text_loss[last_index:index + 1]),
np.mean(rank_loss[last_index:index + 1]),
np.mean(total_loss[last_index:index + 1]))
else:
info = "Epoch {}/{}: Iter batch {}/{}\nLoss_Visual: {:.6f}\nLoss_Text: {:.6f}\nLoss_Total: {:.6f}\n-----".format(
current_epoch + 1, config.numb_epochs, index + 1,
len(batch_dataset),
np.mean(image_loss[last_index:index + 1]),
np.mean(text_loss[last_index:index + 1]),
np.mean(total_loss[last_index:index + 1]))
if (index + 1) % 20 == 0 or index <= 9:
print(info)
if (index + 1) % 20 == 0:
with open(log_filename, 'a') as f:
f.write('{}\n'.format(datetime.datetime.now().strftime(
'%Y-%m-%d %H:%M:%S.%f')))
f.write(info + '\n')
print("Saving model ...")
torch.save(
{
'model_state_dict': model.state_dict(),
'optimizer_state_dict': optimizer.state_dict(),
}, config.save_path +
'checkpoint_{}.pth'.format(current_epoch))
last_index = 0
print(info)
with open(log_filename, 'a') as f:
f.write('{}\n'.format(
datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f')))
f.write(info + '\n')
# Save
torch.save(
{
'model_state_dict': model.state_dict(),
'optimizer_state_dict': optimizer.state_dict(),
}, config.save_path + 'checkpoint_{}.pth'.format(current_epoch))
def main():
# Argument parsing
args = parse_arguments()
top_k_str = args.top_k
top_k_str = top_k_str.replace(" ", "")
top_k_str = top_k_str.split(",")
top_k = [int(x) for x in top_k_str]
start_epoch = args.start_epoch
if args.end_epoch == 0 or args.end_epoch < start_epoch:
end_epoch = start_epoch
else:
end_epoch = args.end_epoch
info = "Evaluation Recall from epoch {} to {} with top_k = [".format(start_epoch, end_epoch)
for k in top_k:
info += "{},".format(k)
info = info[0:-1] + "]"
print("---------------\n\n" + info + "\n\n---------------")
log_filename = os.path.join('report_sgd/','stage_1_recall_train.log')
if not os.path.exists('report_sgd'):
os.mkdir('report_sgd')
images_names = list(dataset_flickr30k.keys())
index_flickr30k = [i for i in range(len(dataset_flickr30k))]
index_train, index_validate, index_test = mylib.generate_3_subsets(index_flickr30k, ratio = [0.93, 0.035, 0.035])
print("Number of samples in 3 subsets are {}, {} and {}". format(len(index_train), len(index_validate), len(index_test)))
images_names_train = [images_names[i] for i in index_train]
images_names_val = [images_names[i] for i in index_validate]
images_names_test = [images_names[i] for i in index_test]
list_dataset = []
all_class = [x[0:-4] for x in images_names_train]
all_class_sort = sorted(all_class)
for idx, img_name in enumerate(images_names_train):
img_class = all_class_sort.index(img_name[0:-4])
for des in dataset_flickr30k[img_name]:
temp = [img_name, des, img_class]
list_dataset.append(temp)
list_img_names = images_names_val
images_class = [x[2] for x in list_dataset]
list_text = [x[1] for x in list_dataset]
text_class = images_class
for epoch in range(start_epoch, end_epoch+1):
print("Calculating Recall in epoch {}". format(epoch))
pretrained_model = 'weights/checkpoints_' + str(epoch) + '/my_checkpoint'
model = create_model(nclass=len(images_names_train), nword=len(my_dictionary), ft_resnet=False)
model.load_weights(pretrained_model)
img_branch, txt_branch = get_branches_from_model(model)
images_out = get_img_ft_from_branch(img_branch, list_img_names)
text_out = get_txt_ft_from_branch(txt_branch, list_text)
a, b = recall_at_k_faster(images_out, images_class, text_out, text_class, top_k)
info = 'Epoch {}\n'.format(epoch)
for k_idx, k_value in enumerate(top_k):
info += "Recall@{}_Image_Query: {:6f}\nRecall@{}_Text_Query: {:6f}\n".format(k_value, np.mean(a[k_idx]), k_value, np.mean(b[k_idx]))
info += '---------------\n'
print("Writing to report file")
with open(log_filename, 'a') as f:
f.write(info)
def main():
num_gpu = int(mirrored_strategy.num_replicas_in_sync)
global_batch_size = config.batch_size * num_gpu
@tf.function
def train_step(dist_img, dist_txt, dist_lbl):
def step_fn(img, txt, lbl):
inputs = [
tf.dtypes.cast(img, tf.float32),
tf.dtypes.cast(txt, tf.float32)
]
with tf.GradientTape() as tape:
loss, loss_vis, loss_text, ranking_loss = total_loss_distributed(
model, inputs, lbl, config.alpha, config.lamb_0,
config.lamb_1, config.lamb_2)
loss = loss * (1.0 / global_batch_size)
loss_vis = loss_vis * (1.0 / global_batch_size)
loss_text = loss_text * (1.0 / global_batch_size)
ranking_loss = ranking_loss * (1.0 / global_batch_size)
grads = tape.gradient(loss, model.trainable_variables)
optimizer.apply_gradients(
list(zip(grads, model.trainable_variables)))
return loss, loss_vis, loss_text, ranking_loss
per_loss, per_loss_vis, per_loss_text, per_ranking_loss = mirrored_strategy.experimental_run_v2(
step_fn, args=(
dist_img,
dist_txt,
dist_lbl,
))
print(per_loss)
mean_loss = mirrored_strategy.reduce(
tf.distribute.ReduceOp.SUM, per_loss, axis=0) / global_batch_size
mean_loss_vis = mirrored_strategy.reduce(tf.distribute.ReduceOp.SUM,
per_loss_vis,
axis=0) / global_batch_size
mean_loss_text = mirrored_strategy.reduce(tf.distribute.ReduceOp.SUM,
per_loss_text,
axis=0) / global_batch_size
mean_ranking_loss = mirrored_strategy.reduce(
tf.distribute.ReduceOp.SUM, per_ranking_loss,
axis=0) / global_batch_size
return mean_loss, mean_loss_vis, mean_loss_text, mean_ranking_loss
args = parse_arguments()
save_path = 'checkpoints_distributed/' # directory path to save checkpoint
if config.stage_2:
log_filename = os.path.join('report_distributed/',
'dualpath_stage_2.log')
else:
log_filename = os.path.join('report_distributed',
'dualpath_stage_1.log')
if args.from_scratch:
try:
shutil.rmtree('report_distributed')
shutil.rmtree('checkpoints_distributed')
except:
pass
if not os.path.exists(save_path):
os.mkdir(save_path)
if not os.path.exists('report_distributed/'):
os.makedirs('report_distributed/')
images_names = list(dataset_flickr30k.keys())
index_flickr30k = [i for i in range(len(dataset_flickr30k))]
index_train, index_validate, index_test = mylib.generate_3_subsets(
index_flickr30k, ratio=[0.93, 0.035, 0.035])
print("Number of samples in 3 subsets are {}, {} and {}".format(
len(index_train), len(index_validate), len(index_test)))
images_names_train = [images_names[i] for i in index_train]
images_names_val = [images_names[i] for i in index_validate]
images_names_test = [images_names[i] for i in index_test]
list_dataset = []
all_class = [x[0:-4] for x in images_names_val]
all_class_sort = sorted(all_class)
for idx, img_name in enumerate(images_names_val):
img_class = all_class_sort.index(img_name[0:-4])
for des in dataset_flickr30k[img_name]:
temp = [img_name, des, img_class]
list_dataset.append(temp)
seeds = [x for x in range(config.numb_epochs)]
#print("Running on " + config_gpu)
with mirrored_strategy.scope():
model = create_model(nclass=len(images_names_val),
nword=len(my_dictionary),
ft_resnet=False)
optimizer = keras.optimizers.SGD(learning_rate=0.001,
momentum=0.9,
nesterov=True)
ckpt = tf.train.Checkpoint(iters=tf.Variable(0),
epoch=tf.Variable(0),
optimizer=optimizer,
model=model)
manager = tf.train.CheckpointManager(ckpt, save_path, max_to_keep=1)
if args.from_scratch == False:
ckpt.restore(manager.latest_checkpoint)
if manager.latest_checkpoint:
print("Restored from {}".format(manager.latest_checkpoint))
else:
print("Initializing from scratch.")
else:
print("Force to initializing from scratch.")
last_epoch = int(ckpt.epoch)
last_index = int(ckpt.iters)
for current_epoch in range(last_epoch, config.numb_epochs):
with tf.device('/device:CPU:0'):
epoch_loss_total_avg = tf.keras.metrics.Mean(
) # track mean loss in current epoch
epoch_loss_visual_avg = tf.keras.metrics.Mean(
) # track mean loss in current epoch
epoch_loss_text_avg = tf.keras.metrics.Mean(
) # track mean loss in current epoch
if config.stage_2:
epoch_loss_ranking_avg = tf.keras.metrics.Mean(
) # track mean loss in current epoch
print("Generate Batch")
batch_dataset = mylib.generate_batch_dataset(list_dataset,
config.batch_size,
seed=seeds[current_epoch])
n_samples = len(batch_dataset)
print("Start Training")
index = 0
last_index = 0
for index in range(last_index, n_samples, num_gpu):
with tf.device('/device:CPU:0'):
for idx_gpu in range(num_gpu):
batch_data = batch_dataset[index + idx_gpu]
img_ft, txt_ft, lbl = mylib.get_feature_from_batch(
batch_data,
image_folders=config.image_folders,
dictionary=my_dictionary,
resize_img=224,
max_len=32)
if idx_gpu == 0:
total_img_ft = img_ft
total_txt_ft = txt_ft
total_lbl = lbl
else:
total_img_ft = np.concatenate((total_img_ft, img_ft))
total_txt_ft = np.concatenate((total_txt_ft, txt_ft))
total_lbl = np.concatenate((total_lbl, lbl))
print(
f"{total_img_ft.shape} --- {total_txt_ft.shape} --- {total_lbl.shape}"
)
n_img = total_img_ft.shape[0]
n_txt = total_txt_ft.shape[0]
n_lbl = total_lbl.shape[0]
total_img_ft = tf.data.Dataset.from_tensor_slices(
total_img_ft).batch(n_img)
total_txt_ft = tf.data.Dataset.from_tensor_slices(
total_txt_ft).batch(n_txt)
total_lbl = tf.data.Dataset.from_tensor_slices(
total_lbl).batch(n_lbl)
total_img_ft = mirrored_strategy.experimental_distribute_dataset(
total_img_ft)
total_txt_ft = mirrored_strategy.experimental_distribute_dataset(
total_txt_ft)
total_lbl = mirrored_strategy.experimental_distribute_dataset(
total_lbl)
with mirrored_strategy.scope():
for img_ft, txt_ft, lbl in zip(total_img_ft, total_txt_ft,
total_lbl):
loss_value, loss_vis, loss_txt, loss_rank = train_step(
img_ft, txt_ft, lbl)
# Track mean loss in current epoch
epoch_loss_total_avg(loss_value)
epoch_loss_visual_avg(loss_vis)
epoch_loss_text_avg(loss_txt)
if config.stage_2:
epoch_loss_ranking_avg(loss_rank)
info = "Epoch {}/{}: Iter batch {}/{}\nLoss_Visual: {:.6f}\nLoss_Text: {:.6f}\nLoss_Ranking: {:.6f}\nLoss_Total: {:.6f}\n-----".format(
current_epoch + 1, config.numb_epochs, index + 1,
n_samples, epoch_loss_visual_avg.result(),
epoch_loss_text_avg.result(),
epoch_loss_ranking_avg.result(),
epoch_loss_total_avg.result())
else:
info = "Epoch {}/{}: Iter batch {}/{}\nLoss_Visual: {:.6f}\nLoss_Text: {:.6f}\nLoss_Total: {:.6f}\n-----".format(
current_epoch + 1, config.numb_epochs, index + 1,
n_samples / num_gpu, epoch_loss_visual_avg.result(),
epoch_loss_text_avg.result(),
epoch_loss_total_avg.result())
ckpt.iters.assign_add(2)
if (index + 1) % 20 == 0:
save_path = manager.save()
print("Saved checkpoint for epoch {} - iter {}: {}".format(
int(ckpt.epoch) + 1, int(ckpt.iters), save_path))
with open(log_filename, 'a') as f:
f.write('{}\n'.format(datetime.datetime.now().strftime(
'%Y-%m-%d %H:%M:%S.%f')))
f.write(info + '\n')
print(info)
print(info)
save_path = manager.save()
print("Saved checkpoint for epoch {} - iter {}: {}".format(
int(ckpt.epoch), int(ckpt.iters), save_path))
last_index = 0
ckpt.iters = tf.Variable(0)
ckpt.epoch.assign_add(1)
def main():
# Argument parsing
# args = parse_arguments()
log_filename = os.path.join('report/version_4/', 'dualpath_v4_stage_1.log')
if not os.path.exists('report/version_4/'):
os.mkdir('report/version_4/')
images_names = list(dataset_flickr30k.keys())
index_flickr30k = [i for i in range(len(dataset_flickr30k))]
index_train, index_validate, index_test = mylib.generate_3_subsets(
index_flickr30k, ratio=[0.93, 0.035, 0.035])
print("Number of samples in 3 subsets are {}, {} and {}".format(
len(index_train), len(index_validate), len(index_test)))
images_names_train = [images_names[i] for i in index_train]
images_names_val = [images_names[i] for i in index_validate]
images_names_test = [images_names[i] for i in index_test]
list_dataset = []
all_class = []
for idx, img_name in enumerate(images_names_train):
img_class = img_name[0:-4] # remove '.jpg'
all_class += [img_class]
for des in dataset_flickr30k[img_name]:
temp = [img_name, des, img_class]
list_dataset.append(temp)
# Restore the weights
if config.last_index == 0:
if config.last_epoch == 0:
pretrained_model = ''
else:
pretrained_model = './checkpoints_v4_' + str(config.last_epoch -
1) + '/my_checkpoint'
else:
pretrained_model = './checkpoints_v4_' + str(
config.last_epoch) + '/my_checkpoint'
model = create_model(dict_length=len(my_dictionary),
pretrained_model=pretrained_model)
seeds = [1509 + x
for x in range(config.numb_epochs)] # Since stage 1 have 50 epoch
if config.stage_2: # parameter for ranking loss (stage 2)
lamb_0 = config.lamb_0
else:
lamb_0 = 0
last_index = config.last_index
# Adadelta
optimizer = keras.optimizers.Adadelta()
for current_epoch in range(config.last_epoch, config.numb_epochs):
epoch_loss_total_avg = tf.keras.metrics.Mean(
) # track mean loss in current epoch
epoch_loss_classify_avg = tf.keras.metrics.Mean(
) # track mean loss in current epoch
if config.stage_2:
epoch_loss_ranking_avg = tf.keras.metrics.Mean(
) # track mean loss in current epoch
print("Generate Batch")
batch_dataset = mylib.generate_batch_dataset_v4(
list_dataset, config.batch_size, seed=seeds[current_epoch])
if config.decay_lr and current_epoch <= 10:
learnRate = max(config.decay_lr_min,
((1 - config.decay_lr_portion)**current_epoch) *
config.learn_rate)
else:
if config.decay_lr:
learnRate = config.decay_lr_min
else:
learnRate = config.learn_rate
# SGD
#optimizer = keras.optimizers.SGD(learning_rate=learnRate,
#momentum=config.moment_val)
print("Start Training")
for index in tqdm(range(last_index, len(batch_dataset))):
batch_data = batch_dataset[index]
img_ft, txt_ft, lbl = mylib.get_feature_from_batch_v4(
batch_data,
image_folders=config.image_folders,
dictionary=my_dictionary,
resize_img=224,
max_len=32)
inputs = [
tf.convert_to_tensor(img_ft, dtype=tf.float32),
tf.convert_to_tensor(txt_ft, dtype=tf.float32)
]
loss_value, loss_classify, loss_rank, grads = grad(
model,
inputs,
lbl,
alpha=config.alpha,
lamb_0=lamb_0,
lamb_1=config.lamb_1)
optimizer.apply_gradients(zip(grads, model.trainable_variables))
# Track mean loss in current epoch
epoch_loss_total_avg(loss_value)
epoch_loss_classify_avg(loss_classify)
if config.stage_2:
epoch_loss_ranking_avg(loss_rank)
if config.stage_2:
info = "Epoch {}/{}: Iter batch {}/{}\nLoss_Classify: {:.6f}\nLoss_Ranking: {:.6f}\nLoss_Total: {:.6f}\n-----".format(
current_epoch + 1, config.numb_epochs, index + 1,
len(batch_dataset), epoch_loss_classify_avg.result(),
epoch_loss_ranking_avg.result(),
epoch_loss_total_avg.result())
else:
info = "Epoch {}/{}: Iter batch {}/{}\nLoss_Classify: {:.6f}\nLoss_Total: {:.6f}\n-----".format(
current_epoch + 1, config.numb_epochs, index + 1,
len(batch_dataset), epoch_loss_classify_avg.result(),
epoch_loss_total_avg.result())
if (index + 1) % 20 == 0 or index <= 9:
print(info)
if (index + 1) % 20 == 0:
with open(log_filename, 'a') as f:
f.write('{}\n'.format(datetime.datetime.now().strftime(
'%Y-%m-%d %H:%M:%S.%f')))
f.write(info)
print("Saving weights ...")
model.save_weights('./checkpoints_v4_' + str(current_epoch) +
'/my_checkpoint')
#dualpath_model.save('./checkpoints_model/my_model.h5')
last_index = 0
print(info)
print("Saving weights ...")
model.save_weights('./checkpoints_v4_' + str(current_epoch) +
'/my_checkpoint')
def main():
args = parse_arguments()
save_path = 'checkpoints_sgd/' # directory path to save checkpoint
save_weights = 'weights/'
if config.stage_2:
log_filename = os.path.join('report_sgd/dualpath_sgd/',
'dualpath_stage_2.log')
else:
log_filename = os.path.join('report_sgd/dualpath_sgd/',
'dualpath_stage_1.log')
if args.from_scratch:
try:
shutil.rmtree('report_sgd')
shutil.rmtree('checkpoints_sgd')
except:
pass
if not os.path.exists(save_path):
os.mkdir(save_path)
if not os.path.exists(save_weights):
os.mkdir(save_weights)
if not os.path.exists('report_sgd/dualpath_sgd/'):
os.makedirs('report_sgd/dualpath_sgd/')
images_names = list(dataset_flickr30k.keys())
index_flickr30k = [i for i in range(len(dataset_flickr30k))]
index_train, index_validate, index_test = mylib.generate_3_subsets(
index_flickr30k, ratio=[0.93, 0.035, 0.035])
print("Number of samples in 3 subsets are {}, {} and {}".format(
len(index_train), len(index_validate), len(index_test)))
images_names_train = [images_names[i] for i in index_train]
images_names_val = [images_names[i] for i in index_validate]
images_names_test = [images_names[i] for i in index_test]
'''
list_dataset = []
all_class = []
for idx, img_name in enumerate(images_names_train):
img_class = img_name[0:-4] # remove '.jpg'
all_class += [img_class]
for des in dataset_flickr30k[img_name]:
temp = [img_name, des, img_class]
list_dataset.append(temp)
label_encoder, onehot_all_class = mylib.create_onehot_all_label(all_class)
'''
list_dataset = []
all_class = [x[0:-4] for x in images_names_train]
all_class_sort = sorted(all_class)
for idx, img_name in enumerate(images_names_train):
img_class = all_class_sort.index(img_name[0:-4])
for des in dataset_flickr30k[img_name]:
temp = [img_name, des, img_class]
list_dataset.append(temp)
seeds = [x for x in range(config.numb_epochs)]
gpu_index = args.gpu_index
if gpu_index == -1:
config_gpu = '/device:CPU:0'
else:
config_gpu = '/device:GPU:{}'.format(gpu_index)
print("Running on " + config_gpu)
try:
with tf.device(config_gpu):
model = create_model(nclass=len(images_names_train),
nword=len(my_dictionary),
ft_resnet=False)
#optimizer = keras.optimizers.SGD(learning_rate=0.001, momentum=0.9, nesterov=True)
optimizer = keras.optimizers.SGD(learning_rate=0.001, momentum=0.9)
ckpt = tf.train.Checkpoint(iters=tf.Variable(0),
epoch=tf.Variable(0),
optimizer=optimizer,
model=model)
manager = tf.train.CheckpointManager(ckpt,
save_path,
max_to_keep=1)
if args.from_scratch == False:
ckpt.restore(manager.latest_checkpoint)
if manager.latest_checkpoint:
print("Restored from {}".format(manager.latest_checkpoint))
else:
print("Initializing from scratch.")
else:
print("Force to initializing from scratch.")
last_epoch = int(ckpt.epoch)
last_index = int(ckpt.iters)
for current_epoch in range(last_epoch, config.numb_epochs):
epoch_loss_total_avg = tf.keras.metrics.Mean(
) # track mean loss in current epoch
epoch_loss_visual_avg = tf.keras.metrics.Mean(
) # track mean loss in current epoch
epoch_loss_text_avg = tf.keras.metrics.Mean(
) # track mean loss in current epoch
if config.stage_2:
epoch_loss_ranking_avg = tf.keras.metrics.Mean(
) # track mean loss in current epoch
with tf.device('/device:CPU:0'):
batch_dataset = mylib.generate_batch_dataset(
list_dataset,
config.batch_size,
seed=seeds[current_epoch])
print("Start Training")
for index in tqdm(range(last_index, len(batch_dataset))):
batch_data = batch_dataset[index]
with tf.device('/device:CPU:0'):
img_ft, txt_ft, lbl = mylib.get_feature_from_batch(
batch_data,
image_folders=config.image_folders,
dictionary=my_dictionary,
resize_img=224,
max_len=32,
prob_image=args.prob_aug_img)
inputs = [
tf.convert_to_tensor(img_ft, dtype=tf.float32),
tf.convert_to_tensor(txt_ft, dtype=tf.float32)
]
loss_value, loss_vis, loss_txt, loss_rank, grads = grad(
model,
inputs,
lbl,
alpha=config.alpha,
lamb_0=config.lamb_0,
lamb_1=config.lamb_1,
lamb_2=config.lamb_2)
optimizer.apply_gradients(
zip(grads, model.trainable_variables))
# Track mean loss in current epoch
epoch_loss_total_avg(loss_value)
epoch_loss_visual_avg(loss_vis)
epoch_loss_text_avg(loss_txt)
if config.stage_2:
epoch_loss_ranking_avg(loss_rank)
info = "Epoch {}/{}: Iter batch {}/{}\nLoss_Visual: {:.6f}\nLoss_Text: {:.6f}\nLoss_Ranking: {:.6f}\nLoss_Total: {:.6f}\n-----".format(
current_epoch + 1, config.numb_epochs, index + 1,
len(batch_dataset), epoch_loss_visual_avg.result(),
epoch_loss_text_avg.result(),
epoch_loss_ranking_avg.result(),
epoch_loss_total_avg.result())
else:
info = "Epoch {}/{}: Iter batch {}/{}\nLoss_Visual: {:.6f}\nLoss_Text: {:.6f}\nLoss_Total: {:.6f}\n-----".format(
current_epoch + 1, config.numb_epochs, index + 1,
len(batch_dataset), epoch_loss_visual_avg.result(),
epoch_loss_text_avg.result(),
epoch_loss_total_avg.result())
ckpt.iters.assign_add(1)
if (index + 1) % 20 == 0:
save_path = manager.save()
print("Saved checkpoint for epoch {} - iter {}: {}".
format(
int(ckpt.epoch) + 1, int(ckpt.iters),
save_path))
with open(log_filename, 'a') as f:
f.write('{}\n'.format(
datetime.datetime.now().strftime(
'%Y-%m-%d %H:%M:%S.%f')))
f.write(info + '\n')
if (index + 1) % 20 == 0 or index <= 9:
print(info)
print(info)
with open(log_filename, 'a') as f:
f.write('{}\n'.format(datetime.datetime.now().strftime(
'%Y-%m-%d %H:%M:%S.%f')))
f.write(info + '\n')
save_path = manager.save()
print("Saved checkpoint for epoch {} - iter {}: {}".format(
int(ckpt.epoch), int(ckpt.iters), save_path))
model.save_weights(save_weights + '/checkpoints_' +
str(current_epoch + 1) + '/my_checkpoint')
last_index = 0
ckpt.iters = tf.Variable(0)
ckpt.epoch.assign_add(1)
except RuntimeError as e:
print(e)