def single_palette_color_transfer(pixel_color, ori_color, modified_color):
def get_boundary(ori, direct, k_min, k_max, max_iter=20):
start = ori + k_min * direct
end = ori + k_max * direct
for _ in range(max_iter):
mid = (start + end) / 2
if ValidLAB(mid) and ValidRGB(LABtoRGB(mid)):
start = mid
else:
end = mid
return (start + end) / 2
pixel_color = np.array(pixel_color)
ori_color = np.array(ori_color)
modified_color = np.array(modified_color)
offset = modified_color - ori_color
c_boundary = get_boundary(ori_color, offset, 1, 255)
lab = pixel_color + offset
if ValidLAB(lab) and ValidRGB(LABtoRGB(lab)):
x_boundary = get_boundary(pixel_color, offset, 1, 255)
else:
x_boundary = get_boundary(modified_color, pixel_color - ori_color, 0, 1)
if distance(x_boundary, pixel_color) == 0: return pixel_color
if distance(c_boundary, ori_color) == 0:
ratio = 1
else:
ratio = min(1, (distance(x_boundary, pixel_color) / distance(c_boundary, ori_color)))
res = pixel_color + ((x_boundary - pixel_color) / distance(x_boundary, pixel_color) * distance(modified_color,
ori_color) * ratio)
return res
def validate(tcn, use_cuda, args):
# Run model on validation data and log results
data_loader = DataLoader(
validation_set,
batch_size=32,
shuffle=False,
pin_memory=use_cuda,
)
correct_with_margin = 0
correct_without_margin = 0
losses = []
for frames, features in data_loader:
# frames = Variable(minibatch, require_grad=False)
if use_cuda:
frames = frames.cuda()
features = features.cuda()
anchor_frames = frames[:, 0, :, :, :]
positive_frames = frames[:, 1, :, :, :]
negative_frames = frames[:, 2, :, :, :]
anchor_features = features[:, 0, :, :, :]
positive_features = features[:, 1, :, :, :]
anchor_frames = frames[:, 0, :, :, :]
positive_frames = frames[:, 1, :, :, :]
negative_frames = frames[:, 2, :, :, :]
anchor_features = features[:, 0, :, :, :]
positive_features = features[:, 1, :, :, :]
negative_features = features[:, 2, :, :, :]
# anchor_output, unnormalized, _ = tcn(anchor_features)
# positive_output, _, _ = tcn(positive_features)
# negative_output, _, _ = tcn(negative_features)
anchor_output, unnormalized, _ = tcn(anchor_frames)
positive_output, _, _ = tcn(positive_frames)
negative_output, _, _ = tcn(negative_frames)
d_positive = distance(anchor_output, positive_output)
d_negative = distance(anchor_output, negative_output)
assert(d_positive.size()[0] == frames.size()[0])
correct_with_margin += ((d_positive + args.margin) < d_negative).data.cpu().numpy().sum()
correct_without_margin += (d_positive < d_negative).data.cpu().numpy().sum()
loss_triplet = torch.clamp(args.margin + d_positive - d_negative, min=0.0).mean()
loss = loss_triplet
losses.append(loss.data.cpu().numpy())
loss = np.mean(losses)
logger.info('val loss: ',loss)
message = "Validation score correct with margin {with_margin}/{total} and without margin {without_margin}/{total}".format(
with_margin=correct_with_margin,
without_margin=correct_without_margin,
total=len(validation_set)
)
logger.info(message)
return correct_with_margin, correct_without_margin, loss
def get_weights(pixel_color, ori_palette):
dist = []
for p1, p2 in itertools.combinations(ori_palette, 2):
dist.append(distance(p1, p2))
mean_dist = sum(dist) / len(dist)
def gaussian(a, b):
r = distance(a, b)
return math.exp(-(r ** 2) / (2 * (mean_dist ** 2)))
palette_cnt = len(ori_palette)
p_matrix = np.zeros((palette_cnt, palette_cnt), dtype='float64')
for i in range(palette_cnt):
for j in range(palette_cnt):
p_matrix[i, j] = gaussian(ori_palette[j], ori_palette[i])
p_matrix = np.linalg.inv(p_matrix)
lamda = []
for i in range(palette_cnt):
ans = np.zeros(palette_cnt)
ans[i] = 1
lamda.append(np.dot(ans, p_matrix))
weights = np.zeros(palette_cnt)
for i in range(palette_cnt):
for j in range(palette_cnt):
weights[i] += lamda[i][j] * gaussian(pixel_color, ori_palette[j])
weights = [w if w > 0 else 0 for w in weights]
weights /= np.sum(weights)
return weights
def k_means(bins, k, init_mean=True, max_iter=1000, black=True):
if init_mean is False: means = random.sample(list(bins),k)
else: means = init_means(bins, k)
if black: means.append([0, 128, 128])
means = np.array(means)
mean_cnt = means.shape[0]
cluster_cnt = np.zeros(mean_cnt)
for _ in range(max_iter):
# print('\niter %d...' % _)
cluster_sum = [np.array([0,0,0],dtype=float) for i in range(mean_cnt)]
cluster_cnt = np.zeros(mean_cnt)
for color, cnt in bins.items():
color = np.array(color)
dists = [distance(color,mean) for mean in means]
cluster_th = dists.index(min(dists))
cluster_sum[cluster_th] += color * cnt
cluster_cnt[cluster_th] += cnt
new_means = [cluster_sum[i] / cluster_cnt[i] if cluster_cnt[i] > 0 else [0,0,0] for i in range(k)]
if black: new_means.append([0,128,128])
new_means = np.array(new_means)
if (new_means == means).all(): break
else: means = new_means
# print(means)
#按照亮度输出
arg_th = np.argsort(means[:k], axis=0)[:,0][::-1]
return means[arg_th], cluster_cnt[arg_th]
def validate(tcn, decoder, use_cuda, args):
# Run model on validation data and log results
data_loader = DataLoader(validation_set,
batch_size=64,
shuffle=False,
pin_memory=use_cuda,
collate_fn=collate_fn)
correct_with_margin = 0
correct_without_margin = 0
for minibatch, captions, lengths in data_loader:
frames = Variable(minibatch, volatile=True)
if use_cuda:
frames = frames.cuda()
anchor_frames = frames[:, 0, :, :, :]
positive_frames = frames[:, 1, :, :, :]
negative_frames = frames[:, 2, :, :, :]
anchor_output, unnormalized = tcn(anchor_frames)
positive_output, _ = tcn(positive_frames)
negative_output, _ = tcn(negative_frames)
d_positive = distance(anchor_output, positive_output)
d_negative = distance(anchor_output, negative_output)
caption_outputs = decoder(unnormalized, captions, lengths)
assert (d_positive.size()[0] == minibatch.size()[0])
correct_with_margin += ((d_positive + args.margin) <
d_negative).data.cpu().numpy().sum()
correct_without_margin += (d_positive <
d_negative).data.cpu().numpy().sum()
message = "Validation score correct with margin {with_margin}/{total} and without margin {without_margin}/{total}".format(
with_margin=correct_with_margin,
without_margin=correct_without_margin,
total=len(validation_set))
logger.info(message)
def match(face):
# tic = time.time()
dis_map = {}
face = cv2.resize(face, (112, 112))
vector = insightface_API.extract(face)
i = 0
for v in vectors:
if names[i] in dis_map.keys():
dis_map[names[i]].append(distance(v, vector, distance_metric)[0])
else:
dis_map[names[i]] = [distance(v, vector, distance_metric)[0]]
i += 1
for key in dis_map.keys():
dis_map[key] = np.mean(sorted(dis_map[key])[:3])
top3 = sorted(dis_map.items(), key=lambda item: item[1])[:3]
result = {"top3": top3, "prob": top3[0][1]}
result["name"] = "陌生人"
if top3[0][1] < threshold_extract:
result["name"] = top3[0][0]
print("match result:", result)
# toc = time.time()
# print("match time:{}".format(toc - tic))
return result
def main():
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
use_cuda = torch.cuda.is_available()
tcn = create_model(use_cuda)
tcn = torch.nn.DataParallel(tcn, device_ids=range(torch.cuda.device_count()))
triplet_builder = builder(args.n_views, \
args.train_directory, args.train_directory_depth, IMAGE_SIZE, args, sample_size=50)
queue = multiprocessing.Queue(1)
dataset_builder_process = multiprocessing.Process(target=build_set, args=(queue, triplet_builder, logger), daemon=True)
dataset_builder_process.start()
optimizer = optim.SGD(tcn.parameters(), lr=args.lr_start, momentum=0.9)
# This will diminish the learning rate at the milestones.
# 0.1, 0.01, 0.001
learning_rate_scheduler = lr_scheduler.MultiStepLR(optimizer, milestones=[30, 50, 100], gamma=0.5)
criterion = nn.CrossEntropyLoss()
trn_losses_ = []
val_losses_= []
val_acc_margin_ = []
val_acc_no_margin_ = []
for epoch in range(args.start_epoch, args.start_epoch + args.epochs):
print("=" * 20)
logger.info("Starting epoch: {0} learning rate: {1}".format(epoch,
learning_rate_scheduler.get_lr()))
learning_rate_scheduler.step()
dataset = queue.get()
data_loader = DataLoader(
dataset=dataset,
batch_size=args.minibatch_size, # batch_size(epoch, args.max_minibatch_size),
shuffle=True,
pin_memory=use_cuda,
)
for _ in range(0, ITERATE_OVER_TRIPLETS):
losses = []
for frames, features in data_loader:
# frames = Variable(minibatch)
if use_cuda:
frames = frames.cuda()
features = features.cuda()
anchor_frames = frames[:, 0, :, :, :]
positive_frames = frames[:, 1, :, :, :]
negative_frames = frames[:, 2, :, :, :]
anchor_features = features[:, 0, :, :, :]
positive_features = features[:, 1, :, :, :]
negative_features = features[:, 2, :, :, :]
anchor_output, unnormalized, _ = tcn(anchor_frames, anchor_features)
positive_output, _, _ = tcn(positive_frames, positive_features)
negative_output, _, _ = tcn(negative_frames, negative_features)
d_positive = distance(anchor_output, positive_output)
d_negative = distance(anchor_output, negative_output)
loss_triplet = torch.clamp(args.margin + d_positive - d_negative, min=0.0).mean()
loss = loss_triplet
losses.append(loss.data.cpu().numpy())
optimizer.zero_grad()
loss.backward()
optimizer.step()
trn_losses_.append(np.mean(losses))
logger.info('train loss: ', np.mean(losses))
if epoch % 1 == 0:
acc_margin, acc_no_margin, loss = validate(tcn, use_cuda, args)
val_losses_.append(loss)
val_acc_margin_.append(acc_margin)
val_acc_no_margin_.append(acc_no_margin)
if epoch % args.save_every == 0 and epoch != 0:
logger.info('Saving model.')
save_model(tcn, model_filename(args.model_name, epoch), args.model_folder)
plot_mean(trn_losses_, args.model_folder, 'train_loss')
plot_mean(val_losses_, args.model_folder, 'validation_loss')
# plot_mean(train_acc_, args.model_folder, 'train_acc')
plot_mean(val_acc_margin_, args.model_folder, 'validation_accuracy_margin')
plot_mean(val_acc_no_margin_, args.model_folder, 'validation_accuracy_no_margin')
def gaussian(a, b):
r = distance(a, b)
return math.exp(-(r ** 2) / (2 * (mean_dist ** 2)))
def validate(tcn, attribute_classifier, criterion, use_cuda, args):
# Run model on validation data and log results
data_loader = DataLoader(
validation_set,
batch_size=32,
shuffle=False,
pin_memory=use_cuda,
)
correct_with_margin = 0
correct_without_margin = 0
losses = []
for frames, captions in data_loader:
# frames = Variable(minibatch, require_grad=False)
if use_cuda:
frames = frames.cuda()
captions = captions.cuda()
anchor_frames = frames[:, 0, :, :, :]
positive_frames = frames[:, 1, :, :, :]
negative_frames = frames[:, 2, :, :, :]
anchor_output, unnormalized, _ = tcn(anchor_frames)
positive_output, _, _ = tcn(positive_frames)
negative_output, _, _ = tcn(negative_frames)
d_positive = distance(anchor_output, positive_output)
d_negative = distance(anchor_output, negative_output)
assert (d_positive.size()[0] == minibatch.size()[0])
correct_with_margin += ((d_positive + args.margin) <
d_negative).data.cpu().numpy().sum()
correct_without_margin += (d_positive <
d_negative).data.cpu().numpy().sum()
loss_triplet = torch.clamp(args.margin + d_positive - d_negative,
min=0.0).mean()
loss = loss_triplet
losses.append(loss.data.cpu().numpy())
label_outputs_1, label_outputs_2 = attribute_classifier(mixed)
labels_1 = captions[:, 0]
# labels_2 = captions[:, 1]
loss_1 = criterion(label_outputs_1, labels_1)
# loss_2 = criterion(label_outputs_2, labels_2)
loss_language = loss_1 #+ loss_2
loss = loss_triplet + args.alpha * loss_language
# loss = loss_language
losses.append(loss.data.cpu().numpy())
_, predicted_1 = torch.max(label_outputs_1.data, 1)
# _, predicted_2 = torch.max(label_outputs_2.data, 1)
total_1 += labels_1.size(0)
torch.LongTensor(10).random_(0, 2)
# total_2 += labels_2.size(0)
correct_1 += (predicted_1 == labels_1).sum().item()
# correct_2 += (predicted_2 == labels_2).sum().item()
length_set += len(anchor_frames)
# print("predicted_1: ", predicted_1)
# print("labels_1: ",labels_1)
# print("predicted_2: ",predicted_2)
# print("labels_2: ", labels_2)
# print('='*10)
predicted_1_rand = torch.LongTensor(predicted_1.size()).random_(
0, 2).to(device)
correct_1_random += (predicted_1_rand == labels_1).sum().item()
print('Accuracy of active label network branch: {} %'.format(
100 * correct_1 / total_1))
print('Accuracy of active label network random: {} %'.format(
100 * correct_1_random / total_1))
loss = np.mean(losses)
logger.info('val loss: ', loss)
message = "Validation score correct with margin {with_margin}/{total} and without margin {without_margin}/{total}".format(
with_margin=correct_with_margin,
without_margin=correct_without_margin,
total=len(validation_set))
logger.info(message)
return correct_with_margin, correct_without_margin, loss
def validate(tcn, attribute_classifier, criterion, use_cuda, args):
# Run model on validation data and log results
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
data_loader = DataLoader(
validation_set,
batch_size=1,
shuffle=False,
pin_memory=use_cuda,
)
correct_with_margin = 0
correct_without_margin = 0
losses = []
length_set = 0
with torch.no_grad():
correct_1 = 0
total_1 = 0
correct_2 = 0
total_2 = 0
for frames, caption, seq_idx in data_loader:
if use_cuda:
frames = frames[0].to(device)
captions = caption[0].to(device)
seq_idx = seq_idx[0]
snaps = validation_set.get_videos(int(seq_idx[0]) * args.n_views)
all_sel_imgs = []
sel_imgs = snaps[0][::20]
all_sel_imgs.append(sel_imgs)
# all_sel_imgs = np.squeeze(np.asarray(all_sel_imgs)).reshape([sel_imgs.shape[0], 3, 299, 299])
_, unnorm, all_sel_imgs_emb = tcn(torch.FloatTensor(sel_imgs).to(device))
all_sel_imgs_emb = torch.mean(all_sel_imgs_emb, dim=0, keepdim=True)
anchor_frames = frames[:, 0, :, :, :]
positive_frames = frames[:, 1, :, :, :]
negative_frames = frames[:, 2, :, :, :]
anchor_output, unnormalized, _ = tcn(anchor_frames)
positive_output, _, _ = tcn(positive_frames)
negative_output, _, _ = tcn(negative_frames)
d_positive = distance(anchor_output, positive_output)
d_negative = distance(anchor_output, negative_output)
# features = encoder(anchor_frames)
loss_triplet = torch.clamp(args.margin + d_positive - d_negative, min=0.0).mean()
label_outputs_1, label_outputs_2 = attribute_classifier(all_sel_imgs_emb)
labels_1 = captions[0, 0].view(-1)
labels_2 = captions[0, 1].view(-1)
loss_1 = criterion(label_outputs_1, labels_1)
loss_2 = criterion(label_outputs_2, labels_2)
loss_language = loss_1 + loss_2
loss = loss_triplet + args.alpha * loss_language
# loss = loss_language
losses.append(loss.data.cpu().numpy())
_, predicted_1 = torch.max(label_outputs_1.data, 1)
_, predicted_2 = torch.max(label_outputs_2.data, 1)
total_1 += labels_1.size(0)
correct_1 += (predicted_1 == labels_1).sum().item()
total_2 += labels_2.size(0)
correct_2 += (predicted_2 == labels_2).sum().item()
length_set += len(anchor_frames)
print("predicted_1: ", predicted_1)
print("labels_1: ",labels_1)
print("predicted_2: ",predicted_2)
print("labels_2: ", labels_2)
print('='*10)
print('Accuracy of active label network branch: {} %'.format(100 * correct_1 / total_1))
print('Accuracy of passive label network branch: {} %'.format(100 * correct_2 / total_2))
print("="*10)
loss = np.mean(losses)
logger.info('val loss: ',loss)
message = "Validation score correct with margin {with_margin}/{total} and without margin {without_margin}/{total}".format(
with_margin=correct_with_margin,
without_margin=correct_without_margin,
total=length_set
)
logger.info(message)
return correct_with_margin, correct_without_margin, loss
def main():
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
use_cuda = torch.cuda.is_available()
tcn = create_model(use_cuda)
tcn = torch.nn.DataParallel(tcn, device_ids=(range(torch.cuda.device_count()))) # Wrapper to distribute load on multiple GPUs
attribute_classifier = DenseClassifier(num_classes=5).to(device) # load labeling network
# triplet_builder = builder(args.n_views, \
# args.train_directory, args.labels_train_directory, IMAGE_SIZE, args, sample_size=200)
# queue = multiprocessing.Queue(1)
# dataset_builder_process = multiprocessing.Process(target=build_set, args=(queue, triplet_builder, logger), daemon=True)
# dataset_builder_process.start()
optimizer = optim.SGD(list(tcn.parameters()) + list(attribute_classifier.parameters()), lr=args.lr_start, momentum=0.9)
# This will diminish the learning rate at the milestones.
# 0.1, 0.01, 0.001
learning_rate_scheduler = lr_scheduler.MultiStepLR(optimizer, milestones=[100, 200, 500], gamma=0.1)
criterion = nn.CrossEntropyLoss()
trn_losses_ = []
val_losses_= []
val_acc_margin_ = []
val_acc_no_margin_ = []
dataset = MultiViewTripletLabelDataset( args.n_views, args.train_directory, args.labels_train_directory, IMAGE_SIZE, sample_size=64)
for epoch in range(args.start_epoch, args.start_epoch + args.epochs):
losses = []
print("=" * 20)
logger.info("Starting epoch: {0} learning rate: {1}".format(epoch,
learning_rate_scheduler.get_lr()))
learning_rate_scheduler.step()
# dataset = queue.get()
data_loader = DataLoader(
dataset=dataset,
batch_size=args.minibatch_size, # batch_size(epoch, args.max_minibatch_size),
shuffle=True,
pin_memory=use_cuda,
)
for i, minibatch in enumerate(data_loader):
frames = minibatch[0]
caption = minibatch[1]
seq_idx = minibatch[2]
if use_cuda:
frames = frames[0].to(device)
captions = caption[0].to(device)
seq_idx = seq_idx[0]
snaps = dataset.get_videos(int(seq_idx[0]) * args.n_views)
all_sel_imgs = []
sel_imgs = snaps[0][::20]
all_sel_imgs.append(sel_imgs)
# all_sel_imgs = np.squeeze(np.asarray(all_sel_imgs)).reshape([sel_imgs.shape[0], 3, 299, 299])
for _ in range(0, ITERATE_OVER_TRIPLETS):
_, unnorm, all_sel_imgs_emb = tcn(torch.FloatTensor(sel_imgs).to(device))
all_sel_imgs_emb = torch.mean(all_sel_imgs_emb, dim=0, keepdim=True)
anchor_frames = frames[:, 0, :, :, :]
positive_frames = frames[:, 1, :, :, :]
negative_frames = frames[:, 2, :, :, :]
anchor_output, unnormalized, _ = tcn(anchor_frames)
positive_output, _, _ = tcn(positive_frames)
negative_output, _, _ = tcn(negative_frames)
d_positive = distance(anchor_output, positive_output)
d_negative = distance(anchor_output, negative_output)
# features = encoder(anchor_frames)
loss_triplet = torch.clamp(args.margin + d_positive - d_negative, min=0.0).mean()
if i
label_outputs_1, label_outputs_2 = attribute_classifier(all_sel_imgs_emb)
labels_1 = captions[0, 0].view(-1)
labels_2 = captions[0, 1].view(-1)
loss_1 = criterion(label_outputs_1, labels_1)
loss_2 = criterion(label_outputs_2, labels_2)
loss_language = loss_1 + loss_2
loss = loss_triplet + args.alpha * loss_language
# loss = loss_language
losses.append(loss.data.cpu().numpy())
tcn.zero_grad()
attribute_classifier.zero_grad()
loss.backward()
optimizer.step()
if (i+1) % 5 == 0:
print ('Epoch [{}/{}], Step [{}/{}], Loss Triplet: {:.4f}, Loss Language: {:.4f}'
.format(epoch+1, args.epochs, i+1, len(dataset), loss_triplet.item(), loss_language.item()))
trn_losses_.append(np.mean(losses))
if epoch % 1 == 0:
acc_margin, acc_no_margin, loss = validate(tcn, attribute_classifier, criterion, use_cuda, args)
val_losses_.append(loss)
val_acc_margin_.append(acc_margin)
val_acc_no_margin_.append(acc_no_margin)
if epoch % args.save_every == 0 and epoch != 0:
logger.info('Saving model.')
save_model(tcn, model_filename(args.model_name, epoch), args.model_folder)
plot_mean(trn_losses_, args.model_folder, 'train_loss')
plot_mean(val_losses_, args.model_folder, 'validation_loss')
# plot_mean(train_acc_, args.model_folder, 'train_acc')
plot_mean(val_acc_margin_, args.model_folder, 'validation_accuracy_margin')
plot_mean(val_acc_no_margin_, args.model_folder, 'validation_accuracy_no_margin')
if __name__ == '__main__':
main()
def main():
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
use_cuda = torch.cuda.is_available()
tcn = create_model(use_cuda)
tcn = torch.nn.DataParallel(
tcn, device_ids=(range(torch.cuda.device_count())
)) # Wrapper to distribute load on multiple GPUs
attribute_classifier = DenseClassifier(num_classes=5).to(
device) # load labeling network
triplet_builder = builder(args.n_views, \
args.train_directory, args.labels_train_directory, IMAGE_SIZE, args, sample_size=32)
queue = multiprocessing.Queue(1)
dataset_builder_process = multiprocessing.Process(target=build_set,
args=(queue,
triplet_builder,
logger),
daemon=True)
dataset_builder_process.start()
optimizer = optim.SGD(list(tcn.parameters()) +
list(attribute_classifier.parameters()),
lr=args.lr_start,
momentum=0.9)
# This will diminish the learning rate at the milestones.
# 0.1, 0.01, 0.001
learning_rate_scheduler = lr_scheduler.MultiStepLR(
optimizer, milestones=[100, 200, 500], gamma=0.1)
criterion = nn.CrossEntropyLoss()
trn_losses_ = []
val_losses_ = []
val_acc_margin_ = []
val_acc_no_margin_ = []
for epoch in range(args.start_epoch, args.start_epoch + args.epochs):
losses = []
print("=" * 20)
logger.info("Starting epoch: {0} learning rate: {1}".format(
epoch, learning_rate_scheduler.get_lr()))
learning_rate_scheduler.step()
dataset = queue.get()
data_loader = DataLoader(
dataset=dataset,
batch_size=args.
minibatch_size, # batch_size(epoch, args.max_minibatch_size),
shuffle=True,
pin_memory=use_cuda,
)
for _ in range(0, ITERATE_OVER_TRIPLETS):
for i, minibatch in enumerate(data_loader):
frames = minibatch[0]
captions = minibatch[1]
if use_cuda:
frames = frames.cuda()
captions = captions.to(device)
print(captions)
print(len(data_loader))
anchor_frames = frames[:, 0, :, :, :]
positive_frames = frames[:, 1, :, :, :]
negative_frames = frames[:, 2, :, :, :]
anchor_output, unnormalized, mixed = tcn(anchor_frames)
positive_output, _, _ = tcn(positive_frames)
negative_output, _, _ = tcn(negative_frames)
d_positive = distance(anchor_output, positive_output)
d_negative = distance(anchor_output, negative_output)
# features = encoder(anchor_frames)
loss_triplet = torch.clamp(args.margin + d_positive -
d_negative,
min=0.0).mean()
label_outputs_1, label_outputs_2 = attribute_classifier(mixed)
labels_1 = captions[:, 0]
# labels_2 = captions[:, 1]
loss_1 = criterion(label_outputs_1, labels_1)
# loss_2 = criterion(label_outputs_2, labels_2)
loss_language = loss_1 #+ loss_2
# loss = loss_triplet + args.alpha * loss_language
loss = loss_language
# loss = loss_triplet
losses.append(loss.data.cpu().numpy())
tcn.zero_grad()
attribute_classifier.zero_grad()
loss.backward()
optimizer.step()
trn_losses_.append(np.mean(losses))
logger.info('train loss: ', np.mean(losses))
if epoch % 1 == 0:
acc_margin, acc_no_margin, loss = validate(tcn,
attribute_classifier,
criterion, use_cuda,
args)
val_losses_.append(loss)
val_acc_margin_.append(acc_margin)
val_acc_no_margin_.append(acc_no_margin)
if epoch % args.save_every == 0 and epoch != 0:
logger.info('Saving model.')
save_model(tcn, model_filename(args.model_name, epoch),
args.model_folder)
plot_mean(trn_losses_, args.model_folder, 'train_loss')
plot_mean(val_losses_, args.model_folder, 'validation_loss')
# plot_mean(train_acc_, args.model_folder, 'train_acc')
plot_mean(val_acc_margin_, args.model_folder,
'validation_accuracy_margin')
plot_mean(val_acc_no_margin_, args.model_folder,
'validation_accuracy_no_margin')
if __name__ == '__main__':
from utils.util import distance
# extract_model_path = os.environ['HOME'] + "/models/insightface/model-r50-am-lfw/model, 0"
# insightface_API = Insightface_API(extract_model_path)
# img1 = cv2.imread(os.environ['HOME'] + "/A311D/insightface/test_112_cz_1.jpg")
# imbeddings1 = insightface_API.extract(img1)
# print(imbeddings1)
#
# img2 = cv2.imread(os.environ['HOME'] + "/A311D/insightface/test_112_cz_5.jpg")
# imbeddings2 = insightface_API.extract(img2)
# print(imbeddings2)
#
# print(distance([imbeddings1], [imbeddings2], 0))
with open('/home/weishu/A311D/insightface/bin_r/output0_512_1_cz_1.txt',
'r') as f:
imbeddings1_ = f.readlines()
for i in range(0, len(imbeddings1_)):
imbeddings1_[i] = float(imbeddings1_[i].rstrip('\n'))
print(imbeddings1_)
with open('/home/weishu/A311D/insightface/bin_r/output0_512_1_cz_5.txt',
'r') as f:
imbeddings2_ = f.readlines()
for i in range(0, len(imbeddings2_)):
imbeddings2_[i] = float(imbeddings2_[i].rstrip('\n'))
print(imbeddings2_)
print(distance([imbeddings1_], [imbeddings2_], 0))
def attenuation(color,last_mean):
return 1 - math.exp(((distance(color, last_mean) / 80) ** 2) * -1)
def main():
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
use_cuda = torch.cuda.is_available()
tcn = create_model(use_cuda)
encoder = EncoderCNN(args.embed_size).to(device)
decoder = DecoderRNN(args.embed_size, args.hidden_size, len(vocab), \
args.num_layers).to(device)
triplet_builder = builder(args.n_views, \
args.train_directory, IMAGE_SIZE, vocab, args, sample_size=200)
queue = multiprocessing.Queue(1)
dataset_builder_process = multiprocessing.Process(target=build_set,
args=(queue,
triplet_builder,
logger),
daemon=True)
dataset_builder_process.start()
opt_params = list(tcn.parameters()) + list(decoder.parameters()) + list(
encoder.parameters())
optimizer = optim.SGD(opt_params, lr=args.lr_start, momentum=0.9)
# This will diminish the learning rate at the milestones.
# 0.1, 0.01, 0.001
learning_rate_scheduler = lr_scheduler.MultiStepLR(
optimizer, milestones=[100, 500, 1000], gamma=0.1)
criterion = nn.CrossEntropyLoss()
for epoch in range(args.start_epoch, args.start_epoch + args.epochs):
print("=" * 20)
logger.info("Starting epoch: {0} learning rate: {1}".format(
epoch, learning_rate_scheduler.get_lr()))
learning_rate_scheduler.step()
dataset = queue.get()
logger.info("Got {0} triplets".format(len(dataset)))
data_loader = DataLoader(
dataset=dataset,
batch_size=args.
minibatch_size, # batch_size(epoch, args.max_minibatch_size),
shuffle=True,
pin_memory=use_cuda,
collate_fn=collate_fn)
if epoch % 10 == 0:
validate(tcn, decoder, use_cuda, args)
for _ in range(0, ITERATE_OVER_TRIPLETS):
losses = []
for minibatch, captions, lengths in data_loader:
frames = Variable(minibatch)
if use_cuda:
frames = frames.cuda()
captions = captions.to(device)
anchor_frames = frames[:, 0, :, :, :]
positive_frames = frames[:, 1, :, :, :]
negative_frames = frames[:, 2, :, :, :]
anchor_output, unnormalized = tcn(anchor_frames)
positive_output, _ = tcn(positive_frames)
negative_output, _ = tcn(negative_frames)
d_positive = distance(anchor_output, positive_output)
d_negative = distance(anchor_output, negative_output)
targets = pack_padded_sequence(captions,
lengths,
batch_first=True)[0]
# features = encoder(anchor_frames)
caption_outputs = decoder(unnormalized, captions, lengths)
loss_triplet = torch.clamp(args.margin + d_positive -
d_negative,
min=0.0).mean()
loss_language = criterion(caption_outputs, targets)
loss = loss_triplet + args.alpha * loss_language
losses.append(loss.data.cpu().numpy())
tcn.zero_grad()
decoder.zero_grad()
encoder.zero_grad()
loss.backward()
optimizer.step()
logger.info('loss: ', np.mean(losses))
# Generate an caption from the image
_, sample_feature = tcn(frames[0, 0, :, :, :][None])
sampled_ids = decoder.sample(sample_feature)
sampled_ids = sampled_ids[0].cpu().numpy(
) # (1, max_seq_length) -> (max_seq_length) sampled_caption = []
sampled_caption = []
for word_id in captions[0, :].cpu().numpy():
word = vocab.idx2word[word_id]
sampled_caption.append(word)
sentence = ' '.join(sampled_caption)
print(
"Target: ",
sentence,
)
for word_id in sampled_ids:
word = vocab.idx2word[word_id]
sampled_caption.append(word)
if word == '<end>':
break
sentence = ' '.join(sampled_caption)
print("Prediction: ", sentence)
if epoch % args.save_every == 0 and epoch != 0:
logger.info('Saving model.')
save_model(tcn, model_filename(args.model_name, epoch),
args.model_folder)
plot_mean(train_loss_, save_dir, 'train_loss')
plot_mean(test_loss_, save_dir, 'test_loss')
plot_mean(train_acc_, save_dir, 'train_acc')
plot_mean(test_acc_, save_dir, 'test_acc')
