def Validate(opt, net, loss_func, validate_loader, tb, epoch):
with torch.no_grad():
net.eval()
n = opt.validate_count
label = validate_loader.dataset.label
a_chain = toTensor(
validate_loader.dataset.dataset[:, :, :opt.length, :])
b_chain = toTensor(validate_loader.dataset.dataset[:, :,
opt.length:, :])
output = net(a_chain, b_chain)
loss = loss_func(output, label).data.numpy()
output = output.view(-1).data.numpy()
label = label.data.numpy()
pred = (output > 0.5)
acc = (pred == label).mean()
auc = get_AUC(output, label, n)
roc = get_ROC(output, label, n)
print("Validate: loss: %lf, acc: %lf, auc: %lf" % (loss, acc, auc))
tb.add_scalar('validate/validate_loss', loss, epoch)
tb.add_scalar('validate/validate_acc', acc, epoch)
tb.add_scalar('validate/validate_prec',
abs((output - 0.5) * 2).mean(), epoch)
tb.add_scalar('validate/validate/auc', auc, epoch)
tb.add_figure('validate/ROC curve', roc, epoch)
net.train()
def match(opt, net, data):
n = data.shape[0]
a = toTensor(data[:, :, :opt.length])
b = toTensor(data[:, :, opt.length:])
feat_a = net.feature_extractor_x(a)
feat_b = net.feature_extractor_y(b)
embed()
target = [i for i in range(n)]
weight = [[net.match(feat_a[i], feat_b[j]) for i in range(n)]
for j in range(n)]
# TODO: weight
# TODO: the method to match a & b
# 1. greedy
# 1.1 for each row
pred = torch.max(weight, 1)[1].data.cpu().numpy().squeeze()
acc1 = sum(pred == target) / n
# 1.2 for each column
pred = torch.max(weight.transpose(0, 1), 1)[1].data.cpu().numpy().squeeze()
acc2 = sum(pred == target) / n
# 2. km
return (acc1, acc2)
def generateDataset(trainExamples, vocab, use_shuffle = True, skip_disagreement = False):
wordToIndex, indexToWord = utils.getIndexTranslations(vocab)
conversations = []
for i, example in enumerate(trainExamples):
#print("example = ", exampleToString(example))
agent_input = example["input"]
dialogue = example["dialogue"]
output = example["output"]
partner_input = example["partner input"]
if skip_disagreement:
output_vector = output[0] + output[1]
if (-1 in output_vector):
##there was disagreement so don't bother training on this example
continue
goals_tensor = utils.toTensor(agent_input, "goals")
word_indexes = []
running_context = [wordToIndex["<START>"]]
contextResponsePairs = []
for sentence_i, sentence in enumerate(dialogue):
speaker = sentence[0]
utterance = [speaker]
utterance += sentence[1].split(" ")
utterance += ["<eos>"]
utterance_idx = [wordToIndex[word] if word in wordToIndex else wordToIndex["<UNK>"] for word in utterance]
#Clip the speaker token
target_tensor = torch.tensor(utterance_idx[1:], dtype=torch.long)
running_context_tensor = torch.tensor(running_context, dtype=torch.long)
contextResponsePairs.append((running_context_tensor, target_tensor))
running_context += utterance_idx
conversations.append((goals_tensor, contextResponsePairs, example))
if (use_shuffle):
random.shuffle(conversations)
return conversations
def rnnBotConsole(trainExamples, vocab):
#import pdb; pdb.set_trace()
vocab.append("<START>")
vocab.append("<sos>")
vocab.append("<eos>")
vocab.append("YOU")
vocab.append("THEM")
if "<UNK>" not in vocab:
vocab.append("<UNK>")
vocab = sorted(vocab)
wordToIndex, indexToWord = utils.getIndexTranslations(vocab)
#encoder = torch.load("savedModels/encoderTrained.pth")
#decoder = torch.load("savedModels/decoderTrained.pth")
#goals_encoder = torch.load("savedModels/goals_encoderTrained.pth")
encoder = torch.load("savedModels/encoder.pth")
decoder = torch.load("savedModels/decoder.pth")
goals_encoder = torch.load("savedModels/goals_encoder.pth")
output_classifier = torch.load("savedModels/outputClassifier.pth")
while(True):
index = random.choice(range(len(trainExamples)))
ex = trainExamples[index]
bot_input = ex["input"]
opponent_input = ex["partner input"]
print("\n\nOpponent input = " + inputToString(opponent_input))
#print("Bot input = ", inputToString(bot_input))
total_counts = [count for count,value in bot_input]
bot_values = [value for count,value in bot_input]
opponent_values = [value for count,value in opponent_input]
'''bot_value_sort = np.argsort((np.multiply(bot_values, total_counts)))
mostValuableIndex = bot_value_sort[len(items) -1]
middleIndex = bot_value_sort[1]
leastValuableIndex = bot_value_sort[0]
botItems, opponentItems = None, None
attemptedTopItem = False
num_attempts = 0'''
bot_goals_tensor = utils.toTensor(bot_input, "goals")
encoded_goals = goals_encoder(bot_goals_tensor)
running_context = ["<START>"]
botStarting = random.choice([True, False])
if(botStarting):
#encode
encoder_input = torch.tensor([wordToIndex["<START>"]], dtype=torch.long)[0]
hidden = encoder.initHidden()
hidden = encoder(encoder_input, hidden, encoded_goals)
#decode
selected_tokens = []
encoded_context = hidden
hidden = decoder.initHidden()
decoder_input = torch.tensor([wordToIndex["<sos>"]], dtype=torch.long)[0]
while(True):
output, hidden = decoder(decoder_input, encoded_context, hidden, encoded_goals)
#categorical = torch.distributions.Categorical(output)
#selected_token = categorical.sample()
selected_token_probability, selected_token = torch.max(output, 1)
selected_token_as_word = indexToWord[int(selected_token[0].numpy())]
if (selected_token_as_word == "<eos>"):
break
#if(selected_token_as_word != "THEM" and selected_token_as_word != "YOU"):
selected_tokens.append(selected_token_as_word)
decoder_input = selected_token[0]
print("Bot statement: ", " ".join(selected_tokens))
running_context += ["YOU"] + selected_tokens + ["<eos>"]
#print("running context = ", running_context)
while(True):
input_str = input('Enter a response: ')
if(input_str == "<selection>"):
running_context += ["THEM", "<selection>"]
break
tokenized = tokenize(input_str)
running_context += ["THEM"] + tokenized + ["<eos>"]
#print("running_context = ", running_context)
running_context_idx = [wordToIndex[word] if word in wordToIndex else wordToIndex["<UNK>"] for word in running_context]
running_context_tensor = torch.tensor(running_context_idx, dtype=torch.long)
context = running_context_tensor
hidden = encoder.initHidden()
num_context_words = context.size(0)
for i in range(num_context_words):
hidden = encoder(context[i], hidden, encoded_goals)
#decode
maxResponseLen = 50
selected_tokens = []
encoded_context = hidden
hidden = decoder.initHidden()
decoder_input = torch.tensor([wordToIndex["<sos>"]], dtype=torch.long)[0]
while(True):
output, hidden = decoder(decoder_input, encoded_context, hidden, encoded_goals)
selected_token_probability, selected_token = torch.max(output, 1)
selected_token_as_word = indexToWord[int(selected_token[0].numpy())]
if (selected_token_as_word == "<eos>"):
break
#if(selected_token_as_word != "THEM" and selected_token_as_word != "YOU"):
selected_tokens.append(selected_token_as_word)
if(len(selected_tokens) >= maxResponseLen):
break
decoder_input = selected_token[0]
print("Bot response: ", " ".join(selected_tokens))
running_context += ["YOU"] + selected_tokens + ["<eos>"]
if (selected_tokens == ["<selection>"]):
break
#print("final running context = ", " ".join(running_context))
running_context_idx = [wordToIndex[word] if word in wordToIndex else wordToIndex["<UNK>"] for word in running_context]
running_context_tensor = torch.tensor(running_context_idx, dtype=torch.long)
num_words = running_context_tensor.size(0)
hidden = output_classifier.initHidden()
for i in range(num_words):
outputs, hidden = output_classifier(running_context_tensor[i], hidden, bot_goals_tensor)
# in form: counts = [count1, count2, count3, opponent_count1, opponent_count2, opponent_count3]
bot_selection = findBestValidPrediction(outputs, bot_input)
bot_counts = bot_selection[:3]
player_counts = bot_selection[3:]
print("Bot input = ", inputToString(bot_input))
print("Bot items: ", countsToString(bot_counts))
print("Player items: ", countsToString(player_counts))
print("Bot score = ", np.sum(np.multiply(bot_counts, bot_values)))
print("Opponent score = ", np.sum(np.multiply(player_counts, opponent_values)))
sentinel = input("Type 'quit' to exit, anything else to run another example: ")
if (sentinel == 'quit'):
break
mask_image = cv2.imread(mask_image)
mask_image_ori = copy.deepcopy(mask_image)
show_from_cv(mask_image)
tmask_image = cv2.imread(tmask_image)
tmask_image_ori = copy.deepcopy(tmask_image)
show_from_cv(tmask_image)
tr = 3
tmask_image = cv2.GaussianBlur(tmask_image, (2 * tr + 1, 2 * tr + 1), tr)
show_from_cv(tmask_image)
cnn = models.vgg19(pretrained=True).features.to(device).eval()
#
content_image = toTensor(content_image).to(device, torch.float)
style_image = toTensor(style_image).to(device, torch.float)
mask_image = toTensor(mask_image).to(device, torch.float)
tmask_image = toTensor(tmask_image).to(device, torch.float)
# a = mask_image.reshape(1,-1)
#
# for i in range(0, 1432200):
# print(a[0][i])
print('===> Initialize the image...')
# input_img = torch.randn(content_image.data.size(), device=device)
input_img = content_image.clone()
print('the image tensor size is {}'.format(input_img.size()))
show_from_tensor(input_img)
def get_model_and_losses(cnn, normalization_mean, normalization_std,
style_img, content_img, mask_image, tmask_image,
style_weight=100, content_weight=5, tv_weight=1e-3,
content_layers=content_layers_default,
style_layers=style_layers_default):
content_losses = []
style_losses = []
normalization = Normalization(normalization_mean, normalization_std).to(device)
model = nn.Sequential(normalization)
tv_loss = None
if tv_weight > 0:
tv_loss = TVLoss(tv_weight)
model.add_module('tv_loss', tv_loss)
# model.add_module('tv_loss', TVLoss(tv_weight))
i = 0
for layer in cnn.children():
if isinstance(layer, nn.Conv2d):
sap = nn.AvgPool2d(kernel_size=3, stride=1, padding=1)
if not isinstance(mask_image, torch.Tensor):
mask_image = toTensor(mask_image).to(device)
mask_image = sap(mask_image)
i += 1
name = "conv_" + str(i)
model.add_module(name, layer)
# why every time we resize the mask image to a smaller image,
# because later we need mask image to fit input image in deep layers
# vgg19 only shrink image size in pooling layer and the rate is 1/2!
elif isinstance(layer, nn.MaxPool2d):
if isinstance(mask_image, torch.Tensor):
mask_image = tensor_to_np(mask_image)
mask_image = cv2.resize(mask_image,
(math.floor(mask_image.shape[1] / 2), math.floor(mask_image.shape[0] / 2)))
mask_image = toTensor(mask_image).to(device)
name = "pool_" + str(i)
model.add_module(name, layer)
elif isinstance(layer, nn.ReLU):
name = "relu_" + str(i)
model.add_module(name, nn.ReLU(inplace=False))
if name in content_layers:
print('-----Setting up content layer-----')
target = model(content_img).detach()
content_loss = ContentLoss(target, mask_image, content_weight)
content_loss.register_backward_hook(content_loss.content_hook)
model.add_module("content_loss_" + str(i), content_loss)
content_losses.append(content_loss)
if name in style_layers:
print('-----Setting up style layer-----')
# content_target = model(content_img).detach()
target_feature = model(style_img).detach()
mask = mask_image[:, 0:1, :, :]
mask = mask.expand_as(target_feature)
target_feature = target_feature * mask
# add a histogram match here
style_loss = StyleLoss(target_feature, mask_image, style_weight)
style_loss.register_backward_hook(style_loss.style_hook)
model.add_module("style_loss" + str(i), style_loss)
style_losses.append(style_loss)
for i in range(len(model) - 1, -1, -1):
if isinstance(model[i], ContentLoss) or isinstance(model[i], StyleLoss):
break
model = model[:i]
return model, style_losses, content_losses, tv_loss
def generateDataset(trainExamples, vocab, use_shuffle = True):
vocab.append("<START>")
vocab.append("<sos>")
vocab.append("<eos>")
vocab.append("YOU")
vocab.append("THEM")
if "<UNK>" not in vocab:
vocab.append("<UNK>")
vocab = sorted(vocab)
wordToIndex, indexToWord = utils.getIndexTranslations(vocab)
contextTargetPairs = []
for i, example in enumerate(trainExamples):
#print("example = ", exampleToString(example))
agent_input = example["input"]
dialogue = example["dialogue"]
output = example["output"]
partner_input = example["partner input"]
goals_tensor = utils.toTensor(agent_input, "goals")
#for now, just use raw tensor, can also train an embedding
encoded_goals = goals_tensor
word_indexes = []
for sentence in dialogue:
speaker = sentence[0]
utterance = [speaker]
utterance += sentence[1].split(" ")
utterance += ["<eos>"]
utterance_idx = [wordToIndex[word] if word in wordToIndex else wordToIndex["<UNK>"] for word in utterance]
word_indexes += utterance_idx
words_tensor = torch.tensor(word_indexes, dtype=torch.long)
#print("words tensor = ", words_tensor)
output_vector = output[0] + output[1]
if (-1 in output_vector):
##there was disagreement so don't bother training on this example
continue
output_tensors = []
for output_idx in range(6):
output_count = output_vector[output_idx]
output_tensor = torch.tensor([output_count], dtype = torch.long)
output_tensors.append(output_tensor)
context = (words_tensor, encoded_goals)
target = output_tensors
contextTargetPairs.append((context, target, example))
if (use_shuffle):
random.shuffle(contextTargetPairs)
return contextTargetPairs