def main():
Logger('.')
#classes = ['pizza', 'pork chops', 'cupcake', 'hamburger', 'green beans']
split = 'test'
dir_exp = '/home/cadene/doc/bootstrap.pytorch/logs/recipe1m/trijoint/2017-12-14-15-04-51'
path_opts = os.path.join(dir_exp, 'options.yaml')
dir_extract = os.path.join(dir_exp, 'extract_mean_features', split)
dir_img = os.path.join(dir_extract, 'image')
dir_rcp = os.path.join(dir_extract, 'recipe')
path_model_ckpt = os.path.join(
dir_exp,
'ckpt_best_val_epoch.metric.recall_at_1_im2recipe_mean_model.pth.tar')
Options.load_from_yaml(path_opts)
utils.set_random_seed(Options()['misc']['seed'])
Logger()('Load dataset...')
dataset = factory(split)
Logger()('Load model...')
model = model_factory()
model_state = torch.load(path_model_ckpt)
model.load_state_dict(model_state)
model.set_mode(split)
if not os.path.isdir(dir_extract):
Logger()('Create extract_dir {}'.format(dir_extract))
os.system('mkdir -p ' + dir_extract)
mean_ingrs = torch.zeros(model.network.recipe_embedding.dim_ingr_out *
2) # bi LSTM
mean_instrs = torch.zeros(model.network.recipe_embedding.dim_instr_out)
for i in tqdm(range(len(dataset))):
item = dataset[i]
batch = dataset.items_tf()([item])
batch = model.prepare_batch(batch)
out_ingrs = model.network.recipe_embedding.forward_ingrs(
batch['recipe']['ingrs'])
out_instrs = model.network.recipe_embedding.forward_instrs(
batch['recipe']['instrs'])
mean_ingrs += out_ingrs.data.cpu().squeeze(0)
mean_instrs += out_instrs.data.cpu().squeeze(0)
mean_ingrs /= len(dataset)
mean_instrs /= len(dataset)
path_ingrs = os.path.join(dir_extract, 'ingrs.pth')
path_instrs = os.path.join(dir_extract, 'instrs.pth')
torch.save(mean_ingrs, path_ingrs)
torch.save(mean_instrs, path_instrs)
Logger()('End')
def main():
classes = ['pizza', 'pork chops', 'cupcake', 'hamburger', 'green beans']
nb_points = 100
split = 'test'
dir_exp = 'logs/recipe1m/trijoint/2017-12-14-15-04-51'
path_opts = os.path.join(dir_exp, 'options.yaml')
dir_extract = os.path.join(dir_exp, 'extract', split)
dir_img = os.path.join(dir_extract, 'image')
dir_rcp = os.path.join(dir_extract, 'recipe')
path_model = os.path.join(dir_exp, 'ckpt_best_val_epoch.metric.recall_at_1_im2recipe_mean_model.pth.tar')
dir_visu = os.path.join(dir_exp, 'visu', 'top5')
#Options(path_opts)
Options.load_from_yaml(path_opts)
dataset = factory(split)
network = Trijoint()
network.eval()
model_state = torch.load(path_model)
network.load_state_dict(model_state['network'])
list_idx = torch.randperm(len(dataset))
img_embs = []
rcp_embs = []
for i in range(nb_points):
idx = list_idx[i]
path_img = os.path.join(dir_img, '{}.pth'.format(idx))
path_rcp = os.path.join(dir_rcp, '{}.pth'.format(idx))
img_embs.append(torch.load(path_img))
rcp_embs.append(torch.load(path_rcp))
img_embs = torch.stack(img_embs, 0)
rcp_embs = torch.stack(rcp_embs, 0)
dist = fast_distance(img_embs, rcp_embs)
im2recipe_ids = np.argsort(dist.numpy(), axis=0)
recipe2im_ids = np.argsort(dist.numpy(), axis=1)
import ipdb; ipdb.set_trace()
def main():
Logger('.')
#classes = ['pizza', 'pork chops', 'cupcake', 'hamburger', 'green beans']
nb_points = 1000
split = 'test'
dir_exp = '/home/cadene/doc/bootstrap.pytorch/logs/recipe1m/trijoint/2017-12-14-15-04-51'
path_opts = os.path.join(dir_exp, 'options.yaml')
dir_extract = os.path.join(dir_exp, 'extract', split)
dir_extract_mean = os.path.join(dir_exp, 'extract_mean_features', split)
dir_img = os.path.join(dir_extract, 'image')
dir_rcp = os.path.join(dir_extract, 'recipe')
path_model_ckpt = os.path.join(
dir_exp,
'ckpt_best_val_epoch.metric.recall_at_1_im2recipe_mean_model.pth.tar')
dir_visu = os.path.join(dir_exp, 'visu', 'mean_to_image')
os.system('mkdir -p ' + dir_visu)
#Options(path_opts)
Options.load_from_yaml(path_opts)
utils.set_random_seed(Options()['misc']['seed'])
dataset = factory(split)
Logger()('Load model...')
model = model_factory()
model_state = torch.load(path_model_ckpt)
model.load_state_dict(model_state)
model.set_mode(split)
#emb = network.recipe_embedding.forward_ingrs(input_['recipe']['ingrs'])
list_idx = torch.randperm(len(dataset))
Logger()('Load embeddings...')
img_embs = []
rcp_embs = []
for i in range(nb_points):
idx = list_idx[i]
path_img = os.path.join(dir_img, '{}.pth'.format(idx))
path_rcp = os.path.join(dir_rcp, '{}.pth'.format(idx))
if not os.path.isfile(path_img):
Logger()('No such file: {}'.format(path_img))
continue
if not os.path.isfile(path_rcp):
Logger()('No such file: {}'.format(path_rcp))
continue
img_embs.append(torch.load(path_img))
rcp_embs.append(torch.load(path_rcp))
img_embs = torch.stack(img_embs, 0)
rcp_embs = torch.stack(rcp_embs, 0)
Logger()('Load means')
path_ingrs = os.path.join(dir_extract_mean, 'ingrs.pth')
path_instrs = os.path.join(dir_extract_mean, 'instrs.pth')
mean_ingrs = torch.load(path_ingrs)
mean_instrs = torch.load(path_instrs)
mean_ingrs = Variable(mean_ingrs.unsqueeze(0).cuda(), requires_grad=False)
mean_instrs = Variable(mean_instrs.unsqueeze(0).cuda(),
requires_grad=False)
Logger()('Forward ingredient...')
ingr_emb = model.network.recipe_embedding.forward_ingrs_instrs(
mean_ingrs, mean_instrs)
ingr_emb = ingr_emb.data.cpu()
ingr_emb = ingr_emb.expand_as(img_embs)
Logger()('Fast distance...')
dist = fast_distance(img_embs, ingr_emb)[:, 0]
sorted_img_ids = np.argsort(dist.numpy())
Logger()('Load/save images...')
for i in range(20):
img_id = sorted_img_ids[i]
img_id = int(img_id)
path_img_from = dataset[img_id]['image']['path']
path_img_to = os.path.join(dir_visu, 'image_top_{}.png'.format(i + 1))
img = Image.open(path_img_from)
img.save(path_img_to)
#os.system('cp {} {}'.format(path_img_from, path_img_to))
Logger()('End')
def main():
Logger('.')
#classes = ['pizza', 'pork chops', 'cupcake', 'hamburger', 'green beans']
nb_points = 1000
split = 'test'
dir_exp = '/home/ubuntu/moochi/recipe1m.bootstrap.pytorch/logs/recipe1m/trijoint/2017-12-14-15-04-51'
path_opts = os.path.join(dir_exp, 'options.yaml')
dir_extract = os.path.join(dir_exp, 'extract', split)
dir_extract_mean = os.path.join(dir_exp, 'extract_mean_features', split)
dir_img = os.path.join(dir_extract, 'image')
dir_rcp = os.path.join(dir_extract, 'recipe')
path_model_ckpt = os.path.join(
dir_exp,
'ckpt_best_val_epoch.metric.recall_at_1_im2recipe_mean_model.pth.tar')
is_mean = True
ingrs_list = ['tomato'] #['tomato', 'salad', 'onion', 'chicken']
#Options(path_opts)
Options.load_from_yaml(path_opts)
utils.set_random_seed(Options()['misc']['seed'])
dataset = factory(split)
Logger()('Load model...')
model = model_factory()
model_state = torch.load(path_model_ckpt)
model.load_state_dict(model_state)
model.eval()
if not os.path.isdir(dir_extract):
os.system('mkdir -p ' + dir_rcp)
os.system('mkdir -p ' + dir_img)
for i in tqdm(range(len(dataset))):
item = dataset[i]
batch = dataset.items_tf()([item])
if model.is_cuda:
batch = model.cuda_tf()(batch)
is_volatile = (model.mode not in ['train', 'trainval'])
batch = model.variable_tf(volatile=is_volatile)(batch)
out = model.network(batch)
path_image = os.path.join(dir_img, '{}.pth'.format(i))
path_recipe = os.path.join(dir_rcp, '{}.pth'.format(i))
torch.save(out['image_embedding'][0].data.cpu(), path_image)
torch.save(out['recipe_embedding'][0].data.cpu(), path_recipe)
# b = dataset.make_batch_loader().__iter__().__next__()
# import ipdb; ipdb.set_trace()
ingrs = torch.LongTensor(1, len(ingrs_list))
for i, ingr_name in enumerate(ingrs_list):
ingrs[0, i] = dataset.recipes_dataset.ingrname_to_ingrid[ingr_name]
input_ = {
'recipe': {
'ingrs': {
'data': Variable(ingrs.cuda(), requires_grad=False),
'lengths': [ingrs.size(1)]
},
'instrs': {
'data':
Variable(torch.FloatTensor(1, 1, 1024).fill_(0).cuda(),
requires_grad=False),
'lengths': [1]
}
}
}
#emb = network.recipe_embedding.forward_ingrs(input_['recipe']['ingrs'])
list_idx = torch.randperm(len(dataset))
# nb_points = list_idx.size(0)
Logger()('Load embeddings...')
img_embs = []
rcp_embs = []
for i in range(nb_points):
idx = list_idx[i]
path_img = os.path.join(dir_img, '{}.pth'.format(idx))
path_rcp = os.path.join(dir_rcp, '{}.pth'.format(idx))
if not os.path.isfile(path_img):
Logger()('No such file: {}'.format(path_img))
continue
if not os.path.isfile(path_rcp):
Logger()('No such file: {}'.format(path_rcp))
continue
img_embs.append(torch.load(path_img))
rcp_embs.append(torch.load(path_rcp))
img_embs = torch.stack(img_embs, 0)
rcp_embs = torch.stack(rcp_embs, 0)
Logger()('Load mean embeddings')
path_ingrs = os.path.join(dir_extract_mean, 'ingrs.pth')
path_instrs = os.path.join(dir_extract_mean, 'instrs.pth')
mean_ingrs = torch.load(path_ingrs)
mean_instrs = torch.load(path_instrs)
Logger()('Forward ingredient...')
#ingr_emb = model.network.recipe_embedding(input_['recipe'])
ingr_emb = model.network.recipe_embedding.forward_one_ingr(
input_['recipe']['ingrs'], emb_instrs=mean_instrs.unsqueeze(0))
ingr_emb = ingr_emb.data.cpu()
ingr_emb = ingr_emb.expand_as(img_embs)
Logger()('Fast distance...')
dist = fast_distance(img_embs, ingr_emb)[:, 0]
sorted_ids = np.argsort(dist.numpy())
dir_visu = os.path.join(
dir_exp, 'visu',
'ingrs_to_image_nb_points:{}_instrs:{}_mean:{}'.format(
nb_points, '-'.join(ingrs_list), is_mean))
os.system('mkdir -p ' + dir_visu)
Logger()('Load/save images to {}...'.format(dir_visu))
for i in range(20):
idx = int(sorted_ids[i])
item_id = list_idx[idx]
item = dataset[item_id]
path_img_from = item['image']['path']
ingrs = [
ingr.replace('/', '\'')
for ingr in item['recipe']['ingrs']['interim']
]
cname = item['recipe']['class_name']
path_img_to = os.path.join(
dir_visu, 'image_top:{}_cname:{}.png'.format(i + 1, cname))
img = Image.open(path_img_from)
img.save(path_img_to)
#os.system('cp {} {}'.format(path_img_from, path_img_to))
Logger()('End')
def main():
Logger('.')
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('modality_to_modality', help='foo help', default='recipe_to_image')
args = parser.parse_args()
#classes = ['pizza', 'pork chops', 'cupcake', 'hamburger', 'green beans']
nb_points = 1000
modality_to_modality = args.modality_to_modality#'image_to_image'
print(modality_to_modality)
split = 'test'
dir_exp = '/home/ubuntu/moochi/recipe1m.bootstrap.pytorch/logs/recipe1m/trijoint/2017-12-14-15-04-51'
path_opts = os.path.join(dir_exp, 'options.yaml')
dir_extract = os.path.join(dir_exp, 'extract', split)
dir_img = os.path.join(dir_extract, 'image')
dir_rcp = os.path.join(dir_extract, 'recipe')
path_model_ckpt = os.path.join(dir_exp, 'ckpt_best_val_epoch.metric.recall_at_1_im2recipe_mean_model.pth.tar')
Options.load_from_yaml(path_opts)
Options()['misc']['seed'] = 11
utils.set_random_seed(Options()['misc']['seed'])
dataset = factory(split)
Logger()('Load model...')
model = model_factory()
model_state = torch.load(path_model_ckpt)
model.load_state_dict(model_state)
model.eval()
if not os.path.isdir(dir_extract):
os.system('mkdir -p '+dir_rcp)
os.system('mkdir -p '+dir_img)
for i in tqdm(range(len(dataset))):
item = dataset[i]
batch = dataset.items_tf()([item])
if model.is_cuda:
batch = model.cuda_tf()(batch)
is_volatile = (model.mode not in ['train', 'trainval'])
batch = model.variable_tf(volatile=is_volatile)(batch)
out = model.network(batch)
path_image = os.path.join(dir_img, '{}.pth'.format(i))
path_recipe = os.path.join(dir_rcp, '{}.pth'.format(i))
torch.save(out['image_embedding'][0].data.cpu(), path_image)
torch.save(out['recipe_embedding'][0].data.cpu(), path_recipe)
indices_by_class = dataset._make_indices_by_class()
# class_name = classes[0] # TODO
# class_id = dataset.cname_to_cid[class_name]
# list_idx = torch.Tensor(indices_by_class[class_id])
# rand_idx = torch.randperm(list_idx.size(0))
# list_idx = list_idx[rand_idx]
# list_idx = list_idx.view(-1).int()
list_idx = torch.randperm(len(dataset))
#nb_points = list_idx.size(0)
dir_visu = os.path.join(dir_exp, 'visu', '{}_top20_seed:{}'.format(modality_to_modality, Options()['misc']['seed']))
os.system('rm -rf '+dir_visu)
os.system('mkdir -p '+dir_visu)
Logger()('Load embeddings...')
img_embs = []
rcp_embs = []
for i in range(nb_points):
idx = list_idx[i]
#idx = i
path_img = os.path.join(dir_img, '{}.pth'.format(idx))
path_rcp = os.path.join(dir_rcp, '{}.pth'.format(idx))
if not os.path.isfile(path_img):
Logger()('No such file: {}'.format(path_img))
continue
if not os.path.isfile(path_rcp):
Logger()('No such file: {}'.format(path_rcp))
continue
img_embs.append(torch.load(path_img))
rcp_embs.append(torch.load(path_rcp))
img_embs = torch.stack(img_embs, 0)
rcp_embs = torch.stack(rcp_embs, 0)
# Logger()('Forward ingredient...')
# #ingr_emb = model.network.recipe_embedding(input_['recipe'])
# ingr_emb = model.network.recipe_embedding.forward_one_ingr(
# input_['recipe']['ingrs'],
# emb_instrs=mean_instrs.unsqueeze(0))
# ingr_emb = ingr_emb.data.cpu()
# ingr_emb = ingr_emb.expand_as(img_embs)
Logger()('Fast distance...')
if modality_to_modality == 'image_to_recipe':
dist = fast_distance(img_embs, rcp_embs)
elif modality_to_modality == 'recipe_to_image':
dist = fast_distance(rcp_embs, img_embs)
elif modality_to_modality == 'recipe_to_recipe':
dist = fast_distance(rcp_embs, rcp_embs)
elif modality_to_modality == 'image_to_image':
dist = fast_distance(img_embs, img_embs)
dist=dist[:, 0]
sorted_ids = np.argsort(dist.numpy())
Logger()('Load/save images in {}...'.format(dir_visu))
for i in range(20):
idx = int(sorted_ids[i])
item_id = list_idx[idx]
#item_id = idx
item = dataset[item_id]
write_img_rcp(dir_visu, item, top=i)
#os.system('cp {} {}'.format(path_img_from, path_img_to))
Logger()('End')
def main():
Logger('.')
#classes = ['pizza', 'pork chops', 'cupcake', 'hamburger', 'green beans']
nb_points = 1000
split = 'test'
dir_exp = 'logs/recipe1m/trijoint/2017-12-14-15-04-51'
path_opts = os.path.join(dir_exp, 'options.yaml')
dir_extract = os.path.join(dir_exp, 'extract', split)
dir_img = os.path.join(dir_extract, 'image')
dir_rcp = os.path.join(dir_extract, 'recipe')
path_model_ckpt = os.path.join(
dir_exp,
'ckpt_best_val_epoch.metric.recall_at_1_im2recipe_mean_model.pth.tar')
#Options(path_opts)
Options.load_from_yaml(path_opts)
utils.set_random_seed(Options()['misc']['seed'])
dataset = factory(split)
Logger()('Load model...')
model = model_factory()
model_state = torch.load(path_model_ckpt)
model.load_state_dict(model_state)
model.set_mode(split)
if not os.path.isdir(dir_extract):
os.system('mkdir -p ' + dir_rcp)
os.system('mkdir -p ' + dir_img)
for i in tqdm(range(len(dataset))):
item = dataset[i]
batch = dataset.items_tf()([item])
if model.is_cuda:
batch = model.cuda_tf()(batch)
is_volatile = (model.mode not in ['train', 'trainval'])
batch = model.variable_tf(volatile=is_volatile)(batch)
out = model.network(batch)
path_image = os.path.join(dir_img, '{}.pth'.format(i))
path_recipe = os.path.join(dir_rcp, '{}.pth'.format(i))
torch.save(out['image_embedding'][0].data.cpu(), path_image)
torch.save(out['recipe_embedding'][0].data.cpu(), path_recipe)
# b = dataset.make_batch_loader().__iter__().__next__()
# class_name = 'pizza'
# ingrs = torch.LongTensor(1, 2)
# ingrs[0, 0] = dataset.recipes_dataset.ingrname_to_ingrid['mushrooms']
# ingrs[0, 1] = dataset.recipes_dataset.ingrname_to_ingrid['mushroom']
class_name = 'hamburger'
ingrs = torch.LongTensor(1, 2)
ingrs[0, 0] = dataset.recipes_dataset.ingrname_to_ingrid['mushroom']
ingrs[0, 1] = dataset.recipes_dataset.ingrname_to_ingrid['mushrooms']
#ingrs[0, 0] = dataset.recipes_dataset.ingrname_to_ingrid['tomato']
#ingrs[0, 1] = dataset.recipes_dataset.ingrname_to_ingrid['salad']
#ingrs[0, 2] = dataset.recipes_dataset.ingrname_to_ingrid['onion']
#ingrs[0, 3] = dataset.recipes_dataset.ingrname_to_ingrid['chicken']
input_ = {
'recipe': {
'ingrs': {
'data': Variable(ingrs.cuda(), requires_grad=False),
'lengths': [ingrs.size(1)]
},
'instrs': {
'data':
Variable(torch.FloatTensor(1, 1, 1024).fill_(0).cuda(),
requires_grad=False),
'lengths': [1]
}
}
}
#emb = network.recipe_embedding.forward_ingrs(input_['recipe']['ingrs'])
#list_idx = torch.randperm(len(dataset))
indices_by_class = dataset._make_indices_by_class()
class_id = dataset.cname_to_cid[class_name]
list_idx = torch.Tensor(indices_by_class[class_id])
rand_idx = torch.randperm(list_idx.size(0))
list_idx = list_idx[rand_idx]
list_idx = list_idx.view(-1).int()
img_embs = []
rcp_embs = []
if nb_points > list_idx.size(0):
nb_points = list_idx.size(0)
Logger()('Load {} embeddings...'.format(nb_points))
for i in range(nb_points):
idx = list_idx[i]
path_img = os.path.join(dir_img, '{}.pth'.format(idx))
path_rcp = os.path.join(dir_rcp, '{}.pth'.format(idx))
if not os.path.isfile(path_img):
Logger()('No such file: {}'.format(path_img))
continue
if not os.path.isfile(path_rcp):
Logger()('No such file: {}'.format(path_rcp))
continue
img_embs.append(torch.load(path_img))
rcp_embs.append(torch.load(path_rcp))
img_embs = torch.stack(img_embs, 0)
rcp_embs = torch.stack(rcp_embs, 0)
Logger()('Forward ingredient...')
#ingr_emb = model.network.recipe_embedding(input_['recipe'])
ingr_emb = model.network.recipe_embedding.forward_one_ingr(
input_['recipe']['ingrs'])
ingr_emb = ingr_emb.data.cpu()
ingr_emb = ingr_emb.expand_as(img_embs)
Logger()('Fast distance...')
dist = fast_distance(img_embs, ingr_emb)[:, 0]
sorted_ids = np.argsort(dist.numpy())
dir_visu = os.path.join(dir_exp, 'visu',
'ingrs_to_image_by_class_{}'.format(class_name))
os.system('mkdir -p ' + dir_visu)
Logger()('Load/save images...')
for i in range(20):
idx = int(sorted_ids[i])
item_id = list_idx[idx]
item = dataset[item_id]
Logger()(item['recipe']['class_name'])
Logger()(item['image']['class_name'])
path_img_from = item['image']['path']
path_img_to = os.path.join(dir_visu, 'image_top_{}.png'.format(i + 1))
img = Image.open(path_img_from)
img.save(path_img_to)
#os.system('cp {} {}'.format(path_img_from, path_img_to))
Logger()('End')
