def pbt(path_opts=None):
global pop
# first call to Options() load the options yaml file from --path_opts command line argument if path_opts=None
Options(path_opts)
# make exp directory if --exp.resume is empty
init_experiment_directory(Options()['exp']['dir'],
Options()['exp']['resume'])
# initialiaze seeds to be able to reproduce experiment on reload
utils.set_random_seed(Options()['misc']['seed'])
# display and save options as yaml file in exp dir
Logger().log_dict('options', Options(), should_print=True)
pop = Population()
pop.create_workers()
time.sleep(5)
if Options()['exp']['resume'] is None:
pop.populate()
else:
pop.resume()
pop.stop_workers()
def application(request):
utils.set_random_seed(Options()['misc']['seed'])
if 'image' not in request.form:
return Response('"image" POST field is missing')
if 'mode' not in request.form:
return Response('"mode" POST field is missing')
if 'top' not in request.form:
return Response('"top" POST field is missing')
if request.form['mode'] not in ['recipe', 'image', 'all']:
return Response('"mode" must be equals to ' +
' | '.join(['recipe', 'image', 'all']))
pil_img = decode_image(request.form['image'])
out = process_image(pil_img,
mode=request.form['mode'],
top=int(request.form['top']))
out = json.dumps(out)
response = Response(out)
response.headers.add('Access-Control-Allow-Origin', '*')
response.headers.add('Access-Control-Allow-Methods',
'GET,PUT,POST,DELETE,PATCH')
response.headers.add('Access-Control-Allow-Headers',
'Content-Type, Authorization')
response.headers.add('X-XSS-Protection', '0')
return response
def main():
Logger('.')
#classes = ['pizza', 'pork chops', 'cupcake', 'hamburger', 'green beans']
split = 'test'
dir_exp = '/home/cadene/doc/bootstrap.pytorch/logs/recipe1m/trijoint/2017-12-14-15-04-51'
path_opts = os.path.join(dir_exp, 'options.yaml')
dir_extract = os.path.join(dir_exp, 'extract_mean_features', split)
dir_img = os.path.join(dir_extract, 'image')
dir_rcp = os.path.join(dir_extract, 'recipe')
path_model_ckpt = os.path.join(
dir_exp,
'ckpt_best_val_epoch.metric.recall_at_1_im2recipe_mean_model.pth.tar')
Options.load_from_yaml(path_opts)
utils.set_random_seed(Options()['misc']['seed'])
Logger()('Load dataset...')
dataset = factory(split)
Logger()('Load model...')
model = model_factory()
model_state = torch.load(path_model_ckpt)
model.load_state_dict(model_state)
model.set_mode(split)
if not os.path.isdir(dir_extract):
Logger()('Create extract_dir {}'.format(dir_extract))
os.system('mkdir -p ' + dir_extract)
mean_ingrs = torch.zeros(model.network.recipe_embedding.dim_ingr_out *
2) # bi LSTM
mean_instrs = torch.zeros(model.network.recipe_embedding.dim_instr_out)
for i in tqdm(range(len(dataset))):
item = dataset[i]
batch = dataset.items_tf()([item])
batch = model.prepare_batch(batch)
out_ingrs = model.network.recipe_embedding.forward_ingrs(
batch['recipe']['ingrs'])
out_instrs = model.network.recipe_embedding.forward_instrs(
batch['recipe']['instrs'])
mean_ingrs += out_ingrs.data.cpu().squeeze(0)
mean_instrs += out_instrs.data.cpu().squeeze(0)
mean_ingrs /= len(dataset)
mean_instrs /= len(dataset)
path_ingrs = os.path.join(dir_extract, 'ingrs.pth')
path_instrs = os.path.join(dir_extract, 'instrs.pth')
torch.save(mean_ingrs, path_ingrs)
torch.save(mean_instrs, path_instrs)
Logger()('End')
def extract(path_opts=None):
Options(path_opts)
utils.set_random_seed(Options()['misc']['seed'])
assert Options(
)['dataset']['eval_split'] is not None, 'eval_split must be set'
assert Options(
)['dataset']['train_split'] is None, 'train_split must be None'
init_logs_options_files(Options()['exp']['dir'],
Options()['exp']['resume'])
Logger().log_dict('options', Options(), should_print=True)
Logger()(os.uname())
if torch.cuda.is_available():
cudnn.benchmark = True
Logger()('Available GPUs: {}'.format(utils.available_gpu_ids()))
engine = engines.factory()
engine.dataset = datasets.factory(engine)
engine.model = models.factory(engine)
engine.view = views.factory(engine)
# init extract directory
dir_extract = os.path.join(Options()['exp']['dir'], 'extract',
Options()['dataset']['eval_split'])
os.system('mkdir -p ' + dir_extract)
path_img_embs = os.path.join(dir_extract, 'image_emdeddings.pth')
path_rcp_embs = os.path.join(dir_extract, 'recipe_emdeddings.pth')
img_embs = torch.FloatTensor(len(engine.dataset['eval']),
Options()['model.network.dim_emb'])
rcp_embs = torch.FloatTensor(len(engine.dataset['eval']),
Options()['model.network.dim_emb'])
def save_embeddings(module, input, out):
nonlocal img_embs
nonlocal rcp_embs
batch = input[0] # tuple of len=1
for j, idx in enumerate(batch['recipe']['index']):
# path_image = os.path.join(dir_image, '{}.pth'.format(idx))
# path_recipe = os.path.join(dir_recipe, '{}.pth'.format(idx))
# torch.save(out['image_embedding'][j].data.cpu(), path_image)
# torch.save(out['recipe_embedding'][j].data.cpu(), path_recipe)
img_embs[idx] = out['image_embedding'][j].data.cpu()
rcp_embs[idx] = out['recipe_embedding'][j].data.cpu()
engine.model.register_forward_hook(save_embeddings)
engine.resume()
engine.eval()
Logger()('Saving image embeddings to {} ...'.format(path_img_embs))
torch.save(img_embs, path_img_embs)
Logger()('Saving recipe embeddings to {} ...'.format(path_rcp_embs))
torch.save(rcp_embs, path_rcp_embs)
def main(path_opts=None):
# first call to Options() load the options yaml file from --path_opts command line argument if path_opts=None
Options(path_opts)
# make exp directory if resume=None
init_experiment_directory(Options()['exp']['dir'],
Options()['exp']['resume'])
# initialiaze seeds to be able to reproduce experiment on reload
utils.set_random_seed(Options()['misc']['seed'])
# display and save options as yaml file in exp dir
Logger().log_dict('options', Options(), should_print=True)
# display server name and GPU(s) id(s)
Logger()(os.uname())
if 'CUDA_VISIBLE_DEVICES' in os.environ:
Logger()('CUDA_VISIBLE_DEVICES=' + os.environ['CUDA_VISIBLE_DEVICES'])
# engine can train, eval, optimize the model
# engine can save and load the model and optimizer
engine = engines.factory()
# dataset is a dictionnary which contains all the needed datasets indexed by splits
# (example: dataset.keys() -> ['train','val'])
dataset = datasets.factory()
engine.dataset = dataset
# model includes a network, a criterion and a metric
# model can register engine hooks (begin epoch, end batch, end batch, etc.)
# (example: "calculate mAP at the end of the evaluation epoch")
model = models.factory(engine)
engine.model = model
# optimizer can register engine hooks
optimizer = optimizers.factory(model, engine)
engine.optimizer = optimizer
# load the model and optimizer from a checkpoint
if Options()['exp']['resume']:
engine.resume()
# if no training split, evaluate the model on the evaluation split
# (example: $ python main.py --dataset.train_split --dataset.eval_split test)
engine.eval()
# optimize the model on the training split for several epochs
# (example: $ python main.py --dataset.train_split train)
# if evaluation split, evaluate the model after each epochs
# (example: $ python main.py --dataset.train_split train --dataset.eval_split val)
engine.train()
def api(path_opts=None):
global engine
global img_embs
global rcp_embs
global all_embs
global all_ids
Options(path_opts)
utils.set_random_seed(Options()['misc']['seed'])
assert Options(
)['dataset']['eval_split'] is not None, 'eval_split must be set'
assert Options(
)['dataset']['train_split'] is None, 'train_split must be None'
init_logs_options_files(Options()['exp']['dir'],
Options()['exp']['resume'])
Logger().log_dict('options', Options(), should_print=True)
Logger()(os.uname())
if torch.cuda.is_available():
cudnn.benchmark = True
Logger()('Available GPUs: {}'.format(utils.available_gpu_ids()))
engine = engines.factory()
engine.dataset = datasets.factory(engine)
engine.model = models.factory(engine)
engine.model.eval()
engine.resume()
dir_extract = os.path.join(Options()['exp']['dir'], 'extract',
Options()['dataset']['eval_split'])
path_img_embs = os.path.join(dir_extract, 'image_emdeddings.pth')
path_rcp_embs = os.path.join(dir_extract, 'recipe_emdeddings.pth')
img_embs = torch.load(path_img_embs)
rcp_embs = torch.load(path_rcp_embs)
all_embs = torch.cat([img_embs, rcp_embs], dim=0)
all_ids = list(range(img_embs.shape[0])) + list(range(rcp_embs.shape[0]))
my_local_ip = '132.227.204.160' # localhost | 192.168.0.41 (hostname --ip-address)
my_local_port = 3456 # 8080 | 3456
run_simple(my_local_ip, my_local_port, application)
def main():
Logger('.')
split = 'train'
dir_exp = '/home/ubuntu/moochi/recipe1m.bootstrap.pytorch/logs/recipe1m/trijoint/2017-12-14-15-04-51'
path_opts = os.path.join(dir_exp, 'options.yaml')
dir_extract = os.path.join(dir_exp, 'extract_count', split)
path_ingrs_count = os.path.join(dir_extract, 'ingrs.pth')
Options(path_opts)
utils.set_random_seed(Options()['misc']['seed'])
dataset = factory(split)
if not os.path.isfile(path_ingrs_count):
ingrs_count = {}
os.system('mkdir -p ' + dir_extract)
for i in tqdm(range(len(dataset.recipes_dataset))):
item = dataset.recipes_dataset[i]
for ingr in item['ingrs']['interim']:
if ingr not in ingrs_count:
ingrs_count[ingr] = 1
else:
ingrs_count[ingr] += 1
torch.save(ingrs_count, path_ingrs_count)
else:
ingrs_count = torch.load(path_ingrs_count)
import ipdb
ipdb.set_trace()
sort = sorted(ingrs_count, key=ingrs_count.get)
import ipdb
ipdb.set_trace()
Logger()('End')
def main():
Logger('.')
#classes = ['hamburger']
#nb_points =
split = 'test'
class_name = None#'potato salad'
modality_to_modality = 'recipe_to_image'
dir_exp = '/home/cadene/doc/bootstrap.pytorch/logs/recipe1m/trijoint/2017-12-14-15-04-51'
path_opts = os.path.join(dir_exp, 'options.yaml')
dir_extract = os.path.join(dir_exp, 'extract', split)
dir_extract_mean = os.path.join(dir_exp, 'extract_mean_features', split)
dir_img = os.path.join(dir_extract, 'image')
dir_rcp = os.path.join(dir_extract, 'recipe')
path_model_ckpt = os.path.join(dir_exp, 'ckpt_best_val_epoch.metric.recall_at_1_im2recipe_mean_model.pth.tar')
#is_mean = True
#ingrs_list = ['carotte', 'salad', 'tomato']#['avocado']
#Options(path_opts)
Options(path_opts)
Options()['misc']['seed'] = 11
utils.set_random_seed(Options()['misc']['seed'])
chosen_item_id = 51259
dataset = factory(split)
if class_name:
class_id = dataset.cname_to_cid[class_name]
indices_by_class = dataset._make_indices_by_class()
nb_points = len(indices_by_class[class_id])
list_idx = torch.Tensor(indices_by_class[class_id])
rand_idx = torch.randperm(list_idx.size(0))
list_idx = list_idx[rand_idx]
list_idx = list_idx.view(-1).int()
dir_visu = os.path.join(dir_exp, 'visu', 'remove_ingrs_item:{}_nb_points:{}_class:{}'.format(chosen_item_id, nb_points, class_name.replace(' ', '_')))
else:
nb_points = 1000
list_idx = torch.randperm(len(dataset))
dir_visu = os.path.join(dir_exp, 'visu', 'remove_ingrs_item:{}_nb_points:{}_removed'.format(chosen_item_id, nb_points))
# for i in range(20):
# item_id = list_idx[i]
# item = dataset[item_id]
# write_img_rcp(dir_visu, item, top=i)
Logger()('Load model...')
model = model_factory()
model_state = torch.load(path_model_ckpt)
model.load_state_dict(model_state)
model.eval()
item = dataset[chosen_item_id]
# from tqdm import tqdm
# ids = []
# for i in tqdm(range(len(dataset.recipes_dataset))):
# item = dataset.recipes_dataset[i]#23534]
# if 'broccoli' in item['ingrs']['interim']:
# print('broccoli', i)
# ids.append(i)
# # if 'mushroom' in item['ingrs']['interim']:
# # print('mushroom', i)
# # break
# import ipdb; ipdb.set_trace()
# input_ = {
# 'recipe': {
# 'ingrs': {
# 'data': item['recipe']['ingrs']['data'],
# 'lengths': item['recipe']['ingrs']['lengths']
# },
# 'instrs': {
# 'data': item['recipe']['instrs']['data'],
# 'lengths': item['recipe']['instrs']['lengths']
# }
# }
# }
instrs = torch.FloatTensor(6,1024)
instrs[0] = item['recipe']['instrs']['data'][0]
instrs[1] = item['recipe']['instrs']['data'][1]
instrs[2] = item['recipe']['instrs']['data'][3]
instrs[3] = item['recipe']['instrs']['data'][4]
instrs[4] = item['recipe']['instrs']['data'][6]
instrs[5] = item['recipe']['instrs']['data'][7]
ingrs = torch.LongTensor([612,585,844,3087,144,188,1])
input_ = {
'recipe': {
'ingrs': {
'data': ingrs,
'lengths': ingrs.size(0)
},
'instrs': {
'data': instrs,
'lengths': instrs.size(0)
}
}
}
batch = dataset.items_tf()([input_])
batch = model.prepare_batch(batch)
out = model.network.recipe_embedding(batch['recipe'])
# path_rcp = os.path.join(dir_rcp, '{}.pth'.format(23534))
# rcp_emb = torch.load(path_rcp)
Logger()('Load embeddings...')
img_embs = []
for i in range(nb_points):
try:
idx = list_idx[i]
except:
import ipdb; ipdb.set_trace()
#idx = i
path_img = os.path.join(dir_img, '{}.pth'.format(idx))
if not os.path.isfile(path_img):
Logger()('No such file: {}'.format(path_img))
continue
img_embs.append(torch.load(path_img))
img_embs = torch.stack(img_embs, 0)
Logger()('Fast distance...')
dist = fast_distance(out.data.cpu().expand_as(img_embs), img_embs)
dist = dist[0, :]
sorted_ids = np.argsort(dist.numpy())
os.system('rm -rf '+dir_visu)
os.system('mkdir -p '+dir_visu)
Logger()('Load/save images in {}...'.format(dir_visu))
write_img_rcp(dir_visu, item, top=0, begin_with='query')
for i in range(20):
idx = int(sorted_ids[i])
item_id = list_idx[idx]
#item_id = idx
item = dataset[item_id]
write_img_rcp(dir_visu, item, top=i, begin_with='nn')
Logger()('End')
def main():
Logger('.')
#classes = ['pizza', 'pork chops', 'cupcake', 'hamburger', 'green beans']
nb_points = 1000
split = 'test'
dir_exp = '/home/cadene/doc/bootstrap.pytorch/logs/recipe1m/trijoint/2017-12-14-15-04-51'
path_opts = os.path.join(dir_exp, 'options.yaml')
dir_extract = os.path.join(dir_exp, 'extract', split)
dir_extract_mean = os.path.join(dir_exp, 'extract_mean_features', split)
dir_img = os.path.join(dir_extract, 'image')
dir_rcp = os.path.join(dir_extract, 'recipe')
path_model_ckpt = os.path.join(
dir_exp,
'ckpt_best_val_epoch.metric.recall_at_1_im2recipe_mean_model.pth.tar')
dir_visu = os.path.join(dir_exp, 'visu', 'mean_to_image')
os.system('mkdir -p ' + dir_visu)
#Options(path_opts)
Options.load_from_yaml(path_opts)
utils.set_random_seed(Options()['misc']['seed'])
dataset = factory(split)
Logger()('Load model...')
model = model_factory()
model_state = torch.load(path_model_ckpt)
model.load_state_dict(model_state)
model.set_mode(split)
#emb = network.recipe_embedding.forward_ingrs(input_['recipe']['ingrs'])
list_idx = torch.randperm(len(dataset))
Logger()('Load embeddings...')
img_embs = []
rcp_embs = []
for i in range(nb_points):
idx = list_idx[i]
path_img = os.path.join(dir_img, '{}.pth'.format(idx))
path_rcp = os.path.join(dir_rcp, '{}.pth'.format(idx))
if not os.path.isfile(path_img):
Logger()('No such file: {}'.format(path_img))
continue
if not os.path.isfile(path_rcp):
Logger()('No such file: {}'.format(path_rcp))
continue
img_embs.append(torch.load(path_img))
rcp_embs.append(torch.load(path_rcp))
img_embs = torch.stack(img_embs, 0)
rcp_embs = torch.stack(rcp_embs, 0)
Logger()('Load means')
path_ingrs = os.path.join(dir_extract_mean, 'ingrs.pth')
path_instrs = os.path.join(dir_extract_mean, 'instrs.pth')
mean_ingrs = torch.load(path_ingrs)
mean_instrs = torch.load(path_instrs)
mean_ingrs = Variable(mean_ingrs.unsqueeze(0).cuda(), requires_grad=False)
mean_instrs = Variable(mean_instrs.unsqueeze(0).cuda(),
requires_grad=False)
Logger()('Forward ingredient...')
ingr_emb = model.network.recipe_embedding.forward_ingrs_instrs(
mean_ingrs, mean_instrs)
ingr_emb = ingr_emb.data.cpu()
ingr_emb = ingr_emb.expand_as(img_embs)
Logger()('Fast distance...')
dist = fast_distance(img_embs, ingr_emb)[:, 0]
sorted_img_ids = np.argsort(dist.numpy())
Logger()('Load/save images...')
for i in range(20):
img_id = sorted_img_ids[i]
img_id = int(img_id)
path_img_from = dataset[img_id]['image']['path']
path_img_to = os.path.join(dir_visu, 'image_top_{}.png'.format(i + 1))
img = Image.open(path_img_from)
img.save(path_img_to)
#os.system('cp {} {}'.format(path_img_from, path_img_to))
Logger()('End')
def main():
Logger('.')
#classes = ['pizza', 'pork chops', 'cupcake', 'hamburger', 'green beans']
nb_points = 1000
split = 'test'
class_name = 'pizza'
dir_exp = '/home/ubuntu/moochi/recipe1m.bootstrap.pytorch/logs/recipe1m/trijoint/2017-12-14-15-04-51'
path_opts = os.path.join(dir_exp, 'options.yaml')
dir_extract = os.path.join(dir_exp, 'extract', split)
dir_extract_mean = os.path.join(dir_exp, 'extract_mean_features', split)
dir_img = os.path.join(dir_extract, 'image')
dir_rcp = os.path.join(dir_extract, 'recipe')
path_model_ckpt = os.path.join(
dir_exp,
'ckpt_best_val_epoch.metric.recall_at_1_im2recipe_mean_model.pth.tar')
is_mean = True
ingrs_list = ['fresh_strawberries'] #['avocado']
Options(path_opts)
Options()['misc']['seed'] = 2
utils.set_random_seed(Options()['misc']['seed'])
dataset = factory(split)
Logger()('Load model...')
model = model_factory()
model_state = torch.load(path_model_ckpt)
model.load_state_dict(model_state)
model.eval()
if not os.path.isdir(dir_extract):
os.system('mkdir -p ' + dir_rcp)
os.system('mkdir -p ' + dir_img)
for i in tqdm(range(len(dataset))):
item = dataset[i]
batch = dataset.items_tf()([item])
if model.is_cuda:
batch = model.cuda_tf()(batch)
is_volatile = (model.mode not in ['train', 'trainval'])
batch = model.variable_tf(volatile=is_volatile)(batch)
out = model.network(batch)
path_image = os.path.join(dir_img, '{}.pth'.format(i))
path_recipe = os.path.join(dir_rcp, '{}.pth'.format(i))
torch.save(out['image_embedding'][0].data.cpu(), path_image)
torch.save(out['recipe_embedding'][0].data.cpu(), path_recipe)
# b = dataset.make_batch_loader().__iter__().__next__()
# import ipdb; ipdb.set_trace()
ingrs = torch.LongTensor(1, len(ingrs_list))
for i, ingr_name in enumerate(ingrs_list):
ingrs[0, i] = dataset.recipes_dataset.ingrname_to_ingrid[ingr_name]
input_ = {
'recipe': {
'ingrs': {
'data': Variable(ingrs.cuda(), requires_grad=False),
'lengths': [ingrs.size(1)]
},
'instrs': {
'data':
Variable(torch.FloatTensor(1, 1, 1024).fill_(0).cuda(),
requires_grad=False),
'lengths': [1]
}
}
}
#emb = network.recipe_embedding.forward_ingrs(input_['recipe']['ingrs'])
#list_idx = torch.randperm(len(dataset))
indices_by_class = dataset._make_indices_by_class()
#import ipdb; ipdb.set_trace()
class_id = dataset.cname_to_cid[class_name]
list_idx = torch.Tensor(indices_by_class[class_id])
rand_idx = torch.randperm(list_idx.size(0))
list_idx = list_idx[rand_idx]
list_idx = list_idx.view(-1).int()
nb_points = list_idx.size(0)
dir_visu = os.path.join(
dir_exp, 'visu',
'ingrs_to_image_nb_points:{}_class:{}_instrs:{}_mean:{}_v2'.format(
nb_points, class_name, '-'.join(ingrs_list), is_mean))
os.system('mkdir -p ' + dir_visu)
Logger()('Load embeddings...')
img_embs = []
rcp_embs = []
for i in range(nb_points):
idx = list_idx[i]
path_img = os.path.join(dir_img, '{}.pth'.format(idx))
path_rcp = os.path.join(dir_rcp, '{}.pth'.format(idx))
if not os.path.isfile(path_img):
Logger()('No such file: {}'.format(path_img))
continue
if not os.path.isfile(path_rcp):
Logger()('No such file: {}'.format(path_rcp))
continue
img_embs.append(torch.load(path_img))
rcp_embs.append(torch.load(path_rcp))
img_embs = torch.stack(img_embs, 0)
rcp_embs = torch.stack(rcp_embs, 0)
Logger()('Load mean embeddings')
path_ingrs = os.path.join(dir_extract_mean, 'ingrs.pth')
path_instrs = os.path.join(dir_extract_mean, 'instrs.pth')
mean_ingrs = torch.load(path_ingrs)
mean_instrs = torch.load(path_instrs)
Logger()('Forward ingredient...')
#ingr_emb = model.network.recipe_embedding(input_['recipe'])
ingr_emb = model.network.recipe_embedding.forward_one_ingr(
input_['recipe']['ingrs'], emb_instrs=mean_instrs.unsqueeze(0))
ingr_emb = ingr_emb.data.cpu()
ingr_emb = ingr_emb.expand_as(img_embs)
Logger()('Fast distance...')
dist = fast_distance(img_embs, ingr_emb)[:, 0]
sorted_ids = np.argsort(dist.numpy())
Logger()('Load/save images in {}...'.format(dir_visu))
for i in range(20):
idx = int(sorted_ids[i])
item_id = list_idx[idx]
item = dataset[item_id]
# path_img_from = item['image']['path']
# ingrs = [ingr.replace('/', '\'') for ingr in item['recipe']['ingrs']['interim']]
# cname = item['recipe']['class_name']
# path_img_to = os.path.join(dir_visu, 'image_top:{}_ingrs:{}_cname:{}.png'.format(i+1, '-'.join(ingrs), cname))
# img = Image.open(path_img_from)
# img.save(path_img_to)
#os.system('cp {} {}'.format(path_img_from, path_img_to))
write_img_rcp(dir_visu, item, top=i, begin_with='nn')
Logger()('End')
def eval_epoch(self, model, dataset, epoch, mode='eval', logs_json=True):
""" Launch evaluation procedures for one epoch
List of the hooks (``mode='eval'`` by default):
- mode_on_start_epoch: before the evaluation procedure for an epoch
- mode_on_start_batch: before the evaluation precedure for a batch
- mode_on_forward: after the forward of the model
- mode_on_print: after the print to the terminal
- mode_on_end_batch: end of the evaluation procedure for a batch
- mode_on_end_epoch: before saving the logs in logs.json
- mode_on_flush: end of the evaluation procedure for an epoch
Returns:
out(dict): mean of all the scalar outputs of the model, indexed by output name, for this epoch
"""
utils.set_random_seed(Options()['misc']['seed'] +
epoch) # to be able to reproduce exps on reload
Logger()('Evaluating model on {}set for epoch {}'.format(
dataset.split, epoch))
model.eval()
model.set_mode(mode)
timer = {
'begin': time.time(),
'elapsed': time.time(),
'process': None,
'load': None,
'run_avg': 0
}
out_epoch = {}
batch_loader = dataset.make_batch_loader()
self.hook('{}_on_start_epoch'.format(mode))
for i, batch in enumerate(batch_loader):
timer['load'] = time.time() - timer['elapsed']
self.hook('{}_on_start_batch'.format(mode))
with torch.no_grad():
out = model(batch)
if 'loss' in out and torch.isnan(out['loss']):
torch.cuda.synchronize()
del out
Logger()('NaN detected')
import gc
gc.collect()
torch.cuda.empty_cache()
torch.cuda.synchronize()
continue
self.hook('{}_on_forward'.format(mode))
timer['process'] = time.time() - timer['elapsed']
if i == 0:
timer['run_avg'] = timer['process']
else:
timer['run_avg'] = timer['run_avg'] * 0.8 + timer[
'process'] * 0.2
Logger().log_value('{}_batch.batch'.format(mode),
i,
should_print=False)
Logger().log_value('{}_batch.epoch'.format(mode),
epoch,
should_print=False)
Logger().log_value('{}_batch.timer.process'.format(mode),
timer['process'],
should_print=False)
Logger().log_value('{}_batch.timer.load'.format(mode),
timer['load'],
should_print=False)
for key, value in out.items():
if torch.is_tensor(value):
if value.dim() <= 1:
value = value.item() # get number from a torch scalar
else:
continue
if type(value) == list:
continue
if type(value) == dict:
continue
if value is None:
continue
if key not in out_epoch:
out_epoch[key] = []
out_epoch[key].append(value)
Logger().log_value('{}_batch.{}'.format(mode, key),
value,
should_print=False)
if i % Options()['engine']['print_freq'] == 0:
Logger()("{}: epoch {} | batch {}/{}".format(
mode, epoch, i,
len(batch_loader) - 1))
Logger()("{} elapsed: {} | left: {}".format(
' ' * len(mode),
datetime.timedelta(seconds=math.floor(time.time() -
timer['begin'])),
datetime.timedelta(
seconds=math.floor(timer['run_avg'] *
(len(batch_loader) - 1 - i)))))
Logger()("{} process: {:.5f} | load: {:.5f}".format(
' ' * len(mode), timer['process'], timer['load']))
self.hook('{}_on_print'.format(mode))
timer['elapsed'] = time.time()
self.hook('{}_on_end_batch'.format(mode))
if Options()['engine']['debug']:
if i > 10:
break
out = {}
for key, value in out_epoch.items():
try:
out[key] = sum(value) / len(value)
except:
import ipdb
ipdb.set_trace()
Logger().log_value('{}_epoch.epoch'.format(mode),
epoch,
should_print=True)
for key, value in out.items():
Logger().log_value('{}_epoch.{}'.format(mode, key),
value,
should_print=True)
self.hook('{}_on_end_epoch'.format(mode))
if logs_json:
Logger().flush()
self.hook('{}_on_flush'.format(mode))
return out
import torch
import bootstrap.lib.utils as utils
import bootstrap.engines as engines
import bootstrap.models as models
import bootstrap.datasets as datasets
from bootstrap.lib.logger import Logger
from bootstrap.lib.options import Options
from bootstrap.run import init_logs_options_files
from block.datasets.vqa_utils import tokenize_mcb
if __name__ == "__main__":
Options()
#print("seed: {}".format(Options()['misc']['seed']))
utils.set_random_seed(Options()['misc']['seed'])
init_logs_options_files(Options()['exp']['dir'],
Options()['exp']['resume'])
engine = engines.factory()
#print("engine: {}".format(engine))
engine.dataset = datasets.factory(engine)
engine.model = models.factory(engine)
#print("engine.model: {}".format(engine.model))
engine.model.eval()
engine.resume()
#img_emb_module = load_img_emb_module().cuda() #engine.model.network.image_embedding
# inputs
img = torch.randn(1, 3, 224, 244).cuda()
question = 'What is the color of the white horse?'
question_words = tokenize_mcb(question)
def main():
Logger('.')
import argparse
parser = argparse.ArgumentParser()
parser.add_argument('modality_to_modality', help='foo help', default='recipe_to_image')
args = parser.parse_args()
#classes = ['pizza', 'pork chops', 'cupcake', 'hamburger', 'green beans']
nb_points = 1000
modality_to_modality = args.modality_to_modality#'image_to_image'
print(modality_to_modality)
split = 'test'
dir_exp = '/home/ubuntu/moochi/recipe1m.bootstrap.pytorch/logs/recipe1m/trijoint/2017-12-14-15-04-51'
path_opts = os.path.join(dir_exp, 'options.yaml')
dir_extract = os.path.join(dir_exp, 'extract', split)
dir_img = os.path.join(dir_extract, 'image')
dir_rcp = os.path.join(dir_extract, 'recipe')
path_model_ckpt = os.path.join(dir_exp, 'ckpt_best_val_epoch.metric.recall_at_1_im2recipe_mean_model.pth.tar')
Options.load_from_yaml(path_opts)
Options()['misc']['seed'] = 11
utils.set_random_seed(Options()['misc']['seed'])
dataset = factory(split)
Logger()('Load model...')
model = model_factory()
model_state = torch.load(path_model_ckpt)
model.load_state_dict(model_state)
model.eval()
if not os.path.isdir(dir_extract):
os.system('mkdir -p '+dir_rcp)
os.system('mkdir -p '+dir_img)
for i in tqdm(range(len(dataset))):
item = dataset[i]
batch = dataset.items_tf()([item])
if model.is_cuda:
batch = model.cuda_tf()(batch)
is_volatile = (model.mode not in ['train', 'trainval'])
batch = model.variable_tf(volatile=is_volatile)(batch)
out = model.network(batch)
path_image = os.path.join(dir_img, '{}.pth'.format(i))
path_recipe = os.path.join(dir_rcp, '{}.pth'.format(i))
torch.save(out['image_embedding'][0].data.cpu(), path_image)
torch.save(out['recipe_embedding'][0].data.cpu(), path_recipe)
indices_by_class = dataset._make_indices_by_class()
# class_name = classes[0] # TODO
# class_id = dataset.cname_to_cid[class_name]
# list_idx = torch.Tensor(indices_by_class[class_id])
# rand_idx = torch.randperm(list_idx.size(0))
# list_idx = list_idx[rand_idx]
# list_idx = list_idx.view(-1).int()
list_idx = torch.randperm(len(dataset))
#nb_points = list_idx.size(0)
dir_visu = os.path.join(dir_exp, 'visu', '{}_top20_seed:{}'.format(modality_to_modality, Options()['misc']['seed']))
os.system('rm -rf '+dir_visu)
os.system('mkdir -p '+dir_visu)
Logger()('Load embeddings...')
img_embs = []
rcp_embs = []
for i in range(nb_points):
idx = list_idx[i]
#idx = i
path_img = os.path.join(dir_img, '{}.pth'.format(idx))
path_rcp = os.path.join(dir_rcp, '{}.pth'.format(idx))
if not os.path.isfile(path_img):
Logger()('No such file: {}'.format(path_img))
continue
if not os.path.isfile(path_rcp):
Logger()('No such file: {}'.format(path_rcp))
continue
img_embs.append(torch.load(path_img))
rcp_embs.append(torch.load(path_rcp))
img_embs = torch.stack(img_embs, 0)
rcp_embs = torch.stack(rcp_embs, 0)
# Logger()('Forward ingredient...')
# #ingr_emb = model.network.recipe_embedding(input_['recipe'])
# ingr_emb = model.network.recipe_embedding.forward_one_ingr(
# input_['recipe']['ingrs'],
# emb_instrs=mean_instrs.unsqueeze(0))
# ingr_emb = ingr_emb.data.cpu()
# ingr_emb = ingr_emb.expand_as(img_embs)
Logger()('Fast distance...')
if modality_to_modality == 'image_to_recipe':
dist = fast_distance(img_embs, rcp_embs)
elif modality_to_modality == 'recipe_to_image':
dist = fast_distance(rcp_embs, img_embs)
elif modality_to_modality == 'recipe_to_recipe':
dist = fast_distance(rcp_embs, rcp_embs)
elif modality_to_modality == 'image_to_image':
dist = fast_distance(img_embs, img_embs)
dist=dist[:, 0]
sorted_ids = np.argsort(dist.numpy())
Logger()('Load/save images in {}...'.format(dir_visu))
for i in range(20):
idx = int(sorted_ids[i])
item_id = list_idx[idx]
#item_id = idx
item = dataset[item_id]
write_img_rcp(dir_visu, item, top=i)
#os.system('cp {} {}'.format(path_img_from, path_img_to))
Logger()('End')
def main():
Logger('.')
#classes = ['pizza', 'pork chops', 'cupcake', 'hamburger', 'green beans']
nb_points = 1000
split = 'test'
dir_exp = 'logs/recipe1m/trijoint/2017-12-14-15-04-51'
path_opts = os.path.join(dir_exp, 'options.yaml')
dir_extract = os.path.join(dir_exp, 'extract', split)
dir_img = os.path.join(dir_extract, 'image')
dir_rcp = os.path.join(dir_extract, 'recipe')
path_model_ckpt = os.path.join(
dir_exp,
'ckpt_best_val_epoch.metric.recall_at_1_im2recipe_mean_model.pth.tar')
#Options(path_opts)
Options.load_from_yaml(path_opts)
utils.set_random_seed(Options()['misc']['seed'])
dataset = factory(split)
Logger()('Load model...')
model = model_factory()
model_state = torch.load(path_model_ckpt)
model.load_state_dict(model_state)
model.set_mode(split)
if not os.path.isdir(dir_extract):
os.system('mkdir -p ' + dir_rcp)
os.system('mkdir -p ' + dir_img)
for i in tqdm(range(len(dataset))):
item = dataset[i]
batch = dataset.items_tf()([item])
if model.is_cuda:
batch = model.cuda_tf()(batch)
is_volatile = (model.mode not in ['train', 'trainval'])
batch = model.variable_tf(volatile=is_volatile)(batch)
out = model.network(batch)
path_image = os.path.join(dir_img, '{}.pth'.format(i))
path_recipe = os.path.join(dir_rcp, '{}.pth'.format(i))
torch.save(out['image_embedding'][0].data.cpu(), path_image)
torch.save(out['recipe_embedding'][0].data.cpu(), path_recipe)
# b = dataset.make_batch_loader().__iter__().__next__()
# class_name = 'pizza'
# ingrs = torch.LongTensor(1, 2)
# ingrs[0, 0] = dataset.recipes_dataset.ingrname_to_ingrid['mushrooms']
# ingrs[0, 1] = dataset.recipes_dataset.ingrname_to_ingrid['mushroom']
class_name = 'hamburger'
ingrs = torch.LongTensor(1, 2)
ingrs[0, 0] = dataset.recipes_dataset.ingrname_to_ingrid['mushroom']
ingrs[0, 1] = dataset.recipes_dataset.ingrname_to_ingrid['mushrooms']
#ingrs[0, 0] = dataset.recipes_dataset.ingrname_to_ingrid['tomato']
#ingrs[0, 1] = dataset.recipes_dataset.ingrname_to_ingrid['salad']
#ingrs[0, 2] = dataset.recipes_dataset.ingrname_to_ingrid['onion']
#ingrs[0, 3] = dataset.recipes_dataset.ingrname_to_ingrid['chicken']
input_ = {
'recipe': {
'ingrs': {
'data': Variable(ingrs.cuda(), requires_grad=False),
'lengths': [ingrs.size(1)]
},
'instrs': {
'data':
Variable(torch.FloatTensor(1, 1, 1024).fill_(0).cuda(),
requires_grad=False),
'lengths': [1]
}
}
}
#emb = network.recipe_embedding.forward_ingrs(input_['recipe']['ingrs'])
#list_idx = torch.randperm(len(dataset))
indices_by_class = dataset._make_indices_by_class()
class_id = dataset.cname_to_cid[class_name]
list_idx = torch.Tensor(indices_by_class[class_id])
rand_idx = torch.randperm(list_idx.size(0))
list_idx = list_idx[rand_idx]
list_idx = list_idx.view(-1).int()
img_embs = []
rcp_embs = []
if nb_points > list_idx.size(0):
nb_points = list_idx.size(0)
Logger()('Load {} embeddings...'.format(nb_points))
for i in range(nb_points):
idx = list_idx[i]
path_img = os.path.join(dir_img, '{}.pth'.format(idx))
path_rcp = os.path.join(dir_rcp, '{}.pth'.format(idx))
if not os.path.isfile(path_img):
Logger()('No such file: {}'.format(path_img))
continue
if not os.path.isfile(path_rcp):
Logger()('No such file: {}'.format(path_rcp))
continue
img_embs.append(torch.load(path_img))
rcp_embs.append(torch.load(path_rcp))
img_embs = torch.stack(img_embs, 0)
rcp_embs = torch.stack(rcp_embs, 0)
Logger()('Forward ingredient...')
#ingr_emb = model.network.recipe_embedding(input_['recipe'])
ingr_emb = model.network.recipe_embedding.forward_one_ingr(
input_['recipe']['ingrs'])
ingr_emb = ingr_emb.data.cpu()
ingr_emb = ingr_emb.expand_as(img_embs)
Logger()('Fast distance...')
dist = fast_distance(img_embs, ingr_emb)[:, 0]
sorted_ids = np.argsort(dist.numpy())
dir_visu = os.path.join(dir_exp, 'visu',
'ingrs_to_image_by_class_{}'.format(class_name))
os.system('mkdir -p ' + dir_visu)
Logger()('Load/save images...')
for i in range(20):
idx = int(sorted_ids[i])
item_id = list_idx[idx]
item = dataset[item_id]
Logger()(item['recipe']['class_name'])
Logger()(item['image']['class_name'])
path_img_from = item['image']['path']
path_img_to = os.path.join(dir_visu, 'image_top_{}.png'.format(i + 1))
img = Image.open(path_img_from)
img.save(path_img_to)
#os.system('cp {} {}'.format(path_img_from, path_img_to))
Logger()('End')
