def pbt(path_opts=None): global pop # first call to Options() load the options yaml file from --path_opts command line argument if path_opts=None Options(path_opts) # make exp directory if --exp.resume is empty init_experiment_directory(Options()['exp']['dir'], Options()['exp']['resume']) # initialiaze seeds to be able to reproduce experiment on reload utils.set_random_seed(Options()['misc']['seed']) # display and save options as yaml file in exp dir Logger().log_dict('options', Options(), should_print=True) pop = Population() pop.create_workers() time.sleep(5) if Options()['exp']['resume'] is None: pop.populate() else: pop.resume() pop.stop_workers()
def application(request): utils.set_random_seed(Options()['misc']['seed']) if 'image' not in request.form: return Response('"image" POST field is missing') if 'mode' not in request.form: return Response('"mode" POST field is missing') if 'top' not in request.form: return Response('"top" POST field is missing') if request.form['mode'] not in ['recipe', 'image', 'all']: return Response('"mode" must be equals to ' + ' | '.join(['recipe', 'image', 'all'])) pil_img = decode_image(request.form['image']) out = process_image(pil_img, mode=request.form['mode'], top=int(request.form['top'])) out = json.dumps(out) response = Response(out) response.headers.add('Access-Control-Allow-Origin', '*') response.headers.add('Access-Control-Allow-Methods', 'GET,PUT,POST,DELETE,PATCH') response.headers.add('Access-Control-Allow-Headers', 'Content-Type, Authorization') response.headers.add('X-XSS-Protection', '0') return response
def main(): Logger('.') #classes = ['pizza', 'pork chops', 'cupcake', 'hamburger', 'green beans'] split = 'test' dir_exp = '/home/cadene/doc/bootstrap.pytorch/logs/recipe1m/trijoint/2017-12-14-15-04-51' path_opts = os.path.join(dir_exp, 'options.yaml') dir_extract = os.path.join(dir_exp, 'extract_mean_features', split) dir_img = os.path.join(dir_extract, 'image') dir_rcp = os.path.join(dir_extract, 'recipe') path_model_ckpt = os.path.join( dir_exp, 'ckpt_best_val_epoch.metric.recall_at_1_im2recipe_mean_model.pth.tar') Options.load_from_yaml(path_opts) utils.set_random_seed(Options()['misc']['seed']) Logger()('Load dataset...') dataset = factory(split) Logger()('Load model...') model = model_factory() model_state = torch.load(path_model_ckpt) model.load_state_dict(model_state) model.set_mode(split) if not os.path.isdir(dir_extract): Logger()('Create extract_dir {}'.format(dir_extract)) os.system('mkdir -p ' + dir_extract) mean_ingrs = torch.zeros(model.network.recipe_embedding.dim_ingr_out * 2) # bi LSTM mean_instrs = torch.zeros(model.network.recipe_embedding.dim_instr_out) for i in tqdm(range(len(dataset))): item = dataset[i] batch = dataset.items_tf()([item]) batch = model.prepare_batch(batch) out_ingrs = model.network.recipe_embedding.forward_ingrs( batch['recipe']['ingrs']) out_instrs = model.network.recipe_embedding.forward_instrs( batch['recipe']['instrs']) mean_ingrs += out_ingrs.data.cpu().squeeze(0) mean_instrs += out_instrs.data.cpu().squeeze(0) mean_ingrs /= len(dataset) mean_instrs /= len(dataset) path_ingrs = os.path.join(dir_extract, 'ingrs.pth') path_instrs = os.path.join(dir_extract, 'instrs.pth') torch.save(mean_ingrs, path_ingrs) torch.save(mean_instrs, path_instrs) Logger()('End')
def extract(path_opts=None): Options(path_opts) utils.set_random_seed(Options()['misc']['seed']) assert Options( )['dataset']['eval_split'] is not None, 'eval_split must be set' assert Options( )['dataset']['train_split'] is None, 'train_split must be None' init_logs_options_files(Options()['exp']['dir'], Options()['exp']['resume']) Logger().log_dict('options', Options(), should_print=True) Logger()(os.uname()) if torch.cuda.is_available(): cudnn.benchmark = True Logger()('Available GPUs: {}'.format(utils.available_gpu_ids())) engine = engines.factory() engine.dataset = datasets.factory(engine) engine.model = models.factory(engine) engine.view = views.factory(engine) # init extract directory dir_extract = os.path.join(Options()['exp']['dir'], 'extract', Options()['dataset']['eval_split']) os.system('mkdir -p ' + dir_extract) path_img_embs = os.path.join(dir_extract, 'image_emdeddings.pth') path_rcp_embs = os.path.join(dir_extract, 'recipe_emdeddings.pth') img_embs = torch.FloatTensor(len(engine.dataset['eval']), Options()['model.network.dim_emb']) rcp_embs = torch.FloatTensor(len(engine.dataset['eval']), Options()['model.network.dim_emb']) def save_embeddings(module, input, out): nonlocal img_embs nonlocal rcp_embs batch = input[0] # tuple of len=1 for j, idx in enumerate(batch['recipe']['index']): # path_image = os.path.join(dir_image, '{}.pth'.format(idx)) # path_recipe = os.path.join(dir_recipe, '{}.pth'.format(idx)) # torch.save(out['image_embedding'][j].data.cpu(), path_image) # torch.save(out['recipe_embedding'][j].data.cpu(), path_recipe) img_embs[idx] = out['image_embedding'][j].data.cpu() rcp_embs[idx] = out['recipe_embedding'][j].data.cpu() engine.model.register_forward_hook(save_embeddings) engine.resume() engine.eval() Logger()('Saving image embeddings to {} ...'.format(path_img_embs)) torch.save(img_embs, path_img_embs) Logger()('Saving recipe embeddings to {} ...'.format(path_rcp_embs)) torch.save(rcp_embs, path_rcp_embs)
def main(path_opts=None): # first call to Options() load the options yaml file from --path_opts command line argument if path_opts=None Options(path_opts) # make exp directory if resume=None init_experiment_directory(Options()['exp']['dir'], Options()['exp']['resume']) # initialiaze seeds to be able to reproduce experiment on reload utils.set_random_seed(Options()['misc']['seed']) # display and save options as yaml file in exp dir Logger().log_dict('options', Options(), should_print=True) # display server name and GPU(s) id(s) Logger()(os.uname()) if 'CUDA_VISIBLE_DEVICES' in os.environ: Logger()('CUDA_VISIBLE_DEVICES=' + os.environ['CUDA_VISIBLE_DEVICES']) # engine can train, eval, optimize the model # engine can save and load the model and optimizer engine = engines.factory() # dataset is a dictionnary which contains all the needed datasets indexed by splits # (example: dataset.keys() -> ['train','val']) dataset = datasets.factory() engine.dataset = dataset # model includes a network, a criterion and a metric # model can register engine hooks (begin epoch, end batch, end batch, etc.) # (example: "calculate mAP at the end of the evaluation epoch") model = models.factory(engine) engine.model = model # optimizer can register engine hooks optimizer = optimizers.factory(model, engine) engine.optimizer = optimizer # load the model and optimizer from a checkpoint if Options()['exp']['resume']: engine.resume() # if no training split, evaluate the model on the evaluation split # (example: $ python main.py --dataset.train_split --dataset.eval_split test) engine.eval() # optimize the model on the training split for several epochs # (example: $ python main.py --dataset.train_split train) # if evaluation split, evaluate the model after each epochs # (example: $ python main.py --dataset.train_split train --dataset.eval_split val) engine.train()
def api(path_opts=None): global engine global img_embs global rcp_embs global all_embs global all_ids Options(path_opts) utils.set_random_seed(Options()['misc']['seed']) assert Options( )['dataset']['eval_split'] is not None, 'eval_split must be set' assert Options( )['dataset']['train_split'] is None, 'train_split must be None' init_logs_options_files(Options()['exp']['dir'], Options()['exp']['resume']) Logger().log_dict('options', Options(), should_print=True) Logger()(os.uname()) if torch.cuda.is_available(): cudnn.benchmark = True Logger()('Available GPUs: {}'.format(utils.available_gpu_ids())) engine = engines.factory() engine.dataset = datasets.factory(engine) engine.model = models.factory(engine) engine.model.eval() engine.resume() dir_extract = os.path.join(Options()['exp']['dir'], 'extract', Options()['dataset']['eval_split']) path_img_embs = os.path.join(dir_extract, 'image_emdeddings.pth') path_rcp_embs = os.path.join(dir_extract, 'recipe_emdeddings.pth') img_embs = torch.load(path_img_embs) rcp_embs = torch.load(path_rcp_embs) all_embs = torch.cat([img_embs, rcp_embs], dim=0) all_ids = list(range(img_embs.shape[0])) + list(range(rcp_embs.shape[0])) my_local_ip = '132.227.204.160' # localhost | 192.168.0.41 (hostname --ip-address) my_local_port = 3456 # 8080 | 3456 run_simple(my_local_ip, my_local_port, application)
def main(): Logger('.') split = 'train' dir_exp = '/home/ubuntu/moochi/recipe1m.bootstrap.pytorch/logs/recipe1m/trijoint/2017-12-14-15-04-51' path_opts = os.path.join(dir_exp, 'options.yaml') dir_extract = os.path.join(dir_exp, 'extract_count', split) path_ingrs_count = os.path.join(dir_extract, 'ingrs.pth') Options(path_opts) utils.set_random_seed(Options()['misc']['seed']) dataset = factory(split) if not os.path.isfile(path_ingrs_count): ingrs_count = {} os.system('mkdir -p ' + dir_extract) for i in tqdm(range(len(dataset.recipes_dataset))): item = dataset.recipes_dataset[i] for ingr in item['ingrs']['interim']: if ingr not in ingrs_count: ingrs_count[ingr] = 1 else: ingrs_count[ingr] += 1 torch.save(ingrs_count, path_ingrs_count) else: ingrs_count = torch.load(path_ingrs_count) import ipdb ipdb.set_trace() sort = sorted(ingrs_count, key=ingrs_count.get) import ipdb ipdb.set_trace() Logger()('End')
def main(): Logger('.') #classes = ['hamburger'] #nb_points = split = 'test' class_name = None#'potato salad' modality_to_modality = 'recipe_to_image' dir_exp = '/home/cadene/doc/bootstrap.pytorch/logs/recipe1m/trijoint/2017-12-14-15-04-51' path_opts = os.path.join(dir_exp, 'options.yaml') dir_extract = os.path.join(dir_exp, 'extract', split) dir_extract_mean = os.path.join(dir_exp, 'extract_mean_features', split) dir_img = os.path.join(dir_extract, 'image') dir_rcp = os.path.join(dir_extract, 'recipe') path_model_ckpt = os.path.join(dir_exp, 'ckpt_best_val_epoch.metric.recall_at_1_im2recipe_mean_model.pth.tar') #is_mean = True #ingrs_list = ['carotte', 'salad', 'tomato']#['avocado'] #Options(path_opts) Options(path_opts) Options()['misc']['seed'] = 11 utils.set_random_seed(Options()['misc']['seed']) chosen_item_id = 51259 dataset = factory(split) if class_name: class_id = dataset.cname_to_cid[class_name] indices_by_class = dataset._make_indices_by_class() nb_points = len(indices_by_class[class_id]) list_idx = torch.Tensor(indices_by_class[class_id]) rand_idx = torch.randperm(list_idx.size(0)) list_idx = list_idx[rand_idx] list_idx = list_idx.view(-1).int() dir_visu = os.path.join(dir_exp, 'visu', 'remove_ingrs_item:{}_nb_points:{}_class:{}'.format(chosen_item_id, nb_points, class_name.replace(' ', '_'))) else: nb_points = 1000 list_idx = torch.randperm(len(dataset)) dir_visu = os.path.join(dir_exp, 'visu', 'remove_ingrs_item:{}_nb_points:{}_removed'.format(chosen_item_id, nb_points)) # for i in range(20): # item_id = list_idx[i] # item = dataset[item_id] # write_img_rcp(dir_visu, item, top=i) Logger()('Load model...') model = model_factory() model_state = torch.load(path_model_ckpt) model.load_state_dict(model_state) model.eval() item = dataset[chosen_item_id] # from tqdm import tqdm # ids = [] # for i in tqdm(range(len(dataset.recipes_dataset))): # item = dataset.recipes_dataset[i]#23534] # if 'broccoli' in item['ingrs']['interim']: # print('broccoli', i) # ids.append(i) # # if 'mushroom' in item['ingrs']['interim']: # # print('mushroom', i) # # break # import ipdb; ipdb.set_trace() # input_ = { # 'recipe': { # 'ingrs': { # 'data': item['recipe']['ingrs']['data'], # 'lengths': item['recipe']['ingrs']['lengths'] # }, # 'instrs': { # 'data': item['recipe']['instrs']['data'], # 'lengths': item['recipe']['instrs']['lengths'] # } # } # } instrs = torch.FloatTensor(6,1024) instrs[0] = item['recipe']['instrs']['data'][0] instrs[1] = item['recipe']['instrs']['data'][1] instrs[2] = item['recipe']['instrs']['data'][3] instrs[3] = item['recipe']['instrs']['data'][4] instrs[4] = item['recipe']['instrs']['data'][6] instrs[5] = item['recipe']['instrs']['data'][7] ingrs = torch.LongTensor([612,585,844,3087,144,188,1]) input_ = { 'recipe': { 'ingrs': { 'data': ingrs, 'lengths': ingrs.size(0) }, 'instrs': { 'data': instrs, 'lengths': instrs.size(0) } } } batch = dataset.items_tf()([input_]) batch = model.prepare_batch(batch) out = model.network.recipe_embedding(batch['recipe']) # path_rcp = os.path.join(dir_rcp, '{}.pth'.format(23534)) # rcp_emb = torch.load(path_rcp) Logger()('Load embeddings...') img_embs = [] for i in range(nb_points): try: idx = list_idx[i] except: import ipdb; ipdb.set_trace() #idx = i path_img = os.path.join(dir_img, '{}.pth'.format(idx)) if not os.path.isfile(path_img): Logger()('No such file: {}'.format(path_img)) continue img_embs.append(torch.load(path_img)) img_embs = torch.stack(img_embs, 0) Logger()('Fast distance...') dist = fast_distance(out.data.cpu().expand_as(img_embs), img_embs) dist = dist[0, :] sorted_ids = np.argsort(dist.numpy()) os.system('rm -rf '+dir_visu) os.system('mkdir -p '+dir_visu) Logger()('Load/save images in {}...'.format(dir_visu)) write_img_rcp(dir_visu, item, top=0, begin_with='query') for i in range(20): idx = int(sorted_ids[i]) item_id = list_idx[idx] #item_id = idx item = dataset[item_id] write_img_rcp(dir_visu, item, top=i, begin_with='nn') Logger()('End')
def main(): Logger('.') #classes = ['pizza', 'pork chops', 'cupcake', 'hamburger', 'green beans'] nb_points = 1000 split = 'test' dir_exp = '/home/cadene/doc/bootstrap.pytorch/logs/recipe1m/trijoint/2017-12-14-15-04-51' path_opts = os.path.join(dir_exp, 'options.yaml') dir_extract = os.path.join(dir_exp, 'extract', split) dir_extract_mean = os.path.join(dir_exp, 'extract_mean_features', split) dir_img = os.path.join(dir_extract, 'image') dir_rcp = os.path.join(dir_extract, 'recipe') path_model_ckpt = os.path.join( dir_exp, 'ckpt_best_val_epoch.metric.recall_at_1_im2recipe_mean_model.pth.tar') dir_visu = os.path.join(dir_exp, 'visu', 'mean_to_image') os.system('mkdir -p ' + dir_visu) #Options(path_opts) Options.load_from_yaml(path_opts) utils.set_random_seed(Options()['misc']['seed']) dataset = factory(split) Logger()('Load model...') model = model_factory() model_state = torch.load(path_model_ckpt) model.load_state_dict(model_state) model.set_mode(split) #emb = network.recipe_embedding.forward_ingrs(input_['recipe']['ingrs']) list_idx = torch.randperm(len(dataset)) Logger()('Load embeddings...') img_embs = [] rcp_embs = [] for i in range(nb_points): idx = list_idx[i] path_img = os.path.join(dir_img, '{}.pth'.format(idx)) path_rcp = os.path.join(dir_rcp, '{}.pth'.format(idx)) if not os.path.isfile(path_img): Logger()('No such file: {}'.format(path_img)) continue if not os.path.isfile(path_rcp): Logger()('No such file: {}'.format(path_rcp)) continue img_embs.append(torch.load(path_img)) rcp_embs.append(torch.load(path_rcp)) img_embs = torch.stack(img_embs, 0) rcp_embs = torch.stack(rcp_embs, 0) Logger()('Load means') path_ingrs = os.path.join(dir_extract_mean, 'ingrs.pth') path_instrs = os.path.join(dir_extract_mean, 'instrs.pth') mean_ingrs = torch.load(path_ingrs) mean_instrs = torch.load(path_instrs) mean_ingrs = Variable(mean_ingrs.unsqueeze(0).cuda(), requires_grad=False) mean_instrs = Variable(mean_instrs.unsqueeze(0).cuda(), requires_grad=False) Logger()('Forward ingredient...') ingr_emb = model.network.recipe_embedding.forward_ingrs_instrs( mean_ingrs, mean_instrs) ingr_emb = ingr_emb.data.cpu() ingr_emb = ingr_emb.expand_as(img_embs) Logger()('Fast distance...') dist = fast_distance(img_embs, ingr_emb)[:, 0] sorted_img_ids = np.argsort(dist.numpy()) Logger()('Load/save images...') for i in range(20): img_id = sorted_img_ids[i] img_id = int(img_id) path_img_from = dataset[img_id]['image']['path'] path_img_to = os.path.join(dir_visu, 'image_top_{}.png'.format(i + 1)) img = Image.open(path_img_from) img.save(path_img_to) #os.system('cp {} {}'.format(path_img_from, path_img_to)) Logger()('End')
def main(): Logger('.') #classes = ['pizza', 'pork chops', 'cupcake', 'hamburger', 'green beans'] nb_points = 1000 split = 'test' class_name = 'pizza' dir_exp = '/home/ubuntu/moochi/recipe1m.bootstrap.pytorch/logs/recipe1m/trijoint/2017-12-14-15-04-51' path_opts = os.path.join(dir_exp, 'options.yaml') dir_extract = os.path.join(dir_exp, 'extract', split) dir_extract_mean = os.path.join(dir_exp, 'extract_mean_features', split) dir_img = os.path.join(dir_extract, 'image') dir_rcp = os.path.join(dir_extract, 'recipe') path_model_ckpt = os.path.join( dir_exp, 'ckpt_best_val_epoch.metric.recall_at_1_im2recipe_mean_model.pth.tar') is_mean = True ingrs_list = ['fresh_strawberries'] #['avocado'] Options(path_opts) Options()['misc']['seed'] = 2 utils.set_random_seed(Options()['misc']['seed']) dataset = factory(split) Logger()('Load model...') model = model_factory() model_state = torch.load(path_model_ckpt) model.load_state_dict(model_state) model.eval() if not os.path.isdir(dir_extract): os.system('mkdir -p ' + dir_rcp) os.system('mkdir -p ' + dir_img) for i in tqdm(range(len(dataset))): item = dataset[i] batch = dataset.items_tf()([item]) if model.is_cuda: batch = model.cuda_tf()(batch) is_volatile = (model.mode not in ['train', 'trainval']) batch = model.variable_tf(volatile=is_volatile)(batch) out = model.network(batch) path_image = os.path.join(dir_img, '{}.pth'.format(i)) path_recipe = os.path.join(dir_rcp, '{}.pth'.format(i)) torch.save(out['image_embedding'][0].data.cpu(), path_image) torch.save(out['recipe_embedding'][0].data.cpu(), path_recipe) # b = dataset.make_batch_loader().__iter__().__next__() # import ipdb; ipdb.set_trace() ingrs = torch.LongTensor(1, len(ingrs_list)) for i, ingr_name in enumerate(ingrs_list): ingrs[0, i] = dataset.recipes_dataset.ingrname_to_ingrid[ingr_name] input_ = { 'recipe': { 'ingrs': { 'data': Variable(ingrs.cuda(), requires_grad=False), 'lengths': [ingrs.size(1)] }, 'instrs': { 'data': Variable(torch.FloatTensor(1, 1, 1024).fill_(0).cuda(), requires_grad=False), 'lengths': [1] } } } #emb = network.recipe_embedding.forward_ingrs(input_['recipe']['ingrs']) #list_idx = torch.randperm(len(dataset)) indices_by_class = dataset._make_indices_by_class() #import ipdb; ipdb.set_trace() class_id = dataset.cname_to_cid[class_name] list_idx = torch.Tensor(indices_by_class[class_id]) rand_idx = torch.randperm(list_idx.size(0)) list_idx = list_idx[rand_idx] list_idx = list_idx.view(-1).int() nb_points = list_idx.size(0) dir_visu = os.path.join( dir_exp, 'visu', 'ingrs_to_image_nb_points:{}_class:{}_instrs:{}_mean:{}_v2'.format( nb_points, class_name, '-'.join(ingrs_list), is_mean)) os.system('mkdir -p ' + dir_visu) Logger()('Load embeddings...') img_embs = [] rcp_embs = [] for i in range(nb_points): idx = list_idx[i] path_img = os.path.join(dir_img, '{}.pth'.format(idx)) path_rcp = os.path.join(dir_rcp, '{}.pth'.format(idx)) if not os.path.isfile(path_img): Logger()('No such file: {}'.format(path_img)) continue if not os.path.isfile(path_rcp): Logger()('No such file: {}'.format(path_rcp)) continue img_embs.append(torch.load(path_img)) rcp_embs.append(torch.load(path_rcp)) img_embs = torch.stack(img_embs, 0) rcp_embs = torch.stack(rcp_embs, 0) Logger()('Load mean embeddings') path_ingrs = os.path.join(dir_extract_mean, 'ingrs.pth') path_instrs = os.path.join(dir_extract_mean, 'instrs.pth') mean_ingrs = torch.load(path_ingrs) mean_instrs = torch.load(path_instrs) Logger()('Forward ingredient...') #ingr_emb = model.network.recipe_embedding(input_['recipe']) ingr_emb = model.network.recipe_embedding.forward_one_ingr( input_['recipe']['ingrs'], emb_instrs=mean_instrs.unsqueeze(0)) ingr_emb = ingr_emb.data.cpu() ingr_emb = ingr_emb.expand_as(img_embs) Logger()('Fast distance...') dist = fast_distance(img_embs, ingr_emb)[:, 0] sorted_ids = np.argsort(dist.numpy()) Logger()('Load/save images in {}...'.format(dir_visu)) for i in range(20): idx = int(sorted_ids[i]) item_id = list_idx[idx] item = dataset[item_id] # path_img_from = item['image']['path'] # ingrs = [ingr.replace('/', '\'') for ingr in item['recipe']['ingrs']['interim']] # cname = item['recipe']['class_name'] # path_img_to = os.path.join(dir_visu, 'image_top:{}_ingrs:{}_cname:{}.png'.format(i+1, '-'.join(ingrs), cname)) # img = Image.open(path_img_from) # img.save(path_img_to) #os.system('cp {} {}'.format(path_img_from, path_img_to)) write_img_rcp(dir_visu, item, top=i, begin_with='nn') Logger()('End')
def eval_epoch(self, model, dataset, epoch, mode='eval', logs_json=True): """ Launch evaluation procedures for one epoch List of the hooks (``mode='eval'`` by default): - mode_on_start_epoch: before the evaluation procedure for an epoch - mode_on_start_batch: before the evaluation precedure for a batch - mode_on_forward: after the forward of the model - mode_on_print: after the print to the terminal - mode_on_end_batch: end of the evaluation procedure for a batch - mode_on_end_epoch: before saving the logs in logs.json - mode_on_flush: end of the evaluation procedure for an epoch Returns: out(dict): mean of all the scalar outputs of the model, indexed by output name, for this epoch """ utils.set_random_seed(Options()['misc']['seed'] + epoch) # to be able to reproduce exps on reload Logger()('Evaluating model on {}set for epoch {}'.format( dataset.split, epoch)) model.eval() model.set_mode(mode) timer = { 'begin': time.time(), 'elapsed': time.time(), 'process': None, 'load': None, 'run_avg': 0 } out_epoch = {} batch_loader = dataset.make_batch_loader() self.hook('{}_on_start_epoch'.format(mode)) for i, batch in enumerate(batch_loader): timer['load'] = time.time() - timer['elapsed'] self.hook('{}_on_start_batch'.format(mode)) with torch.no_grad(): out = model(batch) if 'loss' in out and torch.isnan(out['loss']): torch.cuda.synchronize() del out Logger()('NaN detected') import gc gc.collect() torch.cuda.empty_cache() torch.cuda.synchronize() continue self.hook('{}_on_forward'.format(mode)) timer['process'] = time.time() - timer['elapsed'] if i == 0: timer['run_avg'] = timer['process'] else: timer['run_avg'] = timer['run_avg'] * 0.8 + timer[ 'process'] * 0.2 Logger().log_value('{}_batch.batch'.format(mode), i, should_print=False) Logger().log_value('{}_batch.epoch'.format(mode), epoch, should_print=False) Logger().log_value('{}_batch.timer.process'.format(mode), timer['process'], should_print=False) Logger().log_value('{}_batch.timer.load'.format(mode), timer['load'], should_print=False) for key, value in out.items(): if torch.is_tensor(value): if value.dim() <= 1: value = value.item() # get number from a torch scalar else: continue if type(value) == list: continue if type(value) == dict: continue if value is None: continue if key not in out_epoch: out_epoch[key] = [] out_epoch[key].append(value) Logger().log_value('{}_batch.{}'.format(mode, key), value, should_print=False) if i % Options()['engine']['print_freq'] == 0: Logger()("{}: epoch {} | batch {}/{}".format( mode, epoch, i, len(batch_loader) - 1)) Logger()("{} elapsed: {} | left: {}".format( ' ' * len(mode), datetime.timedelta(seconds=math.floor(time.time() - timer['begin'])), datetime.timedelta( seconds=math.floor(timer['run_avg'] * (len(batch_loader) - 1 - i))))) Logger()("{} process: {:.5f} | load: {:.5f}".format( ' ' * len(mode), timer['process'], timer['load'])) self.hook('{}_on_print'.format(mode)) timer['elapsed'] = time.time() self.hook('{}_on_end_batch'.format(mode)) if Options()['engine']['debug']: if i > 10: break out = {} for key, value in out_epoch.items(): try: out[key] = sum(value) / len(value) except: import ipdb ipdb.set_trace() Logger().log_value('{}_epoch.epoch'.format(mode), epoch, should_print=True) for key, value in out.items(): Logger().log_value('{}_epoch.{}'.format(mode, key), value, should_print=True) self.hook('{}_on_end_epoch'.format(mode)) if logs_json: Logger().flush() self.hook('{}_on_flush'.format(mode)) return out
import torch import bootstrap.lib.utils as utils import bootstrap.engines as engines import bootstrap.models as models import bootstrap.datasets as datasets from bootstrap.lib.logger import Logger from bootstrap.lib.options import Options from bootstrap.run import init_logs_options_files from block.datasets.vqa_utils import tokenize_mcb if __name__ == "__main__": Options() #print("seed: {}".format(Options()['misc']['seed'])) utils.set_random_seed(Options()['misc']['seed']) init_logs_options_files(Options()['exp']['dir'], Options()['exp']['resume']) engine = engines.factory() #print("engine: {}".format(engine)) engine.dataset = datasets.factory(engine) engine.model = models.factory(engine) #print("engine.model: {}".format(engine.model)) engine.model.eval() engine.resume() #img_emb_module = load_img_emb_module().cuda() #engine.model.network.image_embedding # inputs img = torch.randn(1, 3, 224, 244).cuda() question = 'What is the color of the white horse?' question_words = tokenize_mcb(question)
def main(): Logger('.') import argparse parser = argparse.ArgumentParser() parser.add_argument('modality_to_modality', help='foo help', default='recipe_to_image') args = parser.parse_args() #classes = ['pizza', 'pork chops', 'cupcake', 'hamburger', 'green beans'] nb_points = 1000 modality_to_modality = args.modality_to_modality#'image_to_image' print(modality_to_modality) split = 'test' dir_exp = '/home/ubuntu/moochi/recipe1m.bootstrap.pytorch/logs/recipe1m/trijoint/2017-12-14-15-04-51' path_opts = os.path.join(dir_exp, 'options.yaml') dir_extract = os.path.join(dir_exp, 'extract', split) dir_img = os.path.join(dir_extract, 'image') dir_rcp = os.path.join(dir_extract, 'recipe') path_model_ckpt = os.path.join(dir_exp, 'ckpt_best_val_epoch.metric.recall_at_1_im2recipe_mean_model.pth.tar') Options.load_from_yaml(path_opts) Options()['misc']['seed'] = 11 utils.set_random_seed(Options()['misc']['seed']) dataset = factory(split) Logger()('Load model...') model = model_factory() model_state = torch.load(path_model_ckpt) model.load_state_dict(model_state) model.eval() if not os.path.isdir(dir_extract): os.system('mkdir -p '+dir_rcp) os.system('mkdir -p '+dir_img) for i in tqdm(range(len(dataset))): item = dataset[i] batch = dataset.items_tf()([item]) if model.is_cuda: batch = model.cuda_tf()(batch) is_volatile = (model.mode not in ['train', 'trainval']) batch = model.variable_tf(volatile=is_volatile)(batch) out = model.network(batch) path_image = os.path.join(dir_img, '{}.pth'.format(i)) path_recipe = os.path.join(dir_rcp, '{}.pth'.format(i)) torch.save(out['image_embedding'][0].data.cpu(), path_image) torch.save(out['recipe_embedding'][0].data.cpu(), path_recipe) indices_by_class = dataset._make_indices_by_class() # class_name = classes[0] # TODO # class_id = dataset.cname_to_cid[class_name] # list_idx = torch.Tensor(indices_by_class[class_id]) # rand_idx = torch.randperm(list_idx.size(0)) # list_idx = list_idx[rand_idx] # list_idx = list_idx.view(-1).int() list_idx = torch.randperm(len(dataset)) #nb_points = list_idx.size(0) dir_visu = os.path.join(dir_exp, 'visu', '{}_top20_seed:{}'.format(modality_to_modality, Options()['misc']['seed'])) os.system('rm -rf '+dir_visu) os.system('mkdir -p '+dir_visu) Logger()('Load embeddings...') img_embs = [] rcp_embs = [] for i in range(nb_points): idx = list_idx[i] #idx = i path_img = os.path.join(dir_img, '{}.pth'.format(idx)) path_rcp = os.path.join(dir_rcp, '{}.pth'.format(idx)) if not os.path.isfile(path_img): Logger()('No such file: {}'.format(path_img)) continue if not os.path.isfile(path_rcp): Logger()('No such file: {}'.format(path_rcp)) continue img_embs.append(torch.load(path_img)) rcp_embs.append(torch.load(path_rcp)) img_embs = torch.stack(img_embs, 0) rcp_embs = torch.stack(rcp_embs, 0) # Logger()('Forward ingredient...') # #ingr_emb = model.network.recipe_embedding(input_['recipe']) # ingr_emb = model.network.recipe_embedding.forward_one_ingr( # input_['recipe']['ingrs'], # emb_instrs=mean_instrs.unsqueeze(0)) # ingr_emb = ingr_emb.data.cpu() # ingr_emb = ingr_emb.expand_as(img_embs) Logger()('Fast distance...') if modality_to_modality == 'image_to_recipe': dist = fast_distance(img_embs, rcp_embs) elif modality_to_modality == 'recipe_to_image': dist = fast_distance(rcp_embs, img_embs) elif modality_to_modality == 'recipe_to_recipe': dist = fast_distance(rcp_embs, rcp_embs) elif modality_to_modality == 'image_to_image': dist = fast_distance(img_embs, img_embs) dist=dist[:, 0] sorted_ids = np.argsort(dist.numpy()) Logger()('Load/save images in {}...'.format(dir_visu)) for i in range(20): idx = int(sorted_ids[i]) item_id = list_idx[idx] #item_id = idx item = dataset[item_id] write_img_rcp(dir_visu, item, top=i) #os.system('cp {} {}'.format(path_img_from, path_img_to)) Logger()('End')
def main(): Logger('.') #classes = ['pizza', 'pork chops', 'cupcake', 'hamburger', 'green beans'] nb_points = 1000 split = 'test' dir_exp = 'logs/recipe1m/trijoint/2017-12-14-15-04-51' path_opts = os.path.join(dir_exp, 'options.yaml') dir_extract = os.path.join(dir_exp, 'extract', split) dir_img = os.path.join(dir_extract, 'image') dir_rcp = os.path.join(dir_extract, 'recipe') path_model_ckpt = os.path.join( dir_exp, 'ckpt_best_val_epoch.metric.recall_at_1_im2recipe_mean_model.pth.tar') #Options(path_opts) Options.load_from_yaml(path_opts) utils.set_random_seed(Options()['misc']['seed']) dataset = factory(split) Logger()('Load model...') model = model_factory() model_state = torch.load(path_model_ckpt) model.load_state_dict(model_state) model.set_mode(split) if not os.path.isdir(dir_extract): os.system('mkdir -p ' + dir_rcp) os.system('mkdir -p ' + dir_img) for i in tqdm(range(len(dataset))): item = dataset[i] batch = dataset.items_tf()([item]) if model.is_cuda: batch = model.cuda_tf()(batch) is_volatile = (model.mode not in ['train', 'trainval']) batch = model.variable_tf(volatile=is_volatile)(batch) out = model.network(batch) path_image = os.path.join(dir_img, '{}.pth'.format(i)) path_recipe = os.path.join(dir_rcp, '{}.pth'.format(i)) torch.save(out['image_embedding'][0].data.cpu(), path_image) torch.save(out['recipe_embedding'][0].data.cpu(), path_recipe) # b = dataset.make_batch_loader().__iter__().__next__() # class_name = 'pizza' # ingrs = torch.LongTensor(1, 2) # ingrs[0, 0] = dataset.recipes_dataset.ingrname_to_ingrid['mushrooms'] # ingrs[0, 1] = dataset.recipes_dataset.ingrname_to_ingrid['mushroom'] class_name = 'hamburger' ingrs = torch.LongTensor(1, 2) ingrs[0, 0] = dataset.recipes_dataset.ingrname_to_ingrid['mushroom'] ingrs[0, 1] = dataset.recipes_dataset.ingrname_to_ingrid['mushrooms'] #ingrs[0, 0] = dataset.recipes_dataset.ingrname_to_ingrid['tomato'] #ingrs[0, 1] = dataset.recipes_dataset.ingrname_to_ingrid['salad'] #ingrs[0, 2] = dataset.recipes_dataset.ingrname_to_ingrid['onion'] #ingrs[0, 3] = dataset.recipes_dataset.ingrname_to_ingrid['chicken'] input_ = { 'recipe': { 'ingrs': { 'data': Variable(ingrs.cuda(), requires_grad=False), 'lengths': [ingrs.size(1)] }, 'instrs': { 'data': Variable(torch.FloatTensor(1, 1, 1024).fill_(0).cuda(), requires_grad=False), 'lengths': [1] } } } #emb = network.recipe_embedding.forward_ingrs(input_['recipe']['ingrs']) #list_idx = torch.randperm(len(dataset)) indices_by_class = dataset._make_indices_by_class() class_id = dataset.cname_to_cid[class_name] list_idx = torch.Tensor(indices_by_class[class_id]) rand_idx = torch.randperm(list_idx.size(0)) list_idx = list_idx[rand_idx] list_idx = list_idx.view(-1).int() img_embs = [] rcp_embs = [] if nb_points > list_idx.size(0): nb_points = list_idx.size(0) Logger()('Load {} embeddings...'.format(nb_points)) for i in range(nb_points): idx = list_idx[i] path_img = os.path.join(dir_img, '{}.pth'.format(idx)) path_rcp = os.path.join(dir_rcp, '{}.pth'.format(idx)) if not os.path.isfile(path_img): Logger()('No such file: {}'.format(path_img)) continue if not os.path.isfile(path_rcp): Logger()('No such file: {}'.format(path_rcp)) continue img_embs.append(torch.load(path_img)) rcp_embs.append(torch.load(path_rcp)) img_embs = torch.stack(img_embs, 0) rcp_embs = torch.stack(rcp_embs, 0) Logger()('Forward ingredient...') #ingr_emb = model.network.recipe_embedding(input_['recipe']) ingr_emb = model.network.recipe_embedding.forward_one_ingr( input_['recipe']['ingrs']) ingr_emb = ingr_emb.data.cpu() ingr_emb = ingr_emb.expand_as(img_embs) Logger()('Fast distance...') dist = fast_distance(img_embs, ingr_emb)[:, 0] sorted_ids = np.argsort(dist.numpy()) dir_visu = os.path.join(dir_exp, 'visu', 'ingrs_to_image_by_class_{}'.format(class_name)) os.system('mkdir -p ' + dir_visu) Logger()('Load/save images...') for i in range(20): idx = int(sorted_ids[i]) item_id = list_idx[idx] item = dataset[item_id] Logger()(item['recipe']['class_name']) Logger()(item['image']['class_name']) path_img_from = item['image']['path'] path_img_to = os.path.join(dir_visu, 'image_top_{}.png'.format(i + 1)) img = Image.open(path_img_from) img.save(path_img_to) #os.system('cp {} {}'.format(path_img_from, path_img_to)) Logger()('End')