def main(**kwargs): exp_base_dir = coco_paths['exp_dir'] if kwargs['dataset'] == 'flickr': exp_base_dir = flickr_paths['exp_dir'] exp_const = ExpConstants(kwargs['exp_name'], exp_base_dir) exp_const.model_dir = os.path.join(exp_const.exp_dir, 'models') exp_const.seed = 0 exp_const.contextualize = not kwargs['no_context'] exp_const.random_lang = kwargs['random_lang'] data_const = FlickrDatasetConstants(kwargs['subset']) model_const = Constants() model_const.model_num = kwargs['model_num'] model_const.object_encoder = ObjectEncoderConstants() model_const.object_encoder.context_layer.output_attentions = True model_const.object_encoder.object_feature_dim = 2048 model_const.cap_encoder = CapEncoderConstants() model_const.cap_encoder.output_attentions = True model_const.cap_info_nce_layers = kwargs['cap_info_nce_layers'] if model_const.model_num == -100: filename = os.path.join(exp_const.exp_dir, f'results_val_best.json') results = io.load_json_object(filename) model_const.model_num = results['model_num'] print('Selected model num:', model_const.model_num) model_const.object_encoder_path = os.path.join( exp_const.model_dir, f'object_encoder_{model_const.model_num}') model_const.lang_sup_criterion_path = os.path.join( exp_const.model_dir, f'lang_sup_criterion_{model_const.model_num}') if exp_const.random_lang is True: model_const.cap_encoder_path = os.path.join( exp_const.model_dir, f'cap_encoder_{model_const.model_num}') eval_flickr_phrase_loc.main(exp_const, data_const, model_const)
def main(**kwargs): exp_base_dir = coco_paths['exp_dir'] if kwargs['dataset'] == 'flickr': exp_base_dir = flickr_paths['exp_dir'] exp_const = ExpConstants(kwargs['exp_name'], exp_base_dir) exp_const.log_dir = os.path.join(exp_const.exp_dir, 'logs') exp_const.model_dir = os.path.join(exp_const.exp_dir, 'models') exp_const.vis_dir = os.path.join(exp_const.exp_dir, 'vis') exp_const.dataset = kwargs['dataset'] exp_const.optimizer = 'Adam' exp_const.lr = kwargs['lr'] exp_const.momentum = None exp_const.num_epochs = 10 exp_const.log_step = 20 # Save models approx. twice every epoch exp_const.model_save_step = 400000 // (2 * kwargs['train_batch_size'] ) # 4000=400000/(2*50) if exp_const.dataset == 'flickr': exp_const.model_save_step = 150000 // (2 * kwargs['train_batch_size']) val_freq_factor = 2 if kwargs['val_frequently'] is True: val_freq_factor = 1 exp_const.val_step = val_freq_factor * exp_const.model_save_step # set to 1*model_save_step for plotting mi vs perf exp_const.num_val_samples = None exp_const.train_batch_size = kwargs['train_batch_size'] exp_const.val_batch_size = 20 exp_const.num_workers = 10 exp_const.seed = 0 exp_const.neg_noun_loss_wt = kwargs['neg_noun_loss_wt'] exp_const.self_sup_loss_wt = kwargs['self_sup_loss_wt'] exp_const.lang_sup_loss_wt = kwargs['lang_sup_loss_wt'] exp_const.contextualize = not kwargs['no_context'] exp_const.random_lang = kwargs['random_lang'] DatasetConstants = CocoDatasetConstants if exp_const.dataset == 'flickr': DatasetConstants = FlickrDatasetConstants data_const = { 'train': DatasetConstants('train'), 'val': DatasetConstants('val'), } model_const = Constants() model_const.model_num = kwargs['model_num'] model_const.object_encoder = ObjectEncoderConstants() model_const.object_encoder.context_layer.output_attentions = True model_const.object_encoder.object_feature_dim = 2048 model_const.cap_encoder = CapEncoderConstants() model_const.cap_encoder.output_attentions = True model_const.cap_info_nce_layers = kwargs['cap_info_nce_layers'] model_const.object_encoder_path = os.path.join( exp_const.model_dir, f'object_encoder_{model_const.model_num}') model_const.self_sup_criterion_path = os.path.join( exp_const.model_dir, f'self_sup_criterion_{model_const.model_num}') model_const.lang_sup_criterion_path = os.path.join( exp_const.model_dir, f'lang_sup_criterion_{model_const.model_num}') train(exp_const, data_const, model_const)
def main(**kwargs): exp_base_dir = coco_paths['exp_dir'] if kwargs['train_dataset'] == 'flickr': exp_base_dir = flickr_paths['exp_dir'] exp_const = ExpConstants(kwargs['exp_name'], exp_base_dir) exp_const.log_dir = os.path.join(exp_const.exp_dir, 'logs') exp_const.model_dir = os.path.join(exp_const.exp_dir, 'models') exp_const.train_dataset = kwargs['train_dataset'] exp_const.vis_dataset = kwargs['vis_dataset'] exp_const.vis_dir = os.path.join(exp_const.exp_dir, f'vis/attention_{exp_const.vis_dataset}') exp_const.num_vis_samples = 50 exp_const.seed = 0 exp_const.contextualize = not kwargs['no_context'] DatasetConstants = CocoDatasetConstants if exp_const.vis_dataset == 'flickr': DatasetConstants = FlickrDatasetConstants data_const = DatasetConstants('val') if exp_const.vis_dataset == 'coco': data_const.image_dir = os.path.join(coco_paths['image_dir'], data_const.subset_image_dirname) data_const.read_neg_samples = False data_const.read_noun_adj_tokens = False model_const = Constants() model_const.model_num = kwargs['model_num'] model_const.object_encoder = ObjectEncoderConstants() model_const.object_encoder.object_feature_dim = 2048 model_const.cap_encoder = CapEncoderConstants() if model_const.model_num == -100: model_const.object_encoder_path = os.path.join(exp_const.model_dir, f'best_object_encoder') model_const.lang_sup_criterion_path = os.path.join( exp_const.model_dir, f'best_lang_sup_criterion') else: model_const.object_encoder_path = os.path.join( exp_const.model_dir, f'object_encoder_{model_const.model_num}') model_const.lang_sup_criterion_path = os.path.join( exp_const.model_dir, f'lang_sup_criterion_{model_const.model_num}') if exp_const.vis_dataset == 'coco': vis_att(exp_const, data_const, model_const) else: vis_att_flickr(exp_const, data_const, model_const)
def main(**kwargs): exp_base_dir = coco_paths['exp_dir'] if kwargs['dataset'] == 'flickr': exp_base_dir = flickr_paths['exp_dir'] exp_const = ExpConstants(kwargs['exp_name'], exp_base_dir) exp_const.model_dir = os.path.join(exp_const.exp_dir, 'models') exp_const.seed = 0 exp_const.contextualize = not kwargs['no_context'] exp_const.random_lang = kwargs['random_lang'] data_const = FlickrDatasetConstants(kwargs['subset']) model_const = Constants() model_const.object_encoder = ObjectEncoderConstants() model_const.object_encoder.context_layer.output_attentions = True model_const.object_encoder.object_feature_dim = 2048 model_const.cap_encoder = CapEncoderConstants() model_const.cap_encoder.output_attentions = True model_const.cap_info_nce_layers = kwargs['cap_info_nce_layers'] model_nums = find_all_model_numbers(exp_const.model_dir) for num in model_nums: continue if num <= 3000: continue model_const.model_num = num model_const.object_encoder_path = os.path.join( exp_const.model_dir, f'object_encoder_{model_const.model_num}') model_const.lang_sup_criterion_path = os.path.join( exp_const.model_dir, f'lang_sup_criterion_{model_const.model_num}') if exp_const.random_lang is True: model_const.cap_encoder_path = os.path.join( exp_const.model_dir, f'cap_encoder_{model_const.model_num}') filename = os.path.join(exp_const.exp_dir, f'results_{data_const.subset}_{num}.json') if os.path.exists(filename): print(io.load_json_object(filename)) continue eval_flickr_phrase_loc.main(exp_const, data_const, model_const) best_model_num = -1 best_pt_recall = 0 best_results = None for num in model_nums: filename = os.path.join(exp_const.exp_dir, f'results_{data_const.subset}_{num}.json') if not os.path.exists(filename): continue results = io.load_json_object(filename) results['model_num'] = num print(results) if results['pt_recall'] >= best_pt_recall: best_results = results best_pt_recall = results['pt_recall'] best_model_num = num print('-' * 80) best_results['model_num'] = best_model_num print(best_results) filename = os.path.join(exp_const.exp_dir, f'results_{data_const.subset}_best.json') io.dump_json_object(best_results, filename)