def get_test_dataset(cfg: DictConfig): """ Get test dataset :param cfg: general hydra config :return: test dataset """ random.seed(cfg.trainer.seed) with open(f'../../{cfg.data_params.folder_path}{cfg.inference.file_name}', 'r', encoding='utf-8') as f: data = json.load(f) if cfg.data_params.debug_mode: data = random.sample(data, len(data) // 10) sentences = [s['text'] for s in data] labels = [s['label'] for s in data] dataset_class = load_obj(cfg.dataset.class_name) tokenizer = load_obj(cfg.model.tokenizer).from_pretrained( f"../../{cfg.inference.save_dir}", do_lower_case=True) test_dataset = dataset_class( cfg=cfg, data=sentences, targets=labels, tokenizer=tokenizer, ) return test_dataset
def __init__(self, image_ids, image_folder_path, mode = 'train', vocab_file = "", vocab_threshold = 5, batch_size = 10): assert mode in ['train', 'val', 'test'] self.mode = mode self.image_folder_path = image_folder_path self.batch_size = batch_size # Get pre-processed objects all_captions_dict = load_obj('captions_dict') captions_dict = { image_id: all_captions_dict[image_id] for image_id in image_ids } # only include selected subset of captions # Obtain sample of training images #self.training_image_ids, captions_dict = get_training_indices(sample_size = sample_size, mode = "balanced_clean") # self.training_image_ids, self.images_path, self.image_id_dict, captions_dict \ # = get_data(image_folder_path, annotations_path, sample_size, data_type) # Set up vocabulary or load from training set if self.mode == 'train': self.vocab = Vocabulary(captions_dict) print('Vocabulary successfully created') elif vocab_file != "": self.vocab = vocab_file self.word2idx = self.vocab.word2idx self.idx2word = self.vocab.idx2word #print('Vocabulary successfully loaded') else: self.vocab = load_obj("vocab") self.word2idx = self.vocab.word2idx self.idx2word = self.vocab.idx2word print('Vocabulary successfully loaded') # Batch_size set to 1 if is test if self.mode == 'test': self.batch_size = 1 # Set up dataset self.im_ids = [] # with duplicates for indexing, i.e. if caption 1-5 all correspond to image 8, the im_ids will be [8,8,8,8,8] self.captions = [] self.images = [] self.captions_len = [] for im_id, captions_list in captions_dict.items(): for item in captions_list: self.im_ids.append(im_id) self.captions.append(item) self.captions_len.append(len(nltk.tokenize.word_tokenize(item))) # Set up paramteres for image feature extraction self.transform = transforms.Compose([ transforms.Resize(256), transforms.RandomCrop(224), transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize((0.485, 0.456, 0.406),(0.229, 0.224, 0.225)), ])
def get_training_datasets(cfg: DictConfig) -> Tuple: """ Get train and validation datasets :param cfg: general hydra config :return: tuple of train and valid dataset objects """ random.seed(cfg.trainer.seed) with open( f'../../{cfg.data_params.folder_path}{cfg.data_params.file_name}', 'r', encoding='utf-8') as f: data = json.load(f) if cfg.data_params.debug_mode: print("DEBUG MOD ON!") data = random.sample(data, len(data) // 100) sentences = [s['text'] for s in data] labels = [s['label'] for s in data] train_data, valid_data, train_labels, val_labels = train_test_split( sentences, labels, test_size=cfg.data_params.test_size, shuffle=True, random_state=cfg.trainer.seed, ) dataset_class = load_obj(cfg.dataset.class_name) tokenizer = load_obj(cfg.model.tokenizer).from_pretrained( cfg.model.model_path, do_lower_case=True) train_dataset = dataset_class( cfg=cfg, data=train_data, targets=train_labels, tokenizer=tokenizer, ) valid_dataset = dataset_class( cfg=cfg, data=valid_data, targets=val_labels, tokenizer=tokenizer, ) return train_dataset, valid_dataset
def eval_model(image_folder_path, vocab_path='', model_path='', training_image_ids_path='', sample_size=100, mode='balanced_mode'): test_pred_captions = predict_for_test_samples(image_folder_path = image_folder_path,\ sample_size = sample_size, vocab_path = vocab_path, model_path = model_path,\ training_image_ids_path = training_image_ids_path, mode = 'balanced_mode') # load objects im_gender_summary = load_obj("im_gender_summary") captions_dict = load_obj("captions_dict") # populate ref_captions ref_captions = [] for _, captions in captions_dict.items(): for c in captions: ref_captions.append(c.split()) # intiaite list gt = [] pred_gender = [] bleus = [] for image_id, caption in test_pred_captions.items(): gt.append(im_gender_summary[image_id]['pred_gt']) pred_gender.append(caption_to_gender(caption)) bleus.append(sentence_bleu(ref_captions, caption.split())) labels = ['male', 'neutral', 'female'] conf_matrix = confusion_matrix(gt, pred_gender, labels=labels) confusion_matrix_df = pd.DataFrame(conf_matrix, index=labels, columns=labels) accuracy = accuracy_score(gt, pred_gender) bleu = np.mean(bleus) print( f"Confusion Matrix (columns = Ground Truth gender, rows = Predicted gender):\n{confusion_matrix_df}" ) print(f"\nAccuracy score: {accuracy}") print( f"\nTest Bleu Score (compared against original human labels of test set):{bleu}" ) return confusion_matrix_df, accuracy, bleu
def run(cfg: DictConfig) -> Dict: """ Model's inference on hold-out dataset ---------- Example: python predict.py :param cfg: general hydra config :return: dict: test metrics """ hparams = flatten_omegaconf(cfg) exp = Experiment( api_key=cfg.logger.comet_api, project_name=cfg.general.project_name, workspace=cfg.general.workspace, ) exp.log_parameters(hparams) device = torch.device("cuda" if torch.cuda.is_available() else "cpu") bert_model = load_obj(cfg.model.class_name).from_pretrained(cfg) bert_model.model.load_state_dict( torch.load(f"../../{cfg.inference.save_dir}/pytorch_model.bin")) # print(bert_model) bert_model.to(device) bert_model.eval() collator = load_obj(cfg.dataset.collator)( percentile=cfg.data_params.percentile, pad_value=cfg.data_params.pad_value, ) criterion = nn.CrossEntropyLoss() test = get_test_dataset(cfg) evaluator = Evaluator(val_dataset=test, collator=collator, criterion=criterion, cfg=cfg, device=device) set_seed(cfg.trainer.seed) test_metrics = evaluator(model=bert_model, experiment=exp, epoch=0, step=0) return test_metrics
def train_test_split(training_image_ids, test_size=0.3, random_state=123): # Get pre-processed objects im_gender_summary = load_obj('im_gender_summary') X = np.asarray(training_image_ids) y = np.asarray([im_gender_summary[x]['pred_gt'] for x in X]) # Use Stratified shuffle split to ensure the ratio of gender ratio stays the same in train set and validation set (both balanced or random) sss = StratifiedShuffleSplit(n_splits=1, test_size=test_size, random_state=random_state) for train_idx, test_idx in sss.split(X, y): train_image_ids, test_image_ids = X[train_idx], X[test_idx] gender_train, gender_test = y[train_idx], y[test_idx] return train_image_ids, test_image_ids, gender_train, gender_test
def predict_for_test_samples(sample_size,image_folder_path, vocab_path = '', model_path = '', training_image_ids_path = '', embed_size = 256, hidden_size = 512, mode = 'balanced_clean'): random.seed(123) # Init Dict test_pred_captions = dict() # Get the training image id paths if training_image_ids_path == '': # if not specified, assume it is the vocab pickle saved in object training_image_ids = load_obj('training_image_ids') else: with open(training_image_ids_path, 'rb') as f: training_image_ids = pickle.load(f) # Get test image ids test_image_ids = get_test_indices(sample_size, training_image_ids, mode = mode) for test_image_id in test_image_ids: captions = predict_from_COCO(image_folder_path, vocab_path = vocab_path, model_path = model_path, training_image_ids_path = training_image_ids_path,\ test_image_id = test_image_id, is_print = False, return_one = True) test_pred_captions[test_image_id] = captions return test_pred_captions
def get_qualified_dataset(annotations_path, save_file=False): ''' captions_dict (dict)- key: image_id, value: list of captions im_gender_summary (dict of dict)- key: image_id, value: dict() keys in dict: pred_gt- predicted ground truth label of the gender noun per_gt- % of annotations (out of 5 total) that agreed with the GT agreement_score- agreement score calculated using distance between 5 predictions, with 1 being the best male = 1, female = -1, neutral = 0 e.g.0 annotations indicate [f, f, f, f, f], agreement_score = 1.00 e.g.1 annotations indicate [m, m, f, f, f], agreement_score = 0.00 e.g.2 annotation indicate [n, n, f, f, f], agreement_score = 0.50 anno_gender- list of gender sentiment, e.g. ['male', 'female', 'neutral', 'female', 'female'] anno_nouns- list of nouns used to describe human clean_gender- binary variable indicating if all notations used the same gender/ gender-neutral noun clean_noun- binary variable indicating if all notations used the identical noun not_human_im_ids(list)- list of image ids of images with >1 captions that do not mention humans. Since the COCO dataset does not label whether human (or other objects) is the major subject matter of the image. This list helps us isolate images with human figures as the focus. ''' captions_dict = dict() im_gender_summary = dict() not_human_im_ids = list() # load pre-processed data gender_nouns_lookup = load_obj('gender_nouns_lookup') for datatype in ['train', 'val']: print(f"\nEvaluating ground truth labels in {datatype} set") with open(f'{annotations_path}/captions_{datatype}2014.json') as f: captions_json = json.load(f) for i in range(len(captions_json['annotations'])): # Check to make sure image exists, as some images' captions are included in the json file but the image does not exist, image_id = captions_json['annotations'][i]['image_id'] l = len(str(image_id)) fnames = [ "COCO_train2014_" + "0" * (12 - l) + str(image_id) + '.jpg', "COCO_val2014_" + "0" * (12 - l) + str(image_id) + '.jpg' ] image_check = glob.glob('./data/images/*/' + fnames[0]) + glob.glob( './data/images/*/' + fnames[1]) if image_check != []: caption = captions_json['annotations'][i]['caption'] tokens = nltk.word_tokenize(caption) c_female = 0 # count of gender nouns and gender-neutral nouns c_male = 0 c_neutral = 0 noun = [] # Evaluate annotator's noun used to describe humans for t in tokens: t = t.lower() if t in gender_nouns_lookup['female']: c_female += 1 noun.append(t) elif t in gender_nouns_lookup['male']: c_male += 1 noun.append(t) elif t in gender_nouns_lookup['neutral']: c_neutral += 1 noun.append(t) # Only include image for training if more than one caption of the image mention human # Conflicting gender mentions are also dropped, e.g. "a boy and a girl are on a beach" if c_female + c_male + c_neutral == 1: # Assign gender sentiment to the caption if c_female > 0: gender = 'female' elif c_male > 0: gender = 'male' else: gender = 'neutral' # Populate captions dict and image gender summary dict if image_id in captions_dict: captions_dict[image_id] += [caption] im_gender_summary[image_id]['anno_gender'].append( gender) im_gender_summary[image_id]['anno_noun'].append( noun[0]) else: captions_dict[image_id] = [caption] im_gender_summary[image_id] = dict() im_gender_summary[image_id]['anno_gender'] = [ gender ] im_gender_summary[image_id]['anno_noun'] = [ noun[0] ] if i % 100000 == 0: print( f"Caption {i} processed, out of {len(captions_json['annotations'])} captions" ) print( f"No. of qualified images processed: {len(im_gender_summary)}" ) for image_id in im_gender_summary: # Delete images where <3 annotators mentioned the human figure # Because it is impossible to estimate the ground truth using only 1 or 2 captions if len(im_gender_summary[image_id]['anno_gender']) < 3: not_human_im_ids.append(image_id) else: pred = im_gender_summary[image_id]['anno_gender'] # Evaluate groundtruth guesses and agreement scores gt = max(set(pred), key=pred.count) # Populate dictionary im_gender_summary[image_id]['pred_gt'] = gt im_gender_summary[image_id]['per_gt'] = sum( [1 for p in pred if p == gt]) / len(pred) im_gender_summary[image_id]['agreement_score'] = agreement_score( pred) if len(set(pred)) == 1: im_gender_summary[image_id]['clean_gender'] = 1 else: im_gender_summary[image_id]['clean_gender'] = 0 if len(set(im_gender_summary[image_id]['anno_noun'])) == 1: im_gender_summary[image_id]['clean_noun'] = 1 else: im_gender_summary[image_id]['clean_noun'] = 0 for image_id in not_human_im_ids: try: del captions_dict[image_id] del im_gender_summary[image_id] except: pass if save_file == True: export_csv('./data/list/qualified_image_ids.csv', list(im_gender_summary.keys())) save_obj(captions_dict, 'captions_dict') save_obj(im_gender_summary, 'im_gender_summary') else: return captions_dict, im_gender_summary
def get_test_indices(sample_size, training_image_ids=[], mode='random'): ''' training_image_ids: image ids used in training or validation while training the model sample_size: # of image ids needed ''' assert mode in ['random', 'balanced_mode', 'balanced_clean'] assert isinstance(sample_size, int) test_captions_dict = dict() # Get pre-processed objects im_gender_summary = load_obj('im_gender_summary') captions_dict = load_obj('captions_dict') shuffle_im_keys = list(im_gender_summary.keys()) random.shuffle(shuffle_im_keys) if mode == 'random': i = 0 for image_id in shuffle_im_keys: if i < sample_size: if image_id not in training_image_ids: test_captions_dict[image_id] = captions_dict[image_id] i += 1 elif mode == 'balanced_mode': i = 0 male_count = 0 female_count = 0 neutral_count = 0 for image_id in shuffle_im_keys: if i < sample_size: if image_id not in training_image_ids: if im_gender_summary[image_id]['pred_gt'] == 'male' and ( male_count < sample_size / 3): test_captions_dict[image_id] = captions_dict[image_id] male_count += 1 i += 1 elif im_gender_summary[image_id][ 'pred_gt'] == 'female' and (female_count < sample_size / 3): test_captions_dict[image_id] = captions_dict[image_id] female_count += 1 i += 1 elif im_gender_summary[image_id][ 'pred_gt'] == 'neutral' and (neutral_count < sample_size / 3): test_captions_dict[image_id] = captions_dict[image_id] neutral_count += 1 i += 1 elif mode == 'balanced_clean': i = 0 male_count = 0 female_count = 0 neutral_count = 0 for image_id in shuffle_im_keys: if i < sample_size: if image_id not in training_image_ids: if im_gender_summary[image_id]['clean_gender'] == 1: if im_gender_summary[image_id][ 'pred_gt'] == 'male' and (male_count < sample_size / 3): test_captions_dict[image_id] = captions_dict[ image_id] male_count += 1 i += 1 elif im_gender_summary[image_id][ 'pred_gt'] == 'female' and (female_count < sample_size / 3): test_captions_dict[image_id] = captions_dict[ image_id] female_count += 1 i += 1 elif im_gender_summary[image_id][ 'pred_gt'] == 'neutral' and (neutral_count < sample_size / 3): test_captions_dict[image_id] = captions_dict[ image_id] neutral_count += 1 i += 1 return test_captions_dict
def get_training_indices(sample_size, mode='random'): assert mode in ['random','balanced_mode','balanced_clean', 'balanced_gender_only', \ 'balanced_clean_noun', 'clean_noun', 'activity_balanced', 'activity_balanced_clean'] assert isinstance(sample_size, int) ''' 8 different modes of generating data - random: randomized selection of qualified images - balanced_mode: balanced ratio between male, female and neutral - balanced_clean: balanced ratio between male, female and neutral, only use images when all captions agree on using the same gender - balanced_gender_only: same as balanced_mode, but without neutral captions - balanced_clean_noun: balanced ratio between male, female and neutral, only use images when all captions agree on using the same noun - clean_noun: only use images when all captions agree on the same noun - activity_balanced: from activity tagged image sets, choose same ratio of male, female, neutral image - activity_balanced_clean: similar to activity_balanced, but all captions must agree on the same gender Note that it is possible that output size may be smaller than sample_size, especially for activity_balanced and activity_balanced_clean. As for certain activities, the sample size of clean data might be limited for some classes, e.g. women wearing tie. ''' random.seed(123) training_captions_dict = dict() # Get pre-processed objects im_gender_summary = load_obj('im_gender_summary') captions_dict = load_obj('captions_dict') activity_image_ids = load_obj('activity_image_ids') if mode == 'random': training_captions_dict = dict( random.sample(captions_dict.items(), sample_size)) elif mode == 'balanced_mode': i = 0 male_count = 0 female_count = 0 neutral_count = 0 for image_id in im_gender_summary.keys(): if i < sample_size: if im_gender_summary[image_id]['pred_gt'] == 'male' and ( male_count < sample_size / 3): training_captions_dict[image_id] = captions_dict[image_id] male_count += 1 i += 1 elif im_gender_summary[image_id]['pred_gt'] == 'female' and ( female_count < sample_size / 3): training_captions_dict[image_id] = captions_dict[image_id] female_count += 1 i += 1 elif im_gender_summary[image_id]['pred_gt'] == 'neutral' and ( neutral_count < sample_size / 3): training_captions_dict[image_id] = captions_dict[image_id] neutral_count += 1 i += 1 if i % 1000 == 0: print(f"captions of {i} images are added") elif mode == 'balanced_clean': i = 0 male_count = 0 female_count = 0 neutral_count = 0 for image_id in im_gender_summary.keys(): if i < sample_size: if im_gender_summary[image_id]['clean_gender'] == 1: if im_gender_summary[image_id]['pred_gt'] == 'male' and ( male_count < sample_size / 3): training_captions_dict[image_id] = captions_dict[ image_id] male_count += 1 i += 1 elif im_gender_summary[image_id][ 'pred_gt'] == 'female' and (female_count < sample_size / 3): training_captions_dict[image_id] = captions_dict[ image_id] female_count += 1 i += 1 elif im_gender_summary[image_id][ 'pred_gt'] == 'neutral' and (neutral_count < sample_size / 3): training_captions_dict[image_id] = captions_dict[ image_id] neutral_count += 1 i += 1 if i % 1000 == 0: print(f"captions of {i} images are added") elif mode == 'balanced_clean_noun': i = 0 male_count = 0 female_count = 0 neutral_count = 0 for image_id in im_gender_summary.keys(): if i < sample_size: if im_gender_summary[image_id]['clean_noun'] == 1: if im_gender_summary[image_id]['pred_gt'] == 'male' and ( male_count < sample_size / 3): training_captions_dict[image_id] = captions_dict[ image_id] male_count += 1 i += 1 elif im_gender_summary[image_id][ 'pred_gt'] == 'female' and (female_count < sample_size / 3): training_captions_dict[image_id] = captions_dict[ image_id] female_count += 1 i += 1 elif im_gender_summary[image_id][ 'pred_gt'] == 'neutral' and (neutral_count < sample_size / 3): training_captions_dict[image_id] = captions_dict[ image_id] neutral_count += 1 i += 1 if i % 1000 == 0: print(f"captions of {i} images are added") elif mode == 'clean_noun': i = 0 for image_id in im_gender_summary.keys(): if i < sample_size: if im_gender_summary[image_id]['clean_noun'] == 1: training_captions_dict[image_id] = captions_dict[image_id] i += 1 if i % 1000 == 0: print(f"captions of {i} images are added") elif mode == 'balanced_gender_only': i = 0 male_count = 0 female_count = 0 for image_id in im_gender_summary.keys(): if i < sample_size: if im_gender_summary[image_id]['pred_gt'] == 'male' and ( male_count < sample_size / 2): training_captions_dict[image_id] = captions_dict[image_id] male_count += 1 i += 1 elif im_gender_summary[image_id]['pred_gt'] == 'female' and ( female_count < sample_size / 2): training_captions_dict[image_id] = captions_dict[image_id] female_count += 1 i += 1 if i % 1000 == 0: print(f"captions of {i} images are added") elif mode == 'activity_balanced': activity_sample_size = sample_size / len(activity_image_ids.keys()) i = 0 for activity in activity_image_ids.keys(): image_ids = activity_image_ids[activity] j = 0 male_count = 0 female_count = 0 neutral_count = 0 for image_id in image_ids: if j < activity_sample_size: if image_id in im_gender_summary: if im_gender_summary[image_id][ 'pred_gt'] == 'male' and ( male_count < activity_sample_size / 3): training_captions_dict[image_id] = captions_dict[ image_id] male_count += 1 i += 1 j += 1 elif im_gender_summary[image_id][ 'pred_gt'] == 'female' and ( female_count < activity_sample_size / 3): training_captions_dict[image_id] = captions_dict[ image_id] female_count += 1 i += 1 j += 1 elif im_gender_summary[image_id][ 'pred_gt'] == 'neutral' and ( neutral_count < activity_sample_size / 3): training_captions_dict[image_id] = captions_dict[ image_id] neutral_count += 1 i += 1 j += 1 if i > 0 and i % 100 == 0: print(f"captions of {i} images are added") elif mode == 'activity_balanced_clean': activity_sample_size = sample_size / len(activity_image_ids.keys()) i = 0 for activity in activity_image_ids.keys(): image_ids = activity_image_ids[activity] j = 0 male_count = 0 female_count = 0 neutral_count = 0 for image_id in image_ids: if j < activity_sample_size: if image_id in im_gender_summary and im_gender_summary[ image_id]['clean_noun'] == 1: if im_gender_summary[image_id][ 'pred_gt'] == 'male' and ( male_count < activity_sample_size / 3): training_captions_dict[image_id] = captions_dict[ image_id] male_count += 1 i += 1 j += 1 elif im_gender_summary[image_id][ 'pred_gt'] == 'female' and ( female_count < activity_sample_size / 3): training_captions_dict[image_id] = captions_dict[ image_id] female_count += 1 i += 1 j += 1 elif im_gender_summary[image_id][ 'pred_gt'] == 'neutral' and ( neutral_count < activity_sample_size / 3): training_captions_dict[image_id] = captions_dict[ image_id] neutral_count += 1 i += 1 j += 1 if i > 0 and i % 1000 == 0: print(f"captions of {i} images are added") training_image_ids = list(training_captions_dict.keys()) save_obj(training_image_ids, 'training_image_ids') return training_image_ids, training_captions_dict
data_loading_cleaning.cleaning_data(dict_df, 'data/filtered/') print() # transforming the data G, dict_geo_data, dict_distances = graph_transformation.\ graph_transforming('data/filtered/') print() # saving the results so that this step does not have to be performed again utils.save_obj(G, 'objects/graph.pkl') utils.save_obj(dict_geo_data, 'objects/dict_geo_data.pkl') utils.save_obj(dict_distances, 'objects/dict_distances.pkl') # analyzing the graph # loading the objects G = utils.load_obj('objects/graph.pkl') dict_geo_data = utils.load_obj('objects/dict_geo_data.pkl') dict_distances = utils.load_obj('objects/dict_distances.pkl') print('Computing some metrics...') print() graph_metrics.computing_metrics(G, 'current network', 'figures/current_network') dict_avg_speed = utils.computing_avg_speed_mode(G, dict_geo_data) current_efficiency, g_ideal, denom = graph_metrics\ .global_efficiency_weighted(G, dict_distances, dict_avg_speed['RER']) print('Current global efficiency of the network:', current_efficiency) # detecting new routes to create # setting the costs of each type of route # costs of creating a new line and exploiting it, in €/km
def predict_from_image(image_path, vocab_path = '', model_path = '', embed_size = 256, hidden_size = 512, is_print = True, return_one = False): sample_size = 1 # Get model if model_path == '': # if not specified, assume it is best model saved in models model_path = './models/best-model.pkl' if torch.cuda.is_available() == True: checkpoint = torch.load(model_path) else: checkpoint = torch.load(model_path, map_location=lambda storage, loc: storage) #checkpoint = torch.load(model_path, map_location='cpu') #print(f'Best model is loaded from {model_path} . . .') # Get the vocabulary and its size if vocab_path != '': # if not specified, assume it is the vocab pickle saved in object with open(vocab_path, 'rb') as f: vocab = pickle.load(f) #print("Loaded vocab file of pretrained model") else: vocab = load_obj('vocab') vocab_size = len(vocab) # Transform image transform = transforms.Compose([ transforms.Resize(256), transforms.RandomCrop(224), #transforms.RandomHorizontalFlip(), transforms.ToTensor(), transforms.Normalize((0.485, 0.456, 0.406),(0.229, 0.224, 0.225)), ]) image = Image.open(image_path).convert("RGB") original_image = np.array(image) image = transform(image) if is_print == True: transformed_image = image.numpy() transformed_image = np.squeeze(transformed_image) transformed_image = transformed_image.transpose((1, 2, 0)) np.clip(transformed_image, 0, 1) # Print sample image, before and after pre-processing plt.imshow(np.asarray(original_image)) #plt.imshow(np.squeeze(original_image)) plt.title('Test image- original') plt.show() plt.imshow(transformed_image) plt.title('Test image- transformed') plt.show() # Initialize the encoder and decoder, and set each to inference mode encoder = EncoderCNN(embed_size) encoder.eval() decoder = DecoderRNN(embed_size, hidden_size, vocab_size) decoder.eval() # Load the pre-trained weights encoder.load_state_dict(checkpoint['encoder']) decoder.load_state_dict(checkpoint['decoder']) # Move models to GPU if CUDA is available. if torch.cuda.is_available(): encoder.cuda() decoder.cuda() image = image.cuda() image = image.unsqueeze(0) features = encoder(image).unsqueeze(1) output = decoder.sample_beam_search(features) sentences = clean_sentence(output, vocab) if is_print == True: print('Predicted caption: \n') for sentence in set(sentences): print(f'{sentence}') elif return_one == True: return sentences[0] else: return [s for s in set(sentences)]
def predict_from_COCO(image_folder_path, vocab_path = '', model_path = '', training_image_ids_path = '', embed_size = 256, hidden_size = 512, mode = 'balanced_mode', test_image_id = '', is_print = True, return_one = False): sample_size = 1 # Get model if model_path == '': # if not specified, assume it is best model saved in models model_path = './models/best-model.pkl' if torch.cuda.is_available() == True: checkpoint = torch.load(model_path) else: checkpoint = torch.load(model_path, map_location=lambda storage, loc: storage) #checkpoint = torch.load('./models/best-model.pkl', map_location='cpu') #print(f'Best model is loaded from {model_path} . . .') # Get the vocabulary and its size if vocab_path != '': # if not specified, assume it is the vocab pickle saved in object with open(vocab_path, 'rb') as f: vocab = pickle.load(f) #print("Loaded vocab file of pretrained model") else: vocab = load_obj('vocab') vocab_size = len(vocab) # Get the training image id paths if training_image_ids_path == '': # if not specified, assume it is the vocab pickle saved in object training_image_ids = load_obj('training_image_ids') else: with open(training_image_ids_path, 'rb') as f: training_image_ids = pickle.load(f) test_image_ids = get_test_indices(sample_size, training_image_ids, mode = mode) image_id = list(test_image_ids.keys())[0] if is_print == False: test_image_ids[test_image_id] = test_image_ids.pop(image_id) test_loader = load_data(test_image_ids.keys(), image_folder_path, mode = 'test', vocab_file = vocab) original_image, image = next(iter(test_loader)) if is_print == True: transformed_image = image.numpy() transformed_image = np.squeeze(transformed_image) transformed_image = transformed_image.transpose((1, 2, 0)) np.clip(transformed_image, 0, 1) # Print sample image, before and after pre-processing print(f'\nTest_image_id: {image_id}') plt.imshow(np.squeeze(original_image)) plt.title('Test image- original') plt.show() plt.imshow(transformed_image) plt.title('Test image- transformed') plt.show() # Initialize the encoder and decoder, and set each to inference mode encoder = EncoderCNN(embed_size) encoder.eval() decoder = DecoderRNN(embed_size, hidden_size, vocab_size) decoder.eval() # Load the pre-trained weights encoder.load_state_dict(checkpoint['encoder']) decoder.load_state_dict(checkpoint['decoder']) # Move models to GPU if CUDA is available. if torch.cuda.is_available(): encoder.cuda() decoder.cuda() image = image.cuda() features = encoder(image).unsqueeze(1) output = decoder.sample_beam_search(features) sentences = clean_sentence(output, vocab) if is_print == True: print('Predicted caption: \n') for sentence in set(sentences): print(f'{sentence}') original_captions = test_image_ids[image_id] print('\n\nOriginal captions labelled by human annotators: \n') for caption in set(original_captions): print(caption) elif return_one == True: return sentences[0] else: return [s for s in set(sentences)]
def run(config: DictConfig, logger=None): # ----------- setup experiment ------------------- # # modify the datamodule def train_dataloader(self): ds = CutMixDatasetWrapper(self.train, **config.cutmix) return DataLoader(ds, shuffle=True, **self.config) LitDataModule.train_dataloader = train_dataloader # setup logging if logger is None: logger = logging.getLogger(__name__) set_seed(config.training.seed) logger.info(f"using seed {config.training.seed}") wandb.login(key=config.logger.api) # init wandb logger wb = load_obj(config.logger.class_name)(**config.logger.params) # log the training config to wandb # create a new hparam dictionary with the relevant hparams and # log the hparams to wandb wb_hparam = OrderedDict({ "training_fold": config.fold_num, "input_dims": config.training.image_dim, "batch_size": config.training.dataloaders.batch_size, "optimizer": config.optimizer.class_name, "scheduler": config.scheduler.class_name, "learning_rate": config.optimizer.params.lr, "weight_decay": config.optimizer.params.weight_decay, "num_epochs": config.training.num_epochs, }) wb.log_hyperparams(wb_hparam) # ----------- prepare datasets ------------------- # logger.info("Prepare Training/Validation Datasets.") processor = Preprocessor(config.csv_dir, config.json_dir, config.image_dir, 5) df = pd.read_csv(config.fold_csv_dir) imsdir = config.image_dir df.filePath = [ os.path.join(imsdir, df.image_id[i]) for i in range(len(df)) ] processor.dataframe = df fold_num = config.fold_num trainFold, valFold = processor.get_fold(fold_num) # testFold, valFold = train_test_split(valFold, stratify=valFold.label, test_size=0.5) trainFold.reset_index(drop=True, inplace=True) # testFold.reset_index(drop=True, inplace=True) valFold.reset_index(drop=True, inplace=True) # init weights for loss function weights = None # no weights for cutmix tfms_config = config.augmentation trn_augs = A.Compose( [ load_obj(augs.class_name)(**augs.params) for augs in tfms_config.train_augs ], p=1.0, ) valid_augs = A.Compose( [ load_obj(augs.class_name)(**augs.params) for augs in tfms_config.valid_augs ], p=1.0, ) test_augs = A.Compose( [ load_obj(augs.class_name)(**augs.params) for augs in tfms_config.test_augs ], p=1.0, ) tfms = { "train": trn_augs, "valid": valid_augs, "test": test_augs, } # init datamodule dl_config = config.training.dataloaders dm = LitDataModule(trainFold, valFold, valFold, tfms, dl_config) dm.setup() # set training total steps config.training.total_steps = (len(dm.train_dataloader()) * config.training.num_epochs) logger.info(f"Train dataset size: {len(dm.train_dataloader())}") logger.info(f"Validation dataset size: {len(dm.val_dataloader())}") # ----------- load lightning trainer ------------------- # trainer_cfg = config.lightning # init lightning callbacks chkpt = pl.callbacks.ModelCheckpoint(**trainer_cfg.model_checkpoint) cb_config = config.lightning.callbacks cbs = [ load_obj(module.class_name)(**module.params) for module in cb_config ] if config.log_to_stdout: cbs.append(PrintCallback(log=logger)) # init trainer args = trainer_cfg.init_args trainer = pl.Trainer(callbacks=cbs, checkpoint_callback=chkpt, logger=wb, **args) # ----------- init lightning module ------------------- # logger.info("Build network.") model = LitModel(config, weights=weights) # update model loss function to soft cross entropy loss model.loss_fn = SoftTargetCrossEntropy(weight=weights) model.unfreeze_classifier() wb.watch(model.net) model_name = config.model.params.model_name or config.model.class_name logger.info(f"Init from base net: {model_name}") logger.info( f"Uses {str(config.optimizer.class_name).split('.')[-1]} optimizer.") logger.info( f"Learning Rate: {config.optimizer.params.lr}, Weight Decay: {config.optimizer.params.weight_decay}" ) logger.info( f"Uses {str(config.scheduler.class_name).split('.')[-1]} scheduler.") tr_config = config.training logger.info( f"Training over {tr_config.num_epochs} epochs ~ {tr_config.total_steps} steps." ) # ----------- start train/validaiton/test ------------------- # # Pass the datamodule as arg to trainer.fit to override model hooks :) trainer.fit(model, datamodule=dm) # Compute metrics on test dataset _ = trainer.test(model, datamodule=dm, ckpt_path=chkpt.best_model_path) # ----------- finish experiment/cleanup/save weights ------------------- # PATH = chkpt.best_model_path # path to the best performing model WEIGHTS_PATH = config.training.model_save_dir # init best model logger.info(f"Restored best model weights from {PATH}.") params = {"config": config, "weights": weights} loaded_model = model.load_from_checkpoint(PATH, **params) torchmodel = loaded_model.net torch.save(torchmodel.state_dict(), WEIGHTS_PATH) # upload the weights file to wandb wandb.save(WEIGHTS_PATH) # upload the full config file to wandb conf_pth = f"{config.run_name}.yaml" OmegaConf.save(config, f=conf_pth) logger.info(f"Saved config file {conf_pth}.") wandb.save(conf_pth) logger.info(f"Saved model {WEIGHTS_PATH}.") wandb.finish()
def analysis(STATE, method, method_kwargs, hyperparams_to_test, fig, spec, row, precomputed=False, separate=False, two_cols=False, NUM_STATES=1, configurations=None, default_cluster_num=5): #First, define appropriate paths SHAPE_PATH, FIGURE_PATH, RAW_DATA_PATH, INCOME_POPULATION_PATH = define_paths( STATE) #Load the data covid_, X, index_X, columns_X = load_data(RAW_DATA_PATH) #Do dim red print('##################D-RED#################') emb_method = method if not precomputed: errors_results, embeddings_results, trustws_results = choose_dimension( X, emb_method, hyperparams_to_test, **method_kwargs) save_obj(embeddings_results, STATE + '_embeddings_results' + method.__name__) save_obj(errors_results, STATE + '_errors_results' + method.__name__) save_obj(trustws_results, STATE + '_trustws_result' + method.__name__) if precomputed: embeddings_results = load_obj(STATE + '_embeddings_results' + method.__name__) errors_results = load_obj(STATE + '_errors_results' + method.__name__) trustws_results = load_obj(STATE + '_trustws_result' + method.__name__) if (len(hyperparams_to_test['n_components']) > 1) and (errors_results['n_components'][0] is not None): plt.plot(hyperparams_to_test['n_components'], errors_results['n_components']) if (len(hyperparams_to_test['n_components']) > 1): kneedle = KneeLocator(hyperparams_to_test['n_components'], np.array(trustws_results['n_components']), S=1, curve='concave', direction='increasing', interp_method='polynomial', online=False) kneedle.plot_knee() plt.title(emb_method.__name__ + ' trustworthiness') plt.xlabel('n_components') plt.ylabel('trustworhiness') kneedle.knee, kneedle.knee_y #Save the dataframe with optimal dim if (len(hyperparams_to_test['n_components']) > 1): good_dim = int( np.squeeze( np.where(hyperparams_to_test['n_components'] == kneedle.knee))) else: good_dim = 0 X_method = embeddings_results['n_components'][ good_dim] #pick the best (knee point) n_components X_method_df = pd.DataFrame( X_method, columns=['Mode {}'.format(i) for i in range(X_method.shape[1])]) #, index = index_X) X_method_df.to_csv( os.path.join( configurations['DATA_PATH'], 'interim', method.__name__ + str(X_method.shape[1]) + 'D_' + STATE + '.csv')) print('Saving optimal embedding. Method: ', method.__name__, 'shape: ', X_method_df.shape) print('##################INITIAL VIZ#################') #Find the 2D and 3D embeddings and continuous colors based on that filename_initial = os.path.join(FIGURE_PATH, 'initial_' + method.__name__) if method.__name__ == 'Isomap': viz = viz_Isomap if method.__name__ == 'SpectralEmbedding': viz = viz_SE if method.__name__ == 'LocallyLinearEmbedding': viz = viz_LLE if precomputed: load_path = os.path.join('obj', STATE) save_path = None else: load_path = None save_path = os.path.join('obj', STATE) X_2D_emb, X_3D_emb = viz(X, colors=None, filename=filename_initial, alpha=0.5, load_path=load_path, save_path=save_path) cos_colors = find_cos_similarity(X_2D_emb) #Color the manifold continuously filename_initial_colored = os.path.join( FIGURE_PATH, 'initial_' + method.__name__ + '_colored') X_2D_emb, X_3D_emb = viz(X, colors=cos_colors, filename=filename_initial_colored, cbar=None, alpha=0.5, load_path=load_path, save_path=save_path) print('##################GMM CLUSTERING#################') #Import R for clustering base = importr('base') mclust = importr('mclust') ro.r('set.seed(1)') dontprecomputeclusters = not precomputed # if not precomputed: if dontprecomputeclusters: clusters, means, z, uncertainty = GMM_clustering_R( X_method_df, method, default_cluster_num=default_cluster_num ) #could change this to 5 to be consistent across states to auto-id clust # clusters_block_indexed = pd.Series(clusters, index=index_X) avg_per_clust = create_avg_df(clusters, index_X, covid_) reordered_clusters, reordered_means, reordered_z, reordered_uncertainty = relabel_clusters( clusters.astype('int'), avg_per_clust, means, z, uncertainty) reordered_avg_per_clust = create_avg_df(reordered_clusters, index_X, covid_) #Save np.save( os.path.join('obj', STATE + '_reordered_clusters.npy'), reordered_clusters, ) reordered_means.to_csv( os.path.join('obj', STATE + '_reordered_means.csv')) reordered_z.to_csv(os.path.join('obj', STATE + '_reordered_z.csv')) np.save(os.path.join('obj', STATE + '_reordered_uncertainty.npy'), reordered_uncertainty) reordered_avg_per_clust.to_csv( os.path.join('obj', STATE + '_reordered_avg_per_clust.csv')) # if precomputed: if not dontprecomputeclusters: reordered_clusters = np.load( os.path.join('obj', STATE + '_reordered_clusters.npy')) reordered_means = pd.read_csv(os.path.join( 'obj', STATE + '_reordered_means.csv'), index_col=0) reordered_z = pd.read_csv(os.path.join('obj', STATE + '_reordered_z.csv'), index_col=0) reordered_uncertainty = np.load( os.path.join('obj', STATE + '_reordered_uncertainty.npy')) reordered_avg_per_clust = pd.read_csv(os.path.join( 'obj', STATE + '_reordered_avg_per_clust.csv'), index_col=0) #Save the data for Dennis (for only this method) index_with_blocks_and_save(STATE, X_method_df, X_2D_emb, X_3D_emb, reordered_clusters, reordered_z, reordered_uncertainty, index_X, emb_method) N_TIMESERIES = 5 closest_to_mean_samples, closest_to_mean_block_ids = find_closest_time_series( X_method_df, reordered_means, covid_, index_X, n=N_TIMESERIES) print('##################FINAL VIZ#################') sns.set(style="whitegrid") if two_cols: reordered_clusters = cos_colors #Change colors add_state_to_fig(STATE, fig, spec, row, NUM_STATES, X, reordered_clusters, index_X, reordered_avg_per_clust, load_path=load_path, save_path=save_path, separate=separate, two_cols=two_cols, configurations=configurations)