def predict_to_ensemble(model_name, model_class, model_state_pth, image_size, normalize, nb_classes=15, batch_size=15, with_crops=True): print(f'[+] predict {model_name}') model = get_model(model_class, nb_classes, model_state_pth=model_state_pth) model.eval() tta_preprocess = [ preprocess(normalize, image_size), preprocess_hflip(normalize, image_size) ] if with_crops: tta_preprocess += make_transforms( [transforms.Resize((image_size + 20, image_size + 20))], [transforms.ToTensor(), normalize], five_crops(image_size)) print(f'[+] tta size: {len(tta_preprocess)}') data_loaders = [] for transform in tta_preprocess: data_loader = get_data_loader('./test/', data_transform=transform, batch_size=batch_size) data_loaders.append(data_loader) lx, px = utils.predict_tta(model, data_loaders) test_predict = { 'lx': lx.cpu(), 'px': px.cpu(), } torch.save(test_predict, f'{model_name}_test_prediction.pth') data_loaders = [] for transform in tta_preprocess: valid_dataset = ImageFolder('./data/train/', transform=transform) data_loader = get_data_loader('./data/train/', batch_size=batch_size, dataset=valid_dataset) data_loaders.append(data_loader) lx, px = utils.predict_tta(model, data_loaders) val_predict = { 'lx': lx.cpu(), 'px': px.cpu(), } torch.save(val_predict, f'{model_name}_val_prediction.pth') return {'test': test_predict, 'val': val_predict}
def predict(model_name, model_class, weight_pth, image_size, normalize): print(f'[+] predict {model_name}') model = get_model(model_class) model.load_state_dict(torch.load(weight_pth)) model.eval() tta_preprocess = [ preprocess(normalize, image_size), preprocess_hflip(normalize, image_size) ] tta_preprocess += make_transforms( [transforms.Resize((image_size + 20, image_size + 20))], [transforms.ToTensor(), normalize], five_crops(image_size)) tta_preprocess += make_transforms( [transforms.Resize((image_size + 20, image_size + 20))], [HorizontalFlip(), transforms.ToTensor(), normalize], five_crops(image_size)) print(f'[+] tta size: {len(tta_preprocess)}') data_loaders = [] for transform in tta_preprocess: test_dataset = FurnitureDataset('test', transform=transform) data_loader = DataLoader(dataset=test_dataset, num_workers=1, batch_size=BATCH_SIZE, shuffle=False) data_loaders.append(data_loader) lx, px = utils.predict_tta(model, data_loaders) data = { 'lx': lx.cpu(), 'px': px.cpu(), } torch.save(data, f'{model_name}_test_prediction.pth') data_loaders = [] for transform in tta_preprocess: test_dataset = FurnitureDataset('val', transform=transform) data_loader = DataLoader(dataset=test_dataset, num_workers=1, batch_size=BATCH_SIZE, shuffle=False) data_loaders.append(data_loader) lx, px = utils.predict_tta(model, data_loaders) data = { 'lx': lx.cpu(), 'px': px.cpu(), } torch.save(data, f'{model_name}_val_prediction.pth')
def predictCrop(model_name, model_class, weight_pth, image_size, normalize): print("[+] {0} predictCrop".format(model_name)) model = get_model(model_class) model.load_state_dict(torch.load(weight_pth)['state_dict']) model.eval() print('{0} load state dict done'.format(model_name)) tta_preprocess=[preprocess(normalize, image_size)] tta_preprocess += make_transforms([transforms.Resize((image_size + 20, image_size + 20))], [transforms.ToTensor(), normalize], five_crops(image_size)) print('[+] tta size: {0}'.format(len(tta_preprocess))) data_loaders = [] for transform in tta_preprocess: test_dataset = MyDataset(test_inputs,test_labels,transform=transform) data_loader = DataLoader(dataset=test_dataset, num_workers=16, batch_size=BATCH_SIZE, shuffle=False) data_loaders.append(data_loader) print('add transforms') lx, px = utils.predict_tta(model, data_loaders) data = { 'lx': lx.cpu(), 'px': px.cpu(), } if not os.path.exists('../feature/'+model_name): os.makedirs('../feature/'+model_name) torch.save(data, '../feature/'+model_name+'/crop_prediction.pth') print('Done')
def predict(): model = get_model() model.load_state_dict(torch.load('best_val_weight.pth')) model.eval() tta_preprocess = [preprocess, preprocess_hflip] # tta_preprocess = [preprocess, preprocess_hflip, preprocess_with_augmentation, # preprocess_with_augmentation, preprocess_with_augmentation, # preprocess_with_augmentation, preprocess_with_augmentation, # preprocess_with_augmentation, preprocess_with_augmentation, # preprocess_with_augmentation] data_loaders = [] for transform in tta_preprocess: test_dataset = FurnitureDataset('test2', transform=transform) data_loader = DataLoader(dataset=test_dataset, num_workers=1, batch_size=BATCH_SIZE, shuffle=False) data_loaders.append(data_loader) lx, px = utils.predict_tta(model, data_loaders) data = { 'lx': lx.cpu(), 'px': px.cpu(), } torch.save(data, 'test_prediction.pth')
def predict(args): model = get_model(args.name) model.load_state_dict( torch.load('models_trained/{}_{}_{}/best_val_weight_{}.pth'.format( args.name, args.aug, args.alpha, args.name))) model.eval() #tta_preprocess = [preprocess_five_crop, preprocess_five_crop_hflip] tta_preprocess = [preprocess, preprocess_hflip] data_loaders = [] for transform in tta_preprocess: test_dataset = FurnitureDataset('test', transform=transform) data_loader = DataLoader(dataset=test_dataset, num_workers=1, batch_size=BATCH_SIZE, shuffle=False) data_loaders.append(data_loader) lx, px = utils.predict_tta(model, data_loaders) data = { 'lx': lx.cpu(), 'px': px.cpu(), } torch.save( data, 'models_trained/{}_{}_{}/test_prediction_{}.pth'.format( args.name, args.aug, args.alpha, args.name)) data_loaders = [] for transform in tta_preprocess: test_dataset = FurnitureDataset('val', transform=transform) data_loader = DataLoader(dataset=test_dataset, num_workers=1, batch_size=BATCH_SIZE, shuffle=False) data_loaders.append(data_loader) lx, px = utils.predict_tta(model, data_loaders) data = { 'lx': lx.cpu(), 'px': px.cpu(), } torch.save( data, 'models_trained/{}_{}_{}/val_prediction_{}.pth'.format( args.name, args.aug, args.alpha, args.name))
def predict_process(path): model = models.resnet152(pretrained=True) num_ftrs = model.fc.in_features #model.fc = nn.Sequential(nn.Dropout(), nn.Linear(num_ftrs, 128)) model.fc = nn.Linear(num_ftrs, 128) model = model.cuda() model.load_state_dict(torch.load(path)) model.eval() tta = [trans.preprocess, trans.preprocess_hflip] data_loaders = [] for transform in tta: data_loaders.append(misc.test_prep(transform_=transform)) # Do the prediction lx, px = utils.predict_tta(model, data_loaders) # Create the prediction corredponding index list idx_list = [] pbar = tqdm(data_loaders[0], total=len(data_loaders[0])) for image, idx in pbar: idx_list = idx_list + idx.cpu().numpy().tolist() test_prob = F.softmax(Variable(px.cpu()), dim=1).data.numpy() test_prob = test_prob.mean(axis=2) test_predicted = np.argmax(test_prob, axis=1) test_predicted += 1 #print(test_predicted.size) idx_series = pd.Series(idx_list).to_frame() predicted = pd.Series(test_predicted).to_frame() temp = pd.merge(idx_series, predicted, left_index=True, right_index=True, how='outer') temp.columns = ['id', 'predicted'] read_id = pd.Series(range(1, 12801)).rename('id').to_frame() output = pd.merge(read_id, temp, left_on='id', right_on='id', how='left').fillna(20) output['predicted'] = output['predicted'].astype(int) name = path.split('/')[-1][:-4] output.to_csv( '/home/wenyue/Desktop/data_playground/kaggle_furniture/output/' + name + '.csv', index=False) return output
def predict(model_name, outputDir): model = get_model(model_name) model_checkpoint = torch.load(os.path.join(outputDir, 'best_val_acc_weight_' + model_name + '.pth')) model.load_state_dict(model_checkpoint) model.eval() tta_preprocess = [preprocess, preprocess_hflip] data_loaders = [] for transform in tta_preprocess: test_dataset = FurnitureDataset('test', transform=transform) data_loader = DataLoader(dataset=test_dataset, num_workers=1, batch_size=BATCH_SIZE, shuffle=False) data_loaders.append(data_loader) lx, px = utils.predict_tta(model, data_loaders, device) data = { 'lx': lx.cpu(), 'px': px.cpu(), } torch.save(data, os.path.join(outputDir, 'test_prediction_' + model_name + '.pth')) data_loaders = [] for transform in tta_preprocess: test_dataset = FurnitureDataset('val', transform=transform) data_loader = DataLoader(dataset=test_dataset, num_workers=1, batch_size=BATCH_SIZE, shuffle=False) data_loaders.append(data_loader) lx, px = utils.predict_tta(model, data_loaders, device) data = { 'lx': lx.cpu(), 'px': px.cpu(), } torch.save(data, os.path.join(outputDir, 'val_prediction_' + model_name + '.pth'))
def predict_process(paths, model_names): use_gpu = torch.cuda.is_available() for i in range(len(paths)): model = get_model(model_names[i]) if use_gpu: model = model.cuda() model.load_state_dict(torch.load(paths[i])) model.eval() tta = [ preprocess, preprocess_hflip, preprocess_aug, preprocess_scale_1, preprocess_scale_2 ] #,\ #preprocess_brightness_darker, preprocess_brightness_lighter\ # , preprocess_contrast_blur,\ # preprocess_contrast_sharp, preprocess_gamma_large, \ # preprocess_gamma_small] data_loaders = [] for transform in tta: data_loaders.append(misc.test_prep(transform_=transform)) lx, px = utils.predict_tta(model, data_loaders) test_prob = F.softmax(Variable(px.cpu()), dim=1).data.numpy() if i == 0: test_prob_1 = test_prob else: test_prob_1 = np.concatenate((test_prob, test_prob_1), axis=2) test_prob_save = test_prob.mean(axis=2) np.savetxt( '/home/wenyue/Desktop/data_playground/kaggle_furniture/prep/' + paths[i].split('/')[-1].split('.pth')[0], test_prob_save, delimiter=',') test_prob_1 = test_prob_1.mean(axis=2) test_predicted = np.argmax(test_prob_1, axis=1) test_predicted += 1 idx_list = [] pbar = tqdm(data_loaders[0], total=len(data_loaders[0])) for image, idx in pbar: idx_list = idx_list + idx.cpu().numpy().tolist() #print(test_predicted.size) idx_series = pd.Series(idx_list).to_frame() predicted = pd.Series(test_predicted).to_frame() temp = pd.merge(idx_series, predicted, left_index=True, right_index=True, how='outer') temp.columns = ['id', 'predicted'] read_id = pd.Series(range(1, 12801)).rename('id').to_frame() output = pd.merge(read_id, temp, left_on='id', right_on='id', how='left').fillna(20) output['predicted'] = output['predicted'].astype(int) name = paths[0].split('/')[-1][:-4] output.to_csv( '/home/wenyue/Desktop/data_playground/kaggle_furniture/output/' + name + '_' + str(len(tta)) + 'tta_new.csv', index=False) return output
def predict(epoch=None, attention=False): BATCH_SIZE = 8 if not epoch: epoch = EPOCH if attention: save_name = "att_" else: save_name = "" pth_path = OUTPUT_PATH + save_name + 'best_val_weight_%s.pth' % epoch print("loading %s" % pth_path) if not attention: model = get_model() else: print("loading model...") model = dense_attention201(pretrained=False, num_classes=128) if use_gpu: model.cuda() print("done.") model.load_state_dict(torch.load(pth_path)) model.eval() tta_preprocess = [ preprocess_for_test, preprocess_for_test, preprocess_for_test, preprocess, preprocess_hflip ] ################### TEST VALIDATION SET # data_loaders = [] # for transform in [preprocess]: # test_dataset = FurnitureDataset('validation', transform=transform) # data_loader = DataLoader(dataset=test_dataset, num_workers=0, # batch_size=BATCH_SIZE, # shuffle=False) # data_loaders.append(data_loader) # lx, px = utils.predict_tta(model, data_loaders) # data = { # 'lx': lx.cpu(), # 'px': px.cpu(), # } # _, preds = torch.max(px, dim=1) # accuracy = torch.mean((preds.view(-1) != lx).float()) # print("accuracy: {:.5f}".format(accuracy)) # torch.save(data, save_name + 'val_prediction.pth') ################### TEST VALIDATION SET data_loaders = [] print("number of tta: {}".format(len(tta_preprocess))) for transform in tta_preprocess: test_dataset = FurnitureDataset('test', transform=transform) data_loader = DataLoader(dataset=test_dataset, num_workers=0, batch_size=BATCH_SIZE, shuffle=False) data_loaders.append(data_loader) lx, px = utils.predict_tta(model, data_loaders, test=True) data = { #'lx': lx.cpu(), 'px': px.cpu(), } torch.save(data, save_name + 'test_prediction_e%s.pth' % epoch)