def deep_feature_extraction(): batch_size = 400 images_paths, cutoffs = create_image_list() logger.info('load image paths done. #video: {:5d}'.format(len(cutoffs))) dataloader = create_loader(images_paths, batch_size) logger.info('create data loader done') model = vgg16(pretrained=True).to(opt['device']) model.eval() logger.info('create cnn-model done') count = 0 buffer = [] filepath = os.path.join(opt['featurepath'], 'frames-features.h5') fw = h5py.File(filepath, mode='w') with torch.no_grad(): for iter, batch in enumerate(dataloader, 0): images, ind = batch images, ind = images.to(opt['device']), ind.numpy() features = list(model(images).data.cpu().numpy()) current_len = len(features) + len(buffer) buffer += features if current_len >= cutoffs[count][0]: while True: if count < len(cutoffs) and len(buffer) >= cutoffs[count][0]: offset = cutoffs[count][0] next_video = '' try: curr_ind = ind[0] - len(buffer) + len(features) + offset - 1 next_ind = curr_ind + 1 curr_video = images_paths[curr_ind].split('/')[-2] next_video = images_paths[next_ind].split('/')[-2] if curr_video == next_video: logger.info('Split Error: {}/{}'.format(curr_video, next_video)) return except: pass save_features = buffer[:offset] if not len(save_features) == cutoffs[count][0]: logger.info('Length Error: {}/{}'.format(len(save_features), cutoffs[count][0])) return save_features = np.array(save_features) video = cutoffs[count][1] fw.create_dataset(name=video, data=save_features) if opt['verbose'] and count % opt['output_period'] == 0: logger.info('CNT: {:6d}/{:6d}, Video: {}, Next Video: {}'.format( count, len(cutoffs), video, next_video )) buffer = buffer[offset:] count += 1 else: break fw.close()
# 'experiment/EfficientB4_parallel_data2203/checkpoint/best.ckpt', # 'experiment/ResneSt101_parallel_data2203/checkpoint/best.ckpt' ] id_leaf = [2,4,5,6,7,8] id_obs = [0,1,2,3,4,5,6,7,8,9,10,11,12] preds_stack_val = [] labels_stack_val = [] preds_stack_test = [] labels_stack_test = [] for i in range(len(list_batch)): cfg.batch_size = list_batch[i] cfg.long_side = list_res[i] train_loader = create_loader(cfg.train_csv, data_dir, cfg, mode='train', dicom=False, type=cfg.type) val_loader = create_loader(cfg.dev_csv, data_dir, cfg, mode='val', dicom=False, type=cfg.type) test_loader = create_loader(cfg.test_csv, data_dir, cfg, mode='test', dicom=False, type=cfg.type) print(f'{model_names[i]}-{ids[i]}:' ) cfg.backbone = model_names[i] cfg.id = ids[i] cfg.ckp_path = ckp_paths[i] chexpert_model = CheXpert_model(cfg, loss_func, metrics_dict) chexpert_model.load_ckp(cfg.ckp_path) preds, labels, _ = chexpert_model.predict_loader( val_loader, ensemble=False, cal_loss=False) preds_stack_val.append(preds) labels_stack_val.append(labels) preds, labels, _ = chexpert_model.predict_loader( test_loader, ensemble=False, cal_loss=False)
# warnings.simplefilter('always') warnings.filterwarnings("ignore") cfg_path = './config/test_config.json' # cfg_path = './config/example2.json' with open(cfg_path) as f: cfg = edict(json.load(f)) loss_func = BCEWithLogitsLoss() # data_dir = '/home/tungthanhlee/bdi_xray/data/images' data_dir = '/home/single1/BACKUP/thanhtt/assigned_jpeg' torch.cuda.set_device(cfg.device) val_loader = create_loader(cfg.dev_csv, data_dir, cfg, mode='val') test_loader = create_loader(cfg.test_csv, data_dir, cfg, mode='test') metrics_dict = { 'auc': AUC(), 'sensitivity': Recall(), 'specificity': Specificity(), 'f1': F1() } id_leaf = [2, 4, 5, 6, 7, 8] id_obs = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12] #------------------------------- additional config for ensemble --------------------------------------- model_names = [ 'dense',
from model.chexpert import CheXpert_model from data.dataset import create_loader import warnings # warnings.simplefilter('always') warnings.filterwarnings("ignore") # cfg_path = './config/chexmic_config.json' cfg_path = './config/example.json' with open(cfg_path) as f: cfg = edict(json.load(f)) data_dir = '/home/tungthanhlee/thanhtt/assigned_jpeg' train_loader = create_loader(cfg.train_csv, data_dir, cfg, mode='train') val_loader = create_loader(cfg.dev_csv, data_dir, cfg, mode='val') loss_func = BCEWithLogitsLoss() metrics_dict = { 'acc': ACC(), 'auc': AUC(), 'f1': F1(), 'precision': Precision(), 'recall': Recall() } loader_dict = {'train': train_loader, 'val': val_loader} chexpert_model = CheXpert_model(cfg, loss_func, metrics_dict) # print(chexpert_model.model)