def build_model(cfg, paramsfile): model = Classifier(cfg) model = model.to('cpu') ckpt = torch.load(paramsfile, map_location='cpu') state_dict = ckpt['state_dict'] if 'state_dict' in ckpt else ckpt model.load_state_dict(state_dict) if 'step' in ckpt and 'auc_dev_best' in ckpt: print(f"Using model '{paramsfile}' at step: {ckpt['step']} " f"with AUC: {ckpt['auc_dev_best']}") return model.eval()
def run(args): with open(args.cfg_path) as f: cfg = edict(json.load(f)) if args.verbose is True: print(json.dumps(cfg, indent=4)) if not os.path.exists(args.save_path): os.mkdir(args.save_path) if args.logtofile is True: logging.basicConfig(filename=args.save_path + '/log.txt', filemode="w", level=logging.INFO) else: logging.basicConfig(level=logging.INFO) if not args.resume: with open(os.path.join(args.save_path, 'cfg.json'), 'w') as f: json.dump(cfg, f, indent=1) device_ids = list(map(int, args.device_ids.split(','))) num_devices = torch.cuda.device_count() if num_devices < len(device_ids): raise Exception('#available gpu : {} < --device_ids : {}'.format( num_devices, len(device_ids))) device = torch.device('cuda:{}'.format(device_ids[0])) model = Classifier(cfg) if args.verbose is True: from torchsummary import summary if cfg.fix_ratio: h, w = cfg.long_side, cfg.long_side else: h, w = cfg.height, cfg.width summary(model.to(device), (3, h, w)) model = DataParallel(model, device_ids=device_ids).to(device).train() if args.pre_train is not None: if os.path.exists(args.pre_train): ckpt = torch.load(args.pre_train, map_location=device) model.module.load_state_dict(ckpt) optimizer = get_optimizer(model.parameters(), cfg) src_folder = os.path.dirname(os.path.abspath(__file__)) + '/../' dst_folder = os.path.join(args.save_path, 'classification') # rc, size = subprocess.getstatusoutput('dir --max-depth=0 %s | cut -f1' # % src_folder) # if rc != 0: # print(size) # raise Exception('Copy folder error : {}'.format(rc)) # rc, err_msg = subprocess.getstatusoutput('cp -R %s %s' % (src_folder, # dst_folder)) # if rc != 0: # raise Exception('copy folder error : {}'.format(err_msg)) copyfile(cfg.train_csv, os.path.join(args.save_path, 'train.csv')) copyfile(cfg.dev_csv, os.path.join(args.save_path, 'valid.csv')) dataloader_train = DataLoader(ImageDataset(cfg.train_csv, cfg, mode='train'), batch_size=cfg.train_batch_size, num_workers=args.num_workers, drop_last=True, shuffle=True) dataloader_dev = DataLoader(ImageDataset(cfg.dev_csv, cfg, mode='dev'), batch_size=cfg.dev_batch_size, num_workers=args.num_workers, drop_last=False, shuffle=False) dev_header = dataloader_dev.dataset._label_header summary_train = {'epoch': 0, 'step': 0} summary_dev = {'loss': float('inf'), 'acc': 0.0} summary_writer = SummaryWriter(args.save_path) epoch_start = 0 best_dict = { "acc_dev_best": 0.0, "auc_dev_best": 0.0, "loss_dev_best": float('inf'), "fused_dev_best": 0.0, "best_idx": 1 } if args.resume: ckpt_path = os.path.join(args.save_path, 'train.ckpt') ckpt = torch.load(ckpt_path, map_location=device) model.module.load_state_dict(ckpt['state_dict']) summary_train = {'epoch': ckpt['epoch'], 'step': ckpt['step']} best_dict['acc_dev_best'] = ckpt['acc_dev_best'] best_dict['loss_dev_best'] = ckpt['loss_dev_best'] best_dict['auc_dev_best'] = ckpt['auc_dev_best'] epoch_start = ckpt['epoch'] for epoch in range(epoch_start, cfg.epoch): lr = lr_schedule(cfg.lr, cfg.lr_factor, summary_train['epoch'], cfg.lr_epochs) for param_group in optimizer.param_groups: param_group['lr'] = lr summary_train, best_dict = train_epoch(summary_train, summary_dev, cfg, args, model, dataloader_train, dataloader_dev, optimizer, summary_writer, best_dict, dev_header) time_now = time.time() summary_dev, predlist, true_list = test_epoch(summary_dev, cfg, args, model, dataloader_dev) time_spent = time.time() - time_now auclist = [] for i in range(len(cfg.num_classes)): y_pred = predlist[i] y_true = true_list[i] fpr, tpr, thresholds = metrics.roc_curve(y_true, y_pred, pos_label=1) auc = metrics.auc(fpr, tpr) auclist.append(auc) summary_dev['auc'] = np.array(auclist) loss_dev_str = ' '.join( map(lambda x: '{:.5f}'.format(x), summary_dev['loss'])) acc_dev_str = ' '.join( map(lambda x: '{:.3f}'.format(x), summary_dev['acc'])) auc_dev_str = ' '.join( map(lambda x: '{:.3f}'.format(x), summary_dev['auc'])) logging.info('{}, Dev, Step : {}, Loss : {}, Acc : {}, Auc : {},' 'Mean auc: {:.3f} ' 'Run Time : {:.2f} sec'.format( time.strftime("%Y-%m-%d %H:%M:%S"), summary_train['step'], loss_dev_str, acc_dev_str, auc_dev_str, summary_dev['auc'].mean(), time_spent)) for t in range(len(cfg.num_classes)): summary_writer.add_scalar('dev/loss_{}'.format(dev_header[t]), summary_dev['loss'][t], summary_train['step']) summary_writer.add_scalar('dev/acc_{}'.format(dev_header[t]), summary_dev['acc'][t], summary_train['step']) summary_writer.add_scalar('dev/auc_{}'.format(dev_header[t]), summary_dev['auc'][t], summary_train['step']) save_best = False mean_acc = summary_dev['acc'][cfg.save_index].mean() if mean_acc >= best_dict['acc_dev_best']: best_dict['acc_dev_best'] = mean_acc if cfg.best_target == 'acc': save_best = True mean_auc = summary_dev['auc'][cfg.save_index].mean() if mean_auc >= best_dict['auc_dev_best']: best_dict['auc_dev_best'] = mean_auc if cfg.best_target == 'auc': save_best = True mean_loss = summary_dev['loss'][cfg.save_index].mean() if mean_loss <= best_dict['loss_dev_best']: best_dict['loss_dev_best'] = mean_loss if cfg.best_target == 'loss': save_best = True if save_best: torch.save( { 'epoch': summary_train['epoch'], 'step': summary_train['step'], 'acc_dev_best': best_dict['acc_dev_best'], 'auc_dev_best': best_dict['auc_dev_best'], 'loss_dev_best': best_dict['loss_dev_best'], 'state_dict': model.module.state_dict() }, os.path.join(args.save_path, 'best{}.ckpt'.format(best_dict['best_idx']))) best_dict['best_idx'] += 1 if best_dict['best_idx'] > cfg.save_top_k: best_dict['best_idx'] = 1 logging.info('{}, Best, Step : {}, Loss : {}, Acc : {},' 'Auc :{},Best Auc : {:.3f}'.format( time.strftime("%Y-%m-%d %H:%M:%S"), summary_train['step'], loss_dev_str, acc_dev_str, auc_dev_str, best_dict['auc_dev_best'])) torch.save( { 'epoch': summary_train['epoch'], 'step': summary_train['step'], 'acc_dev_best': best_dict['acc_dev_best'], 'auc_dev_best': best_dict['auc_dev_best'], 'loss_dev_best': best_dict['loss_dev_best'], 'state_dict': model.module.state_dict() }, os.path.join(args.save_path, 'train.ckpt')) summary_writer.close()
def run(args, val_h5_file): with open(args.cfg_path) as f: cfg = edict(json.load(f)) if args.verbose is True: print(json.dumps(cfg, indent=4)) if not os.path.exists(args.save_path): os.mkdir(args.save_path) if args.logtofile is True: logging.basicConfig(filename=args.save_path + '/log.txt', filemode="w", level=logging.INFO) else: logging.basicConfig(level=logging.INFO) if not args.resume: with open(os.path.join(args.save_path, 'cfg.json'), 'w') as f: json.dump(cfg, f, indent=1) device_ids = list(map(int, args.device_ids.split(','))) num_devices = torch.cuda.device_count() if num_devices < len(device_ids): raise Exception('#available gpu : {} < --device_ids : {}'.format( num_devices, len(device_ids))) device = torch.device('cuda:{}'.format(device_ids[0])) model = Classifier(cfg) if args.verbose is True: from torchsummary import summary if cfg.fix_ratio: h, w = cfg.long_side, cfg.long_side else: h, w = cfg.height, cfg.width summary(model.to(device), (3, h, w)) model = DataParallel(model, device_ids=device_ids).to(device).train() if args.pre_train is not None: if os.path.exists(args.pre_train): ckpt = torch.load(args.pre_train, map_location=device) model.module.load_state_dict(ckpt) optimizer = get_optimizer(model.parameters(), cfg) #src_folder = os.path.dirname(os.path.abspath(__file__)) + '/../' #dst_folder = os.path.join(args.save_path, 'classification') #rc, size = subprocess.getstatusoutput('du --max-depth=0 %s | cut -f1' % src_folder) #if rc != 0: raise Exception('Copy folder error : {}'.format(rc)) #rc, err_msg = subprocess.getstatusoutput('cp -R %s %s' % (src_folder, dst_folder)) #if rc != 0: raise Exception('copy folder error : {}'.format(err_msg)) #copyfile(cfg.train_csv, os.path.join(args.save_path, 'train.csv')) #copyfile(cfg.dev_csv, os.path.join(args.save_path, 'dev.csv')) # np_train_h5_file = np.array(train_h5_file['train'][:10000], dtype=np.uint8) # np_t_u_ones = np.array(train_h5_file['train_u_ones'][:10000], dtype=np.int8) # np_t_u_zeros = np.array(train_h5_file['train_u_zeros'][:10000], dtype=np.int8) # np_t_u_random = np.array(train_h5_file['train_u_random'][:10000], dtype=np.int8) np_val_h5_file = np.array(val_h5_file['val'], dtype=np.uint8) np_v_u_ones = np.array(val_h5_file['val_u_ones'], dtype=np.int8) np_v_u_zeros = np.array(val_h5_file['val_u_zeros'], dtype=np.int8) np_v_u_random = np.array(val_h5_file['val_u_random'], dtype=np.int8) train_labels = {} with h5py.File(f'{args.train_chunks}/train_labels.h5', 'r') as fp: train_labels['train_u_ones'] = np.array(fp['train_u_ones'], dtype=np.int8) train_labels['train_u_zeros'] = np.array(fp['train_u_zeros'], dtype=np.int8) train_labels['train_u_random'] = np.array(fp['train_u_random'], dtype=np.int8) np_train_samples = None for i in range(args.chunk_count): with open(f'{args.train_chunks}/chexpert_dset_chunk_{i+1}.npy', 'rb') as f: if np_train_samples is None: np_train_samples = np.load(f) else: np_train_samples = np.concatenate( (np_train_samples, np.load(f))) dataloader_train = DataLoader(ImageDataset( [np_train_samples, train_labels], cfg, mode='train'), batch_size=cfg.train_batch_size, num_workers=args.num_workers, drop_last=True, shuffle=True) dataloader_dev = DataLoader(ImageDataset( [np_val_h5_file, np_v_u_zeros, np_v_u_ones, np_v_u_random], cfg, mode='val'), batch_size=cfg.dev_batch_size, num_workers=args.num_workers, drop_last=False, shuffle=False) #dev_header = dataloader_dev.dataset._label_header dev_header = [ 'No_Finding', 'Enlarged_Cardiomediastinum', 'Cardiomegaly', 'Lung_Opacity', 'Lung_Lesion', 'Edema', 'Consolidation', 'Pneumonia', 'Atelectasis', 'Pneumothorax', 'Pleural_Effusion', 'Pleural_Other', 'Fracture', 'Support_Devices' ] print(f'dataloaders are set. train count: {np_train_samples.shape[0]}') logging.info("[LOGGING TEST]: dataloaders are set...") summary_train = {'epoch': 0, 'step': 0} summary_dev = {'loss': float('inf'), 'acc': 0.0} summary_writer = SummaryWriter(args.save_path) epoch_start = 0 best_dict = { "acc_dev_best": 0.0, "auc_dev_best": 0.0, "loss_dev_best": float('inf'), "fused_dev_best": 0.0, "best_idx": 1 } if args.resume: ckpt_path = os.path.join(args.save_path, 'train.ckpt') ckpt = torch.load(ckpt_path, map_location=device) model.module.load_state_dict(ckpt['state_dict']) summary_train = {'epoch': ckpt['epoch'], 'step': ckpt['step']} best_dict['acc_dev_best'] = ckpt['acc_dev_best'] best_dict['loss_dev_best'] = ckpt['loss_dev_best'] best_dict['auc_dev_best'] = ckpt['auc_dev_best'] epoch_start = ckpt['epoch'] q_list = [] k_list = [] for i in range(len(cfg.num_classes)): q_list.append(args.q) k_list.append(args.k) k_list = torch.FloatTensor(k_list) q_list = torch.FloatTensor(q_list) loss_sq_hinge = MultiClassSquaredHingeLoss() print('Everything is set starting to train...') before = datetime.datetime.now() for epoch in range(epoch_start, cfg.epoch): lr = lr_schedule(cfg.lr, cfg.lr_factor, summary_train['epoch'], cfg.lr_epochs) for param_group in optimizer.param_groups: param_group['lr'] = lr summary_train, best_dict = train_epoch(summary_train, summary_dev, cfg, args, model, dataloader_train, dataloader_dev, optimizer, summary_writer, best_dict, dev_header, q_list, k_list, loss_sq_hinge) time_now = time.time() summary_dev, predlist, true_list = test_epoch(summary_dev, cfg, args, model, dataloader_dev, q_list, k_list, loss_sq_hinge) time_spent = time.time() - time_now auclist = [] for i in range(len(cfg.num_classes)): y_pred = predlist[i] y_true = true_list[i] fpr, tpr, thresholds = metrics.roc_curve(y_true, y_pred, pos_label=1) auc = metrics.auc(fpr, tpr) auclist.append(auc) summary_dev['auc'] = np.array(auclist) loss_dev_str = ' '.join( map(lambda x: '{:.5f}'.format(x), summary_dev['loss'])) acc_dev_str = ' '.join( map(lambda x: '{:.3f}'.format(x), summary_dev['acc'])) auc_dev_str = ' '.join( map(lambda x: '{:.3f}'.format(x), summary_dev['auc'])) logging.info('{}, Dev, Step : {}, Loss : {}, Acc : {}, Auc : {},' 'Mean auc: {:.3f} ' 'Run Time : {:.2f} sec'.format( time.strftime("%Y-%m-%d %H:%M:%S"), summary_train['step'], loss_dev_str, acc_dev_str, auc_dev_str, summary_dev['auc'].mean(), time_spent)) for t in range(len(cfg.num_classes)): summary_writer.add_scalar('dev/loss_{}'.format(dev_header[t]), summary_dev['loss'][t], summary_train['step']) summary_writer.add_scalar('dev/acc_{}'.format(dev_header[t]), summary_dev['acc'][t], summary_train['step']) summary_writer.add_scalar('dev/auc_{}'.format(dev_header[t]), summary_dev['auc'][t], summary_train['step']) save_best = False mean_acc = summary_dev['acc'][cfg.save_index].mean() if mean_acc >= best_dict['acc_dev_best']: best_dict['acc_dev_best'] = mean_acc if cfg.best_target == 'acc': save_best = True mean_auc = summary_dev['auc'][cfg.save_index].mean() if mean_auc >= best_dict['auc_dev_best']: best_dict['auc_dev_best'] = mean_auc if cfg.best_target == 'auc': save_best = True mean_loss = summary_dev['loss'][cfg.save_index].mean() if mean_loss <= best_dict['loss_dev_best']: best_dict['loss_dev_best'] = mean_loss if cfg.best_target == 'loss': save_best = True if save_best: torch.save( { 'epoch': summary_train['epoch'], 'step': summary_train['step'], 'acc_dev_best': best_dict['acc_dev_best'], 'auc_dev_best': best_dict['auc_dev_best'], 'loss_dev_best': best_dict['loss_dev_best'], 'state_dict': model.module.state_dict() }, os.path.join(args.save_path, 'best{}.ckpt'.format(best_dict['best_idx']))) best_dict['best_idx'] += 1 if best_dict['best_idx'] > cfg.save_top_k: best_dict['best_idx'] = 1 logging.info('{}, Best, Step : {}, Loss : {}, Acc : {},' 'Auc :{},Best Auc : {:.3f}'.format( time.strftime("%Y-%m-%d %H:%M:%S"), summary_train['step'], loss_dev_str, acc_dev_str, auc_dev_str, best_dict['auc_dev_best'])) torch.save( { 'epoch': summary_train['epoch'], 'step': summary_train['step'], 'acc_dev_best': best_dict['acc_dev_best'], 'auc_dev_best': best_dict['auc_dev_best'], 'loss_dev_best': best_dict['loss_dev_best'], 'state_dict': model.module.state_dict() }, os.path.join(args.save_path, 'train.ckpt')) print_remaining_time(before, epoch + 1, cfg.epoch, additional='[training]') summary_writer.close()
def run_fl(args): with open(args.cfg_path) as f: cfg = edict(json.load(f)) if args.verbose is True: print(json.dumps(cfg, indent=4)) if not os.path.exists(args.save_path): os.mkdir(args.save_path) if args.logtofile is True: logging.basicConfig(filename=args.save_path + '/log.txt', filemode="w", level=logging.INFO) else: logging.basicConfig(level=logging.INFO) if not args.resume: with open(os.path.join(args.save_path, 'cfg.json'), 'w') as f: json.dump(cfg, f, indent=1) device_ids = list(map(int, args.device_ids.split(','))) num_devices = torch.cuda.device_count() if num_devices < len(device_ids): raise Exception('#available gpu : {} < --device_ids : {}'.format( num_devices, len(device_ids))) device = torch.device('cuda:{}'.format(device_ids[0])) # initialise global model model = Classifier(cfg).to(device).train() if args.verbose is True: from torchsummary import summary if cfg.fix_ratio: h, w = cfg.long_side, cfg.long_side else: h, w = cfg.height, cfg.width summary(model.to(device), (3, h, w)) if args.pre_train is not None: if os.path.exists(args.pre_train): ckpt = torch.load(args.pre_train, map_location=device) model.load_state_dict(ckpt) src_folder = os.path.dirname(os.path.abspath(__file__)) + '/../' dst_folder = os.path.join(args.save_path, 'classification') rc, size = subprocess.getstatusoutput('du --max-depth=0 %s | cut -f1' % src_folder) if rc != 0: raise Exception('Copy folder error : {}'.format(rc)) else: print('Successfully determined size of directory') rc, err_msg = subprocess.getstatusoutput('cp -R %s %s' % (src_folder, dst_folder)) if rc != 0: raise Exception('copy folder error : {}'.format(err_msg)) else: print('Successfully copied folder') # copy train files train_files = cfg.train_csv clients = {} for i, c in enumerate(string.ascii_uppercase): if i < len(train_files): clients[c] = {} else: break # initialise clients for i, client in enumerate(clients): copyfile(train_files[i], os.path.join(args.save_path, f'train_{client}.csv')) clients[client]['dataloader_train'] =\ DataLoader( ImageDataset(train_files[i], cfg, mode='train'), batch_size=cfg.train_batch_size, num_workers=args.num_workers,drop_last=True, shuffle=True ) clients[client]['bytes_uploaded'] = 0.0 clients[client]['epoch'] = 0 copyfile(cfg.dev_csv, os.path.join(args.save_path, 'dev.csv')) dataloader_dev = DataLoader(ImageDataset(cfg.dev_csv, cfg, mode='dev'), batch_size=cfg.dev_batch_size, num_workers=args.num_workers, drop_last=False, shuffle=False) dev_header = dataloader_dev.dataset._label_header w_global = model.state_dict() summary_train = {'epoch': 0, 'step': 0} summary_dev = {'loss': float('inf'), 'acc': 0.0} summary_writer = SummaryWriter(args.save_path) comm_rounds = cfg.epoch best_dict = { "acc_dev_best": 0.0, "auc_dev_best": 0.0, "loss_dev_best": float('inf'), "fused_dev_best": 0.0, "best_idx": 1 } # Communication rounds loop for cr in range(comm_rounds): logging.info('{}, Start communication round {} of FL - {} ...'.format( time.strftime("%Y-%m-%d %H:%M:%S"), cr + 1, cfg.fl_technique)) w_locals = [] for client in clients: logging.info( '{}, Start local training process for client {}, communication round: {} ...' .format(time.strftime("%Y-%m-%d %H:%M:%S"), client, cr + 1)) # Load previous current global model as start point model = Classifier(cfg).to(device).train() model.load_state_dict(w_global) if cfg.fl_technique == "FedProx": global_weight_collector = get_global_weights(model, device) else: global_weight_collector = None optimizer = get_optimizer(model.parameters(), cfg) # local training loops for epoch in range(cfg.local_epoch): lr = lr_schedule(cfg.lr, cfg.lr_factor, epoch, cfg.lr_epochs) for param_group in optimizer.param_groups: param_group['lr'] = lr summary_train, best_dict = train_epoch_fl( summary_train, summary_dev, cfg, args, model, clients[client]['dataloader_train'], dataloader_dev, optimizer, summary_writer, best_dict, dev_header, epoch, global_weight_collector) summary_train['step'] += 1 bytes_to_upload = sys.getsizeof(model.state_dict()) clients[client]['bytes_uploaded'] += bytes_to_upload logging.info( '{}, Completed local rounds for client {} in communication round {}. ' 'Uploading {} bytes to server, {} bytes in total sent from client' .format(time.strftime("%Y-%m-%d %H:%M:%S"), client, cr + 1, bytes_to_upload, clients[client]['bytes_uploaded'])) w_locals.append(model.state_dict()) if cfg.fl_technique == "FedAvg": w_global = fed_avg(w_locals) elif cfg.fl_technique == 'WFedAvg': w_global = weighted_fed_avg(w_locals, cfg.train_proportions) elif cfg.fl_technique == 'FedProx': # Use weighted FedAvg when using FedProx w_global = weighted_fed_avg(w_locals, cfg.train_proportions) # Test the performance of the averaged model avged_model = Classifier(cfg).to(device) avged_model.load_state_dict(w_global) time_now = time.time() summary_dev, predlist, true_list = test_epoch(summary_dev, cfg, args, avged_model, dataloader_dev) time_spent = time.time() - time_now auclist = [] for i in range(len(cfg.num_classes)): y_pred = predlist[i] y_true = true_list[i] fpr, tpr, thresholds = metrics.roc_curve(y_true, y_pred, pos_label=1) auc = metrics.auc(fpr, tpr) auclist.append(auc) auc_summary = np.array(auclist) loss_dev_str = ' '.join( map(lambda x: '{:.5f}'.format(x), summary_dev['loss'])) acc_dev_str = ' '.join( map(lambda x: '{:.3f}'.format(x), summary_dev['acc'])) auc_dev_str = ' '.join(map(lambda x: '{:.3f}'.format(x), auc_summary)) logging.info( '{}, Averaged Model -> Dev, Step : {}, Loss : {}, Acc : {}, Auc : {},' 'Mean auc: {:.3f} ' 'Run Time : {:.2f} sec'.format(time.strftime("%Y-%m-%d %H:%M:%S"), summary_train['step'], loss_dev_str, acc_dev_str, auc_dev_str, auc_summary.mean(), time_spent)) for t in range(len(cfg.num_classes)): summary_writer.add_scalar('dev/loss_{}'.format(dev_header[t]), summary_dev['loss'][t], summary_train['step']) summary_writer.add_scalar('dev/acc_{}'.format(dev_header[t]), summary_dev['acc'][t], summary_train['step']) summary_writer.add_scalar('dev/auc_{}'.format(dev_header[t]), auc_summary[t], summary_train['step']) save_best = False mean_acc = summary_dev['acc'][cfg.save_index].mean() if mean_acc >= best_dict['acc_dev_best']: best_dict['acc_dev_best'] = mean_acc if cfg.best_target == 'acc': save_best = True mean_auc = auc_summary[cfg.save_index].mean() if mean_auc >= best_dict['auc_dev_best']: best_dict['auc_dev_best'] = mean_auc if cfg.best_target == 'auc': save_best = True mean_loss = summary_dev['loss'][cfg.save_index].mean() if mean_loss <= best_dict['loss_dev_best']: best_dict['loss_dev_best'] = mean_loss if cfg.best_target == 'loss': save_best = True if save_best: torch.save( { 'epoch': summary_train['epoch'], 'step': summary_train['step'], 'acc_dev_best': best_dict['acc_dev_best'], 'auc_dev_best': best_dict['auc_dev_best'], 'loss_dev_best': best_dict['loss_dev_best'], 'state_dict': avged_model.state_dict() }, os.path.join(args.save_path, 'best{}.ckpt'.format(best_dict['best_idx']))) best_dict['best_idx'] += 1 if best_dict['best_idx'] > cfg.save_top_k: best_dict['best_idx'] = 1 logging.info('{}, Best, Step : {}, Loss : {}, Acc : {},' 'Auc :{},Best Auc : {:.3f}'.format( time.strftime("%Y-%m-%d %H:%M:%S"), summary_train['step'], loss_dev_str, acc_dev_str, auc_dev_str, best_dict['auc_dev_best'])) torch.save( { 'epoch': cr, 'step': summary_train['step'], 'acc_dev_best': best_dict['acc_dev_best'], 'auc_dev_best': best_dict['auc_dev_best'], 'loss_dev_best': best_dict['loss_dev_best'], 'state_dict': avged_model.state_dict() }, os.path.join(args.save_path, 'train.ckpt'))
with open(join(args.save_path, 'cfg.json'), 'w') as f: json.dump(cfg, f, indent=1) device_ids = list(map(int, args.device_ids.split(','))) num_devices = torch.cuda.device_count() assert num_devices >= len(device_ids), f""" #available gpu : {num_devices} < --device_ids : {len(device_ids)}""" device = torch.device(f"cuda:{device_ids[0]}") model = Classifier(cfg) if args.verbose: from torchsummary import summary h, w = (cfg.long_side, cfg.long_side) if cfg.fix_ratio \ else (cfg.height, cfg.width) summary(model.to(device), (3, h, w)) model = DataParallel(model, device_ids=device_ids).to(device) if args.pre_train is not None: if exists(args.pre_train): ckpt = torch.load(args.pre_train, map_location=device) model.module.load_state_dict(ckpt) optimizer = get_optimizer(model.parameters(), cfg) trainset = ImageDataset(cfg.train_csv, cfg, mode='train') testset = ImageDataset(cfg.dev_csv, cfg, mode='val') trainloader = DataLoader(trainset, batch_size=cfg.train_batch_size, num_workers=args.num_workers, drop_last=True, shuffle=True) testloader = DataLoader(testset, batch_size=cfg.dev_batch_size,
with open(args.model_path+'cfg.json') as f: cfg = edict(json.load(f)) model_file = "model/best.pth" device = torch.device('cpu') # PyTorch v0.4.0 net = Classifier(cfg) ckpt = torch.load("model/best.ckpt") net.load_state_dict(ckpt['state_dict'], strict=False) torch.save(net, model_file) net.eval() dummy_input = torch.ones([1, 3, 1024, 1024]) net.to(device) output = net(dummy_input) device = torch.device("cuda") # PyTorch v0.4.0 summary(net.to(device), (3, 1024, 1024)) pytorch_parser = PytorchParser(model_file, [3, 1024, 1024]) # pytorch_parser.run(model_file) Model_FILE = model_file + '.prototxt' PRETRAINED = model_file + '.caffemodel' net = caffe.Classifier(Model_FILE, PRETRAINED)