def runQualityBenchForModel(arch: str, only_best=False, persist_results=False)-> None: #data_path = 'data/imagenet_full' data_path = DATA_PATH arch_states = [d for d in os.listdir(STATE_DICT_PATH) if arch.split("-")[0] in d] if only_best: arch_states = [d for d in arch_states if 'best' in d] print(arch_states) _, _, val_loader = get_zipped_dataloaders(data_path, BATCH_SIZE, use_valid=True) with torch.no_grad(): for state in arch_states: measurements = [] for _ in range(0, QUALITY_RUNS): model = getModelWithOptimized(arch, n=LAYERS_TO_SKIP, batch_size=BATCH_SIZE) model.eval() model, _, _, _ = resumeFromPath(os.path.join(STATE_DICT_PATH, state), model) #logging.info(f"Resuming {state} from epoch {epoch} with best precision {prec}...") classes = getLabelToClassMapping(os.path.join(os.getcwd(), data_path)) grndT, pred = evaluateModel(model, val_loader, classes, BATCH_SIZE) measurements.append(metrics.accuracy_score(grndT, pred)) if persist_results: logging.info(metrics.classification_report(grndT, pred, digits=3)) generateAndStoreClassificationReportCSV(grndT, pred, f'{arch}_report.csv') logging.debug(measurements) logging.info('') logging.info(measurements) logging.info(f'Precission: {sum(measurements) / len(measurements):.4f}')
def loadAndEvaluate(args): model = getModel(args.arch) if os.path.exists( os.path.join(CHECKPOINT_DIR, args.arch + '_model_best.pth.tar')): logging.debug("Loading best model") load_path = os.path.join(CHECKPOINT_DIR, args.arch + '_model_best.pth.tar') else: logging.debug("Loading default model") load_path = os.path.join(CHECKPOINT_DIR, args.arch + '_checkpoint.pth.tar') logging.debug('Loading: ' + load_path) model, _, _, _ = resumeFromPath(load_path, model) logging.debug('Loading Test Data..') _, _, testLoader = get_zipped_dataloaders(DATA_PATH, BATCH_SIZE, use_valid=True) grndT, pred = evaluateModel(model, testLoader) printStats(grndT, pred)
def test000_testDenseNet121Output_withLoss_noException(self): test_batch = 1 test_loader, _, _ = get_zipped_dataloaders(self.TEST_DATASET_PATH, test_batch) test_criterion = nn.CrossEntropyLoss() model = getModel('densenet121') for i, (img, target) in enumerate(test_loader): output = model(img) test_criterion(output, target) if i == 0: break
def test040_DenseNetWithDenseNetDropLastNPolicy_NoExcpetion_OnForwardingWithBatchSize(self): test_batch_size = 8 test_loader, _ , _ = get_zipped_dataloaders(self.TEST_DATASET_PATH, test_batch_size) img, _ = next(iter(test_loader)) test_net = getModelWithOptimized('densenet121-skip-last', 3, test_batch_size) with torch.no_grad(): output = test_net(img) self.assertIsNotNone(output) self.assertEqual(img.shape[0], test_batch_size)
def test030_labelAndIndexMapping(self): test_batch = 1 test_loader, _, _ = get_zipped_dataloaders(self.TEST_DATASET_PATH, test_batch) img, target = next(iter(test_loader)) index_path = os.path.join(self.TEST_DATASET_PATH, 'index-train.txt') class_to_global_index = getClassToIndexMapping(index_path) label_to_class = list(set(class_to_global_index)) label_to_class.sort() self.assertEqual(len(label_to_class), 40) self.assertEqual(len(class_to_global_index), len(test_loader)) index_path = os.path.join(self.TEST_DATASET_PATH, 'index-val.txt') class_to_global_val_index = getClassToIndexMapping(index_path) label_to_class_val = list(set(class_to_global_val_index)) label_to_class_val.sort() self.assertEqual(len(label_to_class_val), len(label_to_class)) self.assertEqual(label_to_class_val, label_to_class)
def executeBenchmark(args): _, _, loader = get_zipped_dataloaders(args.data_root, args.batch_size, use_valid=True) label_to_classes = getLabelToClassMapping( os.path.join(os.getcwd(), args.data_root)) for bench_type in args.bench_types: for arch, pol in args.arch_pol_tupl_ls: d = { 'run': [], 'skip_n': [], 'bench_type': [], 'arch': [], 'pol': [], 'prec': [], 'rec': [], 'acc': [], 'f1': [], 'time': [] } #config tqdm logging.info(f'Running {bench_type}-Bench on {arch}-{pol}...') skip_layers_list = args.skip_layers_values runs = args.runs arch_name = f'{arch}-{pol}' if pol == 'none': arch_name = arch skip_layers_list = [0] if bench_type == 'quality': runs = 1 with tqdm(total=(len(skip_layers_list) * runs), ncols=80, desc=f'Progress-{bench_type}-{arch}-{pol}') as pbar: for skip_n in skip_layers_list: for run in range(runs): prec = 0.0 rec = 0.0 acc = 0.0 f1 = 0.0 time = 0.0 try: if bench_type == 'quality': prec, acc, rec, f1 = executeQualityBench( arch_name, loader, skip_n, label_to_classes, args.batch_size) #print(f'{run} - {skip_n} - {bench_type} - {arch} - {pol} - {prec:.6f} - {rec:.6f} - {acc:.6f} - {f1:.6f}') elif bench_type == 'speed': time = executeSpeedBench(arch_name, skip_n) #print(f'{run} - {skip_n} - {bench_type} - {arch} - {pol} - {time:.6f}') else: print('Benchmark type not supported') quit(1) except Exception as e: logging.info( f'Exception occured in {bench_type}: {e}\n continueing...' ) print(f'run: {run}') print(f'skip: {skip_n}') print(f'bench_type: {bench_type}') print(f'arch: {arch}') print(f'pol: {pol}') traceback.print_exc() continue d['run'].append(run) d['skip_n'].append(skip_n) d['bench_type'].append(bench_type) d['arch'].append(arch) d['pol'].append(pol) d['prec'].append(prec) d['rec'].append(rec) d['acc'].append(acc) d['f1'].append(f1) d['time'].append(time) pbar.update(1) filename = f'{bench_type}-{arch}-{pol}-run.csv' storeReportToCSV(args.reports_path, filename, d)
def main(args): torch.cuda.empty_cache() n_gpus_per_node = torch.cuda.device_count() logging.info(f"Found {n_gpus_per_node} GPU(-s)") # MAIN LOOP #model = get_msd_net_model() model = msdnet.models.msdnet(args) criterion = nn.CrossEntropyLoss() if torch.cuda.is_available(): logging.debug("Cuda is available.") logging.info("Using all available GPUs") for i in range(torch.cuda.device_count()): logging.info(f"gpu:{i} - {torch.cuda.get_device_name(i)}") model = nn.DataParallel(model).cuda() logging.info("Moving criterion to device.") criterion = criterion.cuda() cudnn.benchmark = True else: logging.info("Using slow CPU training.") optimizer = torch.optim.SGD(model.parameters(), args.lr, momentum=args.momentum, weight_decay=args.weight_decay) calc_lr = lambda epoch: args.lr**(1 + epoch // 30) scheduler = torch.optim.lr_scheduler.LambdaLR(optimizer, lr_lambda=calc_lr) train_loader, val_loader, test_loader = get_zipped_dataloaders( args.data_root, args.batch_size, use_valid=True) best_prec1, best_epoch, start_epoch = 0.0, 0, 0 if hasattr(args, 'epoch') and args.epoch: # RESUME model, optimizer, start_epoch, best_prec1 = resumeFromPath( os.path.join(os.getcwd(), CHECKPOINT_DIR, f'msdnet10_{args.epoch}{CHECKPOINT_POSTFIX}'), model, optimizer) for epoch in range(start_epoch, args.epochs): logging.info(f"Started Epoch {epoch + 1}/{args.epochs}") # train() train_loss, train_prec1, train_prec5, lr = train( train_loader, model, criterion, optimizer, scheduler, epoch) logging.info('******** Train result: *********') logging.info(f'Train - Epoch: [{epoch}]\t' f'Loss {train_loss:.4f}\t' f'Acc@1 {train_prec1:.4f}\t' f'Acc@5 {train_prec5:.4f}\t' f'LR {lr}\t') # validate() val_loss, val_prec1, val_prec5 = validate(val_loader, model, criterion) scheduler.step() is_best = val_prec1 > best_prec1 if is_best: best_prec1 = val_prec1 best_epoch = epoch logging.info(f'Best val_prec1 {best_prec1}') if is_best or epoch % CHECKPOINT_INTERVALL == 0: save_checkpoint( getStateDict(model, epoch, 'msdnet10', best_prec1, optimizer), is_best, 'msdnet10', CHECKPOINT_DIR) if epoch % args.test_interval == 0: avg_loss, avg_top1, avg_top5 = validate(test_loader, model, criterion) logging.info(f'Best val_prec1: {best_prec1:.4f} at epoch {best_epoch}') logging.info('*************** Final prediction results ***************') validate(test_loader, model, criterion)
def main(args): torch.cuda.empty_cache() n_gpus_per_node = torch.cuda.device_count() logging.info(f"Found {n_gpus_per_node} GPU(-s)") # create model model = getModel(args.arch) logging.info(f"Training Arch:{args.arch}") if not torch.cuda.is_available(): logging.warning("Using CPU for slow training process") else: logging.debug("Cuda is available") if GPU_ID is not None: logging.info(f"Using specific GPU: {GPU_ID}") logging.warning( "This will reduce the training speed significantly.") torch.cuda.set_device(GPU_ID) model.cuda(GPU_ID) else: logging.info("Using all available GPUs") for i in range(torch.cuda.device_count()): logging.info(f"gpu:{i} - {torch.cuda.get_device_name(i)}") model = nn.DataParallel(model).cuda() # loss function (criterion) and optimizer if torch.cuda.is_available(): logging.info("Move cross entropy to device") criterion = nn.CrossEntropyLoss().cuda() else: criterion = nn.CrossEntropyLoss() optimizer = torch.optim.SGD(model.parameters(), LEARNING_RATE, momentum=MOMENTUM, weight_decay=WEIGHT_DECAY) cudnn.benchmark = True batch_size = BATCH_SIZE if not 'batch' in args else args.batch train_loader, test_loader, _ = get_zipped_dataloaders(os.path.join( os.getcwd(), "data", "imagenet_full"), batch_size, use_valid=True) # size of batch: logging.debug(get_batch_size_stats(train_loader)) if args.resume: model, optimizer, start_epoch, best_acc = resumeFromPath( os.path.join(os.getcwd(), CHECKPOINT_DIR, f"{args.arch}_{args.epoch}{CHECKPOINT_POSTFIX}"), model, optimizer) else: start_epoch = START_EPOCH best_acc = 0.0 checkpoint_time = AverageMeter('Checkpoint Time', ':6.3f') epoch_time = AverageMeter('Epoch Time', ':6.3f') # train loop end = time.time() for epoch in range(start_epoch, EPOCHS): adjust_learning_rate(optimizer, epoch) # train for one epoch logging.debug('Running train loop') train(train_loader, model, criterion, optimizer, epoch) #evaluate the network on test set logging.debug('Compute accuracy') acc = validate(test_loader, model, criterion) # remember top acc is_best = acc > best_acc best_acc = max(acc, best_acc) # safe model if epoch % CHECKPOINT_INTERVALL == 0 or is_best or IS_DEBUG or epoch == EPOCHS - 1: start = time.time() save_checkpoint( getStateDict(model, epoch, args.arch, best_acc, optimizer), is_best, args.arch, os.path.join(os.getcwd(), CHECKPOINT_DIR)) checkpoint_time.update(time.time() - start) logging.info(checkpoint_time) if IS_DEBUG: break epoch_time.update(time.time() - end) end = time.time() logging.info(epoch) logging.info( f"Avg-Epoch={epoch_time.avg}sec, Avg-Checkp.={checkpoint_time.avg}sec" ) logging.info(f"Best accuracy: {best_acc}")
def getDataLoader(args): _, _, loader = get_zipped_dataloaders(args.data_root, args.batch_size, use_valid=True) return loader