def main(): # Use first line of file docstring as description if it exists. parser = argparse.ArgumentParser( description=__doc__.split('\n')[0] if __doc__ else '', formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument( '--dat-dir', type=Path, help=('Contains a .dat file for each sequence, unless ' '--dat-files-in-subdirs is specified, in which case it should ' 'contain a subdirectory for each sequence, which in turn has a ' 'single dat file.')) parser.add_argument('--dat-file', type=Path) parser.add_argument( '--output', help=('Output directory. If --dat-file is specified, this refers to ' 'the output file. Default: {{dat_file_without_extension}}.npy ' 'if --dat-file is specified, else ' '{{dat_dir}}/numpy-predictions if --dat-dir is specified.')) parser.add_argument( '--label-size-space-separated', action='store_true', help=('The FBMS code requires that there is a newline between <track ' 'label> and <track size>, but one of the outputs I downloaded ' 'from a method has the two fields separated by a space. This ' 'boolean allows parsing such files.')) parser.add_argument( '--dat-files-in-subdirs', action='store_true', help=('See --dat-dir help for information. Ignored if --dat-file is ' 'specified.')) args = parser.parse_args() assert (args.dat_file is None) != (args.dat_dir is None), ( 'Exactly one of --dat-file or --dat-dir must be specified.') if args.dat_file: if not args.output: args.output = '{dat_file_without_extension}.npy' args.output = Path( args.output.format( dat_file_without_extension=args.dat_file.with_suffix(''))) else: if not args.output: args.output = '{dat_dir}/numpy-predictions' args.output = Path(args.output.format(dat_dir=args.dat_dir)) assert not args.output.exists() if args.dat_dir is not None: args.output.mkdir(parents=True) setup_logging(args.output / (Path(__file__).name + '.log')) else: setup_logging(args.output + '.log') logging.info('Args:\n%s' % args) if args.dat_dir is not None: sequences = [] inputs = [] if args.dat_files_in_subdirs: for sequence_dir in args.dat_dir.iterdir(): if sequence_dir.is_dir(): dat_file = list(sequence_dir.glob('*.dat')) if len(dat_file) != 1: logging.error( 'Found %s (!= 1) dat files in %s, skipping...' % (len(dat_file), sequence_dir)) continue inputs.append(dat_file[0]) sequences.append(sequence_dir.name) else: for sequence_dat in args.dat_dir.glob('*.dat'): sequences.append(sequence_dat.stem) inputs.append(sequence_dat) outputs = [args.output / (x + '.npy') for x in sequences] else: inputs = [args.dat_file] outputs = [args.output] single_input = len(inputs) == 1 for input_path, output_path in zip(tqdm(inputs, disable=single_input), outputs): with open(input_path, 'r') as f: tracks_txt = f.read() width, height = Image.open(next(input_path.parent.glob('*.ppm'))).size segmentation = parse_tracks( tracks_txt, image_shape=(height, width), track_label_size_space_separated=args.label_size_space_separated, progress=True) np.save(output_path, segmentation)
def main(): tracking_parser = create_tracking_parser() # Use first line of file docstring as description if it exists. parser = argparse.ArgumentParser( description=__doc__.split('\n')[0] if __doc__ else '', parents=[tracking_parser], formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('--detectron-dir', type=Path, required=True) parser.add_argument('--images-dir', type=Path, required=True) parser.add_argument('--output-images-dir', type=Path) parser.add_argument('--output-video', type=Path) parser.add_argument('--output-video-fps', default=30, type=float) parser.add_argument( '--output-track-file', type=Path, help='Optional; path to output MOT17 style tracking output.') parser.add_argument('--extensions', nargs='*', default=IMG_EXTENSIONS) parser.add_argument('--dataset', default='coco', choices=['coco', 'objectness']) parser.add_argument( '--filename-format', choices=[ 'frame', 'frameN', 'sequence_frame', 'sequence-frame', 'fbms' ], default='frame', help=('Specifies how to get frame number from the filename. ' '"frame": the filename is the frame number, ' '"frameN": format <frame><number>, ' '"sequence_frame": frame number is separated by an underscore, ' '"sequence-frame": frame number is separated by a dash, ' '"fbms": assume fbms style frame numbers')) tracking_params, remaining_argv = tracking_parser.parse_known_args() args = parser.parse_args(remaining_argv) tracking_params = vars(tracking_params) assert ( args.output_images_dir is not None or args.output_video is not None or args.output_track_file is not None), ( 'One of --output-dir, --output-video, or --output-track-file must ' 'be specified.') if args.output_track_file is not None: output_log_file = args.output_track_file.with_name( args.output_track_file.stem + '-tracker.log') elif args.output_video is not None: output_log_file = args.output_video.with_name(args.output_video.stem + '-tracker.log') elif args.output_images_dir is not None: output_log_file = args.output_images_dir / 'tracker.log' output_log_file.parent.mkdir(exist_ok=True, parents=True) setup_logging(output_log_file) logging.info('Printing source code to logging file') with open(__file__, 'r') as f: logging.debug(f.read()) logging.info('Args: %s', pprint.pformat(vars(args))) logging.info('Tracking params: %s', pprint.pformat(tracking_params)) subprocess.call([ './git-state/save_git_state.sh', output_log_file.with_suffix('.git-state') ]) detectron_input = args.detectron_dir if not detectron_input.is_dir(): raise ValueError('--detectron-dir %s is not a directory!' % args.detectron_dir) if args.filename_format == 'fbms': from utils.fbms.utils import get_framenumber elif args.filename_format == 'frameN': def get_framenumber(x): return int(x.split('frame')[1]) elif args.filename_format == 'sequence-frame': def get_framenumber(x): return int(x.split('-')[-1]) elif args.filename_format == 'sequence_frame': def get_framenumber(x): return int(x.split('_')[-1]) elif args.filename_format == 'frame': get_framenumber = int else: raise ValueError('Unknown --filename-format: %s' % args.filename_format) detection_results = load_detectron_pickles(args.detectron_dir, frame_parser=get_framenumber) track_and_visualize(detection_results, args.images_dir, tracking_params, get_framenumber, args.extensions, vis_dataset=args.dataset, output_images_dir=args.output_images_dir, output_video=args.output_video, output_video_fps=args.output_video_fps, output_track_file=args.output_track_file)
def main_worker(args): global best_prec1, dtype best_prec1 = 0 dtype = torch_dtypes.get(args.dtype) torch.manual_seed(args.seed) time_stamp = datetime.now().strftime('%Y-%m-%d_%H-%M-%S') if args.eval_path: args.results_dir = os.path.join(args.results_dir, 'evaluating_results') else: args.results_dir = os.path.join(args.results_dir, 'training_results') if not os.path.exists(args.results_dir): os.mkdir(args.results_dir) if args.save is '': args.save = time_stamp save_path = os.path.join(args.results_dir, args.save) args.distributed = args.local_rank >= 0 or args.world_size > 1 if not os.path.exists(save_path) and not (args.distributed and args.local_rank > 0): os.makedirs(save_path) setup_logging(os.path.join(save_path, 'log.txt'), resume=args.resume is not '', dummy=args.distributed and args.local_rank > 0) results_path = os.path.join(save_path, 'results') results = ResultsLog(results_path, title='Training Results - %s' % args.save) if 'cuda' in args.device and torch.cuda.is_available(): torch.cuda.manual_seed_all(args.seed) torch.cuda.set_device(args.device_ids[0]) cudnn.benchmark = True else: args.device_ids = None dump_args(args, os.path.join(save_path, 'args.txt')) # Pruning and evaluating if len(args.pruning_percs) > 0: results = [] checkpoint = torch.load(args.eval_path, map_location="cpu") for perc in args.pruning_percs: model, criterion = create_model_and_criterion(args) model = register_stats_collectors(model) model = prune_model(model, checkpoint['state_dict'], prune_perc=perc) res = eval_checkpoint(args, model, criterion)['prec1'] results.append(res) # After we gathered min/max statistics we can gather also histograms. if args.gather_histograms == True: model = register_hist_collectors(model) _ = eval_checkpoint(args, model, criterion)['prec1'] dump_buffers(model, save_path) for perc, res in zip(args.pruning_percs, results): print("prune%:", perc, " acc:", res)
def main(): global args, best_psnr args = parser.parse_args() # massage args block_opts = [] block_opts = args.block_opts block_opts.append(args.block_overlap) time_stamp = datetime.now().strftime('%Y-%m-%d_%H-%M-%S') if args.save is '': args.save = time_stamp save_path = os.path.join(args.results_dir, args.save) if not os.path.exists(save_path): os.makedirs(save_path) setup_logging(os.path.join(save_path, 'log_%s.txt' % time_stamp)) results_file = os.path.join(save_path, 'results.%s') results = ResultsLog(results_file % 'csv', results_file % 'html') logging.info("saving to %s", save_path) logging.debug("run arguments: %s", args) random.seed(args.seed) np.random.seed(args.seed) torch.manual_seed(args.seed) torch.cuda.manual_seed(args.seed) if args.encoder_lr > 0: encoder_learn = True else: encoder_learn = False # create model if args.pretrained_net is not None: logging.info("=> using pre-trained model '{}'".format(args.arch)) model = models.__dict__[args.arch]( block_opts, pretrained=args.pretrained_net, mask_path=args.mask_path, mean=args.mean, std=args.std, noise=args.noise, encoder_learn=encoder_learn, p=args.bernoulli_p, K=args.layers_k) else: logging.info("=> creating model '{}'".format(args.arch)) model = models.__dict__[args.arch]( block_opts, mask_path=args.mask_path, mean=args.mean, std=args.std, noise=args.noise, encoder_learn=encoder_learn, p=args.bernoulli_p, K=args.layers_k) model = torch.nn.DataParallel(model, device_ids=args.gpus).cuda() # define loss function (criterion) and optimizer mseloss = loss.EuclideanDistance(args.batch_size) # annual scedule if encoder_learn: optimizer = torch.optim.SGD([ {'params': model.module.measurements.parameters(), 'lr': args.encoder_lr}, {'params': model.module.reconstruction.parameters()}], args.decoder_lr, momentum=args.momentum, weight_decay=args.weight_decay) def lambda1(epoch): return 0.0 if epoch >= args.encoder_annual[2] else ( args.encoder_annual[0] ** bisect_right(range(args.encoder_annual[1], args.encoder_annual[2], args.encoder_annual[1]), epoch)) def lambda2( epoch): return args.decoder_annual[0] ** bisect_right([args.decoder_annual[1]], epoch) scheduler = torch.optim.lr_scheduler.LambdaLR( optimizer, lr_lambda=[lambda1, lambda2]) else: optimizer = torch.optim.SGD([ {'params': model.module.reconstruction.parameters()}], args.decoder_lr, momentum=args.momentum, weight_decay=args.weight_decay) scheduler = torch.optim.lr_scheduler.MultiStepLR( optimizer, milestones=[args.decoder_annual[1]], gamma=args.decoder_annual[0]) # optionally resume from a checkpoint if args.resume: if os.path.isfile(args.resume): logging.info("=> loading checkpoint '{}'".format(args.resume)) checkpoint = torch.load(args.resume) args.start_epoch = checkpoint['epoch'] best_psnr = checkpoint['best_psnr'] model.load_state_dict(checkpoint['state_dict']) optimizer.load_state_dict(checkpoint['optimizer']) logging.info("=> loaded checkpoint '{}' (epoch {})" .format(args.resume, checkpoint['epoch'])) else: logging.info("=> no checkpoint found at '{}'".format(args.resume)) cudnn.benchmark = True # Data loading code train_loader = torch.utils.data.DataLoader( datasets.videocs.VideoCS(args.data_train, args.block_opts, transforms.Compose([ transforms.ToTensor(), ]), hdf5=args.hdf5), batch_size=args.batch_size, shuffle=False, num_workers=args.workers, pin_memory=True) val_loader = torch.utils.data.DataLoader( datasets.videocs.VideoCS(args.data_val, args.block_opts, transforms.Compose([ transforms.ToTensor(), ]), hdf5=False), batch_size=1, shuffle=False, num_workers=0, pin_memory=True) # Save initial mask if encoder_learn: initial_weights = binarization( model.module.measurements.weight.clone()) perc_1 = initial_weights.mean().cpu().data.numpy()[0] logging.info('Percentage of 1: {}'.format(perc_1)) np.save(save_path + '/initial_mask.npy', model.module.measurements.weight.clone()) else: # binarize weights model.module.measurements.binarization() perc_1 = model.module.measurements.weight.clone().mean().cpu().data.numpy()[ 0] logging.info('Percentage of 1: {}'.format(perc_1)) # perform first validation validate(val_loader, model, encoder_learn) for epoch in range(args.start_epoch, args.epochs): # Annual schedule enforcement scheduler.step() logging.info(scheduler.get_lr()) if encoder_learn: save_binary_weights_before = binarization( model.module.measurements.weight.clone()) # train for one epoch train_loss = train(train_loader, model, optimizer, epoch, mseloss, encoder_learn, args.gradient_clipping) if encoder_learn: save_binary_weights_after = binarization( model.module.measurements.weight.clone()) diff = np.int(torch.abs(save_binary_weights_after - save_binary_weights_before).sum().cpu().data.numpy()) perc_1 = save_binary_weights_after.mean().cpu().data.numpy()[0] logging.info( 'Binary Weights Changed: {} - Percentage of 1: {}'.format(diff, perc_1)) else: perc1 = model.module.measurements.weight.clone().mean().cpu().data.numpy()[0] logging.info('Percentage of 1: {}'.format(perc_1)) # evaluate on validation set psnr = validate(val_loader, model, encoder_learn) # remember best psnr and save checkpoint is_best = psnr > best_psnr best_psnr = max(psnr, best_psnr) save_checkpoint({ 'epoch': epoch + 1, 'arch': args.arch, 'state_dict': model.state_dict(), 'best_psnr': best_psnr, 'optimizer': optimizer.state_dict(), }, is_best, path=save_path) results_add(epoch, results, train_loss, psnr) if encoder_learn: model.module.measurements.restore()
def main(): # Use first line of file docstring as description if it exists. parser = argparse.ArgumentParser( description=__doc__.split('\n')[0] if __doc__ else '', formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument( '--input-dir', help=('Contains numpy file of predictions for each sequence. Each ' 'file should be named {sequence}.npy and contain a (height, ' 'width, num_frames) numpy array. Also see ' '--predictions-in-subdir for alternative directory structure.')) parser.add_argument( '--predictions-pickle', help=('Use pickle file containing dictionary mapping image_id to ' 'prediction numpy array, instead of --input-dir. This can also ' 'be used to resume from a previous run of this script, as this ' 'script dumps an "predictions.pkl" file in the output dir. ' 'Mutually exclusive with --input-dir.')) parser.add_argument( '--filename-format', choices=['frame', 'sequence_frame', 'sequence-frame', 'fbms'], default='frame', help=('Specifies how to get frame number from the "file_name" stored ' 'in COCO annotations. ' '"frame": the filename is the frame number, ' '"sequence_frame": frame number is separated by an underscore, ' '"sequence-frame": frame number is separated by a dash, ' '"fbms": assume fbms style frame numbers')) parser.add_argument('--annotations-json', required=True) parser.add_argument('--output-dir', required=True) parser.add_argument('--visualize', action='store_true', help='Whether to visualize masks.') parser.add_argument( '--no-remove-largest-mask', dest='remove_largest_mask', action='store_false', help=('By default, we remove the mask with the largest area, making ' 'the assumption that it is background. Setting this flag ' 'avoids this behavior (i.e. keeps the largest area mask).')) parser.add_argument( '--min-mask-portion', default=0.001, type=float, help='Remove masks that are less than this portion of image size.') parser.add_argument( '--predictions-in-subdir', action='store_true', help=('If true, assume --input-dir contains a subdirectory for each ' 'sequence, each of which contains a "results.npy" file.')) args = parser.parse_args() assert (args.input_dir is None) != (args.predictions_pickle is None), ( 'Exactly one of --input-dir or --predictions-pickle required.') if args.input_dir: input_dir = Path(args.input_dir) assert input_dir.exists() output_dir = Path(args.output_dir) output_dir.mkdir(exist_ok=True, parents=True) _source_path = Path(__file__) logging_path = output_dir / (_source_path.stem + _source_path.suffix + '.log') setup_logging(str(logging_path)) logging.info('Args:\n%s', vars(args)) groundtruth = COCO(args.annotations_json) if args.filename_format == 'fbms': def parse_frame_offset(path): return get_frameoffset(path.parent.stem, get_framenumber(path)) elif args.filename_format == 'sequence-frame': def parse_frame_offset(x): return int(x.split('-')[-1]) elif args.filename_format == 'sequence_frame': def parse_frame_offset(x): return int(x.split('_')[-1]) elif args.filename_format == 'frame': parse_frame_offset = int else: raise ValueError('Unknown --filename-format: %s' % args.filename_format) if args.input_dir: image_to_predictions_numpy = load_numpy_annotations( input_dir, groundtruth, parse_frame_offset, predictions_in_results_npy=args.predictions_in_subdir) with open(output_dir / 'predictions.pkl', 'wb') as f: pickle.dump(image_to_predictions_numpy, f) else: # Use to continue from a previous run. with open(args.predictions_pickle, 'rb') as f: image_to_predictions_numpy = pickle.load(f) if args.visualize: from PIL import Image max_masks_per_image = max( len(np.unique(x)) for x in image_to_predictions_numpy.values()) colors = (np.random.rand(max_masks_per_image, 3) * 256).round() vis_dir = output_dir / 'vis' vis_dir.mkdir() annotations = [] for image_id, annotation_np in tqdm(image_to_predictions_numpy.items()): ids = sorted(np.unique(annotation_np)) masks = [annotation_np == object_id for object_id in ids] # Sort masks by area masks = sorted(masks, key=lambda mask: mask.sum()) if args.remove_largest_mask: masks = masks[:-1] # Remove mask with largest area (background) if args.visualize: vis_mask = np.zeros( (annotation_np.shape[0], annotation_np.shape[1], 3), dtype=np.uint8) for i, mask in enumerate(masks): vis_mask[mask] = colors[i] Image.fromarray(vis_mask).save(vis_dir / ('%s.png' % image_id)) if not masks: continue image_area = masks[0].shape[0] * masks[0].shape[1] masks_np = np.array(masks, dtype=np.uint8).transpose(1, 2, 0) rle_masks = mask_util.encode(np.asfortranarray(masks_np)) for rle_mask in rle_masks: # See https://github.com/cocodataset/cocoapi/issues/70 rle_mask['counts'] = rle_mask['counts'].decode('ascii') area = mask_util.area(rle_mask).item() ratio = area / image_area score = 0.75 # ratio * 0.3 + 0.7 # Map to (0.7, 1) range if ratio < args.min_mask_portion: continue annotations.append({ 'image_id': image_id, 'segmentation': rle_mask, 'category_id': 1, 'score': score, 'area': ratio, 'bbox': mask_util.toBbox(rle_mask).tolist() }) for ann_id, ann in enumerate(annotations): ann['id'] = ann_id + 1 ann['iscrowd'] = 0 with open(output_dir / 'results.json', 'w') as f: json.dump(annotations, f) predictions = groundtruth.loadRes(annotations) coco_eval = COCOeval(groundtruth, predictions, 'bbox') coco_eval.evaluate() coco_eval.accumulate() summary_f = StringIO() with redirect_stdout(summary_f): coco_eval.summarize() logging.info('Detection evaluation summary:\n%s', summary_f.getvalue()) coco_eval = COCOeval(groundtruth, predictions, 'segm') coco_eval.evaluate() coco_eval.accumulate() summary_f = StringIO() with redirect_stdout(summary_f): coco_eval.summarize() logging.info('Segmentation evaluation summary:\n%s', summary_f.getvalue())
def main_worker(args): global best_prec1, dtype acc = -1 loss = -1 best_prec1 = 0 dtype = torch_dtypes.get(args.dtype) random.seed(args.seed) np.random.seed(args.seed) torch.manual_seed(args.seed) time_stamp = datetime.now().strftime('%Y-%m-%d_%H-%M-%S') if args.evaluate: args.results_dir = '/tmp' if args.save is '': args.save = time_stamp save_path = os.path.join(args.results_dir, args.save) args.distributed = args.local_rank >= 0 or args.world_size > 1 if args.distributed: dist.init_process_group(backend=args.dist_backend, init_method=args.dist_init, world_size=args.world_size, rank=args.local_rank) args.local_rank = dist.get_rank() args.world_size = dist.get_world_size() if args.dist_backend == 'mpi': # If using MPI, select all visible devices args.device_ids = list(range(torch.cuda.device_count())) else: args.device_ids = [args.local_rank] if not os.path.exists(save_path) and not (args.distributed and args.local_rank > 0): os.makedirs(save_path) setup_logging(os.path.join(save_path, 'log.txt'), resume=args.resume is not '', dummy=args.distributed and args.local_rank > 0) results_path = os.path.join(save_path, 'results') results = ResultsLog(results_path, title='Training Results - %s' % args.save) logging.info("saving to %s", save_path) logging.debug("run arguments: %s", args) logging.info("creating model %s", args.model) if 'cuda' in args.device and torch.cuda.is_available(): torch.cuda.manual_seed_all(args.seed) torch.cuda.set_device(args.device_ids[0]) cudnn.benchmark = True else: args.device_ids = None # create model model = models.__dict__[args.model] dataset_type = 'imagenet' if args.dataset == 'imagenet_calib' else args.dataset model_config = {'dataset': dataset_type} if args.model_config is not '': if isinstance(args.model_config, dict): for k, v in args.model_config.items(): if k not in model_config.keys(): model_config[k] = v else: args_dict = literal_eval(args.model_config) for k, v in args_dict.items(): model_config[k] = v if (args.absorb_bn or args.load_from_vision or args.pretrained) and not args.batch_norn_tuning: if args.load_from_vision: import torchvision exec_lfv_str = 'torchvision.models.' + args.load_from_vision + '(pretrained=True)' model = eval(exec_lfv_str) if 'pytcv' in args.model: from pytorchcv.model_provider import get_model as ptcv_get_model exec_lfv_str = 'ptcv_get_model("' + args.load_from_vision + '", pretrained=True)' model_pytcv = eval(exec_lfv_str) model = convert_pytcv_model(model, model_pytcv) else: if not os.path.isfile(args.absorb_bn): parser.error('invalid checkpoint: {}'.format(args.evaluate)) model = model(**model_config) checkpoint = torch.load(args.absorb_bn, map_location=lambda storage, loc: storage) checkpoint = checkpoint[ 'state_dict'] if 'state_dict' in checkpoint.keys( ) else checkpoint model.load_state_dict(checkpoint, strict=False) if 'batch_norm' in model_config and not model_config['batch_norm']: logging.info('Creating absorb_bn state dict') search_absorbe_bn(model) filename_ab = args.absorb_bn + '.absorb_bn' if args.absorb_bn else save_path + '/' + args.model + '.absorb_bn' torch.save(model.state_dict(), filename_ab) else: filename_bn = save_path + '/' + args.model + '.with_bn' torch.save(model.state_dict(), filename_bn) if (args.load_from_vision or args.absorb_bn) and not args.evaluate_init_configuration: return if 'inception' in args.model: model = model(init_weights=False, **model_config) else: model = model(**model_config) logging.info("created model with configuration: %s", model_config) num_parameters = sum([l.nelement() for l in model.parameters()]) logging.info("number of parameters: %d", num_parameters) # optionally resume from a checkpoint if args.evaluate: if not os.path.isfile(args.evaluate): parser.error('invalid checkpoint: {}'.format(args.evaluate)) checkpoint = torch.load(args.evaluate, map_location="cpu") # Overrride configuration with checkpoint info args.model = checkpoint.get('model', args.model) args.model_config = checkpoint.get('config', args.model_config) if not model_config['batch_norm']: search_absorbe_fake_bn(model) # load checkpoint if 'state_dict' in checkpoint.keys(): model.load_state_dict(checkpoint['state_dict']) logging.info("loaded checkpoint '%s'", args.evaluate) else: model.load_state_dict(checkpoint, strict=False) logging.info("loaded checkpoint '%s'", args.evaluate) if args.resume: checkpoint_file = args.resume if os.path.isdir(checkpoint_file): results.load(os.path.join(checkpoint_file, 'results.csv')) checkpoint_file = os.path.join(checkpoint_file, 'model_best.pth.tar') if os.path.isfile(checkpoint_file): logging.info("loading checkpoint '%s'", args.resume) checkpoint = torch.load(checkpoint_file) if args.start_epoch < 0: # not explicitly set args.start_epoch = checkpoint[ 'epoch'] - 1 if 'epoch' in checkpoint.keys() else 0 best_prec1 = checkpoint[ 'best_prec1'] if 'best_prec1' in checkpoint.keys() else -1 sd = checkpoint['state_dict'] if 'state_dict' in checkpoint.keys( ) else checkpoint model.load_state_dict(sd, strict=False) logging.info("loaded checkpoint '%s' (epoch %s)", checkpoint_file, args.start_epoch) else: logging.error("no checkpoint found at '%s'", args.resume) # define loss function (criterion) and optimizer loss_params = {} if args.label_smoothing > 0: loss_params['smooth_eps'] = args.label_smoothing criterion = getattr(model, 'criterion', CrossEntropyLoss)(**loss_params) if args.kld_loss: criterion = nn.KLDivLoss(reduction='mean') criterion.to(args.device, dtype) model.to(args.device, dtype) # Batch-norm should always be done in float if 'half' in args.dtype: FilterModules(model, module=is_bn).to(dtype=torch.float) # optimizer configuration optim_regime = getattr(model, 'regime', [{ 'epoch': 0, 'optimizer': args.optimizer, 'lr': args.lr, 'momentum': args.momentum, 'weight_decay': args.weight_decay }]) if args.fine_tune or args.prune: if not args.resume: args.start_epoch = 0 if args.update_only_th: #optim_regime = [ # {'epoch': 0, 'optimizer': 'Adam', 'lr': 1e-4}] optim_regime = [{ 'epoch': 0, 'optimizer': 'SGD', 'lr': 1e-1 }, { 'epoch': 10, 'lr': 1e-2 }, { 'epoch': 15, 'lr': 1e-3 }] else: optim_regime = [{ 'epoch': 0, 'optimizer': 'SGD', 'lr': 1e-4, 'momentum': 0.9 }, { 'epoch': 2, 'lr': 1e-5, 'momentum': 0.9 }, { 'epoch': 10, 'lr': 1e-6, 'momentum': 0.9 }] optimizer = optim_regime if isinstance(optim_regime, OptimRegime) \ else OptimRegime(model, optim_regime, use_float_copy='half' in args.dtype) # Training Data loading code train_data = DataRegime(getattr(model, 'data_regime', None), defaults={ 'datasets_path': args.datasets_dir, 'name': args.dataset, 'split': 'train', 'augment': False, 'input_size': args.input_size, 'batch_size': args.batch_size, 'shuffle': not args.seq_adaquant, 'num_workers': args.workers, 'pin_memory': True, 'drop_last': True, 'distributed': args.distributed, 'duplicates': args.duplicates, 'autoaugment': args.autoaugment, 'cutout': { 'holes': 1, 'length': 16 } if args.cutout else None, 'inception_prep': 'inception' in args.model }) if args.names_sp_layers is None and args.layers_precision_dict is None: args.names_sp_layers = [ key[:-7] for key in model.state_dict().keys() if 'weight' in key and 'running' not in key and ( 'conv' in key or 'downsample.0' in key or 'fc' in key) ] if args.keep_first_last: args.names_sp_layers = [ name for name in args.names_sp_layers if name != 'conv1' and name != 'fc' and name != 'Conv2d_1a_3x3.conv' ] args.names_sp_layers = [ k for k in args.names_sp_layers if 'downsample' not in k ] if args.ignore_downsample else args.names_sp_layers if args.num_sp_layers == 0 and not args.keep_first_last: args.names_sp_layers = [] if args.layers_precision_dict is not None: print(args.layers_precision_dict) prunner = None trainer = Trainer(model, prunner, criterion, optimizer, device_ids=args.device_ids, device=args.device, dtype=dtype, distributed=args.distributed, local_rank=args.local_rank, mixup=args.mixup, loss_scale=args.loss_scale, grad_clip=args.grad_clip, print_freq=args.print_freq, adapt_grad_norm=args.adapt_grad_norm, epoch=args.start_epoch, update_only_th=args.update_only_th, optimize_rounding=args.optimize_rounding) # Evaluation Data loading code args.eval_batch_size = args.eval_batch_size if args.eval_batch_size > 0 else args.batch_size dataset_type = 'imagenet' if args.dataset == 'imagenet_calib' else args.dataset val_data = DataRegime(getattr(model, 'data_eval_regime', None), defaults={ 'datasets_path': args.datasets_dir, 'name': dataset_type, 'split': 'val', 'augment': False, 'input_size': args.input_size, 'batch_size': args.eval_batch_size, 'shuffle': True, 'num_workers': args.workers, 'pin_memory': True, 'drop_last': False }) if args.evaluate or args.resume: from utils.layer_sensativity import search_replace_layer, extract_save_quant_state_dict, search_replace_layer_from_dict if args.layers_precision_dict is not None: model = search_replace_layer_from_dict( model, ast.literal_eval(args.layers_precision_dict)) else: model = search_replace_layer(model, args.names_sp_layers, num_bits_activation=args.nbits_act, num_bits_weight=args.nbits_weight) cached_input_output = {} quant_keys = [ '.weight', '.bias', '.equ_scale', '.quantize_input.running_zero_point', '.quantize_input.running_range', '.quantize_weight.running_zero_point', '.quantize_weight.running_range', '.quantize_input1.running_zero_point', '.quantize_input1.running_range' '.quantize_input2.running_zero_point', '.quantize_input2.running_range' ] if args.adaquant: def Qhook(name, module, input, output): if module not in cached_qinput: cached_qinput[module] = [] # Meanwhile store data in the RAM. cached_qinput[module].append(input[0].detach().cpu()) # print(name) def hook(name, module, input, output): if module not in cached_input_output: cached_input_output[module] = [] # Meanwhile store data in the RAM. cached_input_output[module].append( (input[0].detach().cpu(), output.detach().cpu())) # print(name) from models.modules.quantize import QConv2d, QLinear handlers = [] count = 0 for name, m in model.named_modules(): if isinstance(m, QConv2d) or isinstance(m, QLinear): #if isinstance(m, QConv2d) or isinstance(m, QLinear): # if isinstance(m, QConv2d): m.quantize = False if count < 1000: # if (isinstance(m, QConv2d) and m.groups == 1) or isinstance(m, QLinear): handlers.append( m.register_forward_hook(partial(hook, name))) count += 1 # Store input/output for all quantizable layers trainer.validate(train_data.get_loader()) print("Input/outputs cached") for handler in handlers: handler.remove() for m in model.modules(): if isinstance(m, QConv2d) or isinstance(m, QLinear): m.quantize = True mse_df = pd.DataFrame( index=np.arange(len(cached_input_output)), columns=['name', 'bit', 'shape', 'mse_before', 'mse_after']) print_freq = 100 for i, layer in enumerate(cached_input_output): if i > 0 and args.seq_adaquant: count = 0 cached_qinput = {} for name, m in model.named_modules(): if layer.name == name: if count < 1000: handler = m.register_forward_hook( partial(Qhook, name)) count += 1 # Store input/output for all quantizable layers trainer.validate(train_data.get_loader()) print("cashed quant Input%s" % layer.name) cached_input_output[layer][0] = ( cached_qinput[layer][0], cached_input_output[layer][0][1]) handler.remove() print("\nOptimize {}:{} for {} bit of shape {}".format( i, layer.name, layer.num_bits, layer.weight.shape)) mse_before, mse_after, snr_before, snr_after, kurt_in, kurt_w = \ optimize_layer(layer, cached_input_output[layer], args.optimize_weights, batch_size=args.batch_size, model_name=args.model) print("\nMSE before optimization: {}".format(mse_before)) print("MSE after optimization: {}".format(mse_after)) mse_df.loc[i, 'name'] = layer.name mse_df.loc[i, 'bit'] = layer.num_bits mse_df.loc[i, 'shape'] = str(layer.weight.shape) mse_df.loc[i, 'mse_before'] = mse_before mse_df.loc[i, 'mse_after'] = mse_after mse_df.loc[i, 'snr_before'] = snr_before mse_df.loc[i, 'snr_after'] = snr_after mse_df.loc[i, 'kurt_in'] = kurt_in mse_df.loc[i, 'kurt_w'] = kurt_w mse_csv = args.evaluate + '.mse.csv' mse_df.to_csv(mse_csv) filename = args.evaluate + '.adaquant' torch.save(model.state_dict(), filename) train_data = None cached_input_output = None val_results = trainer.validate(val_data.get_loader()) logging.info(val_results) if args.res_log is not None: if not os.path.exists(args.res_log): df = pd.DataFrame() else: df = pd.read_csv(args.res_log, index_col=0) ckp = ntpath.basename(args.evaluate) if args.cmp is not None: ckp += '_{}'.format(args.cmp) adaquant_type = 'adaquant_seq' if args.seq_adaquant else 'adaquant_parallel' df.loc[ckp, 'acc_' + adaquant_type] = val_results['prec1'] df.to_csv(args.res_log) # print(df) elif args.per_layer: # Store input/output for all quantizable layers calib_all_8_results = trainer.validate(train_data.get_loader()) print('########## All 8bit results ###########', calib_all_8_results) int8_opt_model_state_dict = torch.load(args.int8_opt_model_path) int4_opt_model_state_dict = torch.load(args.int4_opt_model_path) per_layer_results = {} args.names_sp_layers = [ key[:-7] for key in model.state_dict().keys() if 'weight' in key and 'running' not in key and 'quantize' not in key and ('conv' in key or 'downsample.0' in key or 'fc' in key) ] for layer_idx, layer in enumerate(args.names_sp_layers): model.load_state_dict(int8_opt_model_state_dict, strict=False) model = search_replace_layer(model, [layer], num_bits_activation=args.nbits_act, num_bits_weight=args.nbits_weight) layer_keys = [ key for key in int8_opt_model_state_dict for qpkey in quant_keys if layer + qpkey == key ] for key in layer_keys: model.state_dict()[key].copy_(int4_opt_model_state_dict[key]) calib_results = trainer.validate(train_data.get_loader()) model = search_replace_layer(model, [layer], num_bits_activation=8, num_bits_weight=8) print('finished %d out of %d' % (layer_idx, len(args.names_sp_layers))) logging.info(layer) logging.info(calib_results) per_layer_results[layer] = { 'base precision': 8, 'replaced precision': args.nbits_act, 'replaced layer': layer, 'accuracy': calib_results['prec1'], 'loss': calib_results['loss'], 'Parameters Size [Elements]': model.state_dict()[layer + '.weight'].numel(), 'MACs': '-' } torch.save( per_layer_results, args.evaluate + '.per_layer_accuracy.A' + str(args.nbits_act) + '.W' + str(args.nbits_weight)) all_8_dict = { 'base precision': 8, 'replaced precision': args.nbits_act, 'replaced layer': '-', 'accuracy': calib_all_8_results['prec1'], 'loss': calib_all_8_results['loss'], 'Parameters Size [Elements]': '-', 'MACs': '-' } columns = [key for key in all_8_dict] with open( args.evaluate + '.per_layer_accuracy.A' + str(args.nbits_act) + '.W' + str(args.nbits_weight) + '.csv', "w") as f: f.write(",".join(columns) + "\n") col = [str(all_8_dict[c]) for c in all_8_dict.keys()] f.write(",".join(col) + "\n") for layer in per_layer_results: r = per_layer_results[layer] col = [str(r[c]) for c in r.keys()] f.write(",".join(col) + "\n") elif args.mixed_builder: if isinstance(args.names_sp_layers, list): print('loading int8 model" ', args.int8_opt_model_path) int8_opt_model_state_dict = torch.load(args.int8_opt_model_path) print('loading int4 model" ', args.int4_opt_model_path) int4_opt_model_state_dict = torch.load(args.int4_opt_model_path) model.load_state_dict(int8_opt_model_state_dict, strict=False) model = search_replace_layer(model, args.names_sp_layers, num_bits_activation=args.nbits_act, num_bits_weight=args.nbits_weight) for layer_idx, layer in enumerate(args.names_sp_layers): layer_keys = [ key for key in int8_opt_model_state_dict for qpkey in quant_keys if layer + qpkey == key ] for key in layer_keys: model.state_dict()[key].copy_( int4_opt_model_state_dict[key]) print('switched layer %s to 4 bit' % (layer)) elif isinstance(args.names_sp_layers, dict): quant_models = {} base_precision = args.precisions[0] for m, prec in zip(args.opt_model_paths, args.precisions): print('For precision={}, loading {}'.format(prec, m)) quant_models[prec] = torch.load(m) model.load_state_dict(quant_models[base_precision], strict=False) for layer_name, nbits_list in args.names_sp_layers.items(): model = search_replace_layer(model, [layer_name], num_bits_activation=nbits_list[0], num_bits_weight=nbits_list[0]) layer_keys = [ key for key in quant_models[base_precision] for qpkey in quant_keys if layer_name + qpkey == key ] for key in layer_keys: model.state_dict()[key].copy_( quant_models[nbits_list[0]][key]) print('switched layer {} to {} bit'.format( layer_name, nbits_list[0])) if os.environ.get('DEBUG') == 'True': from utils.layer_sensativity import check_quantized_model fp_names = check_quantized_model(trainer.model) if len(fp_names) > 0: logging.info('Found FP32 layers in the model:') logging.info(fp_names) if args.eval_on_train: mixedIP_results = trainer.validate(train_data.get_loader()) else: mixedIP_results = trainer.validate(val_data.get_loader()) torch.save( { 'state_dict': model.state_dict(), 'config-ip': args.names_sp_layers }, args.evaluate + '.mixed-ip-results.' + args.suffix) logging.info(mixedIP_results) acc = mixedIP_results['prec1'] loss = mixedIP_results['loss'] elif args.batch_norn_tuning: from utils.layer_sensativity import search_replace_layer, extract_save_quant_state_dict, search_replace_layer_from_dict from models.modules.quantize import QConv2d if args.layers_precision_dict is not None: model = search_replace_layer_from_dict( model, literal_eval(args.layers_precision_dict)) else: model = search_replace_layer(model, args.names_sp_layers, num_bits_activation=args.nbits_act, num_bits_weight=args.nbits_weight) exec_lfv_str = 'torchvision.models.' + args.load_from_vision + '(pretrained=True)' model_orig = eval(exec_lfv_str) model_orig.to(args.device, dtype) search_copy_bn_params(model_orig) layers_orig = dict([(n, m) for n, m in model_orig.named_modules() if isinstance(m, nn.Conv2d)]) layers_q = dict([(n, m) for n, m in model.named_modules() if isinstance(m, QConv2d)]) for l in layers_orig: conv_orig = layers_orig[l] conv_q = layers_q[l] conv_q.register_parameter('gamma', nn.Parameter(conv_orig.gamma.clone())) conv_q.register_parameter('beta', nn.Parameter(conv_orig.beta.clone())) del model_orig search_add_bn(model) print("Run BN tuning") for tt in range(args.tuning_iter): print(tt) trainer.cal_bn_stats(train_data.get_loader()) search_absorbe_tuning_bn(model) filename = args.evaluate + '.bn_tuning' print("Save model to: {}".format(filename)) torch.save(model.state_dict(), filename) val_results = trainer.validate(val_data.get_loader()) logging.info(val_results) if args.res_log is not None: if not os.path.exists(args.res_log): df = pd.DataFrame() else: df = pd.read_csv(args.res_log, index_col=0) ckp = ntpath.basename(args.evaluate) df.loc[ckp, 'acc_bn_tuning'] = val_results['prec1'] df.loc[ckp, 'loss_bn_tuning'] = val_results['loss'] df.to_csv(args.res_log) # print(df) elif args.bias_tuning: for epoch in range(args.epochs): trainer.epoch = epoch train_data.set_epoch(epoch) val_data.set_epoch(epoch) logging.info('\nStarting Epoch: {0}\n'.format(epoch + 1)) # train for one epoch repeat_train = 20 if args.update_only_th else 1 for tt in range(repeat_train): print(tt) train_results = trainer.train( train_data.get_loader(), duplicates=train_data.get('duplicates'), chunk_batch=args.chunk_batch) logging.info(train_results) val_results = trainer.validate(val_data.get_loader()) logging.info(val_results) if args.res_log is not None: if not os.path.exists(args.res_log): df = pd.DataFrame() else: df = pd.read_csv(args.res_log, index_col=0) ckp = ntpath.basename(args.evaluate) if 'bn_tuning' in ckp: ckp = ckp.replace('.bn_tuning', '') df.loc[ckp, 'acc_bias_tuning'] = val_results['prec1'] df.to_csv(args.res_log) # import pdb; pdb.set_trace() else: #print('Please Choose one of the following ....') if model_config['measure']: results = trainer.validate(train_data.get_loader(), rec=args.rec) # results = trainer.validate(val_data.get_loader()) # print(results) else: if args.evaluate_init_configuration: results = trainer.validate(val_data.get_loader()) if args.res_log is not None: if not os.path.exists(args.res_log): df = pd.DataFrame() else: df = pd.read_csv(args.res_log, index_col=0) ckp = ntpath.basename(args.evaluate) if args.cmp is not None: ckp += '_{}'.format(args.cmp) df.loc[ckp, 'acc_base'] = results['prec1'] df.to_csv(args.res_log) if args.extract_bias_mean: file_name = 'bias_mean_measure' if model_config[ 'measure'] else 'bias_mean_quant' torch.save(trainer.bias_mean, file_name) if model_config['measure']: filename = args.evaluate + '.measure' if 'perC' in args.model_config: filename += '_perC' torch.save(model.state_dict(), filename) logging.info(results) else: if args.evaluate_init_configuration: logging.info(results) return acc, loss
def main(): global args, best_prec1, dtype best_prec1 = 0 args = parser.parse_args() dtype = torch_dtypes.get(args.dtype) torch.manual_seed(args.seed) time_stamp = datetime.now().strftime('%Y-%m-%d_%H-%M-%S') if args.evaluate: args.results_dir = '/tmp' if args.save is '': args.save = time_stamp if args.log_dir is None: import time writer = SummaryWriter(log_dir="runs/time-%d" % int(time.time())) else: writer = SummaryWriter(log_dir="runs/%s" % (args.log_dir)) save_path = os.path.join(args.results_dir, args.save) if not os.path.exists(save_path): os.makedirs(save_path) setup_logging(os.path.join(save_path, 'log.txt'), resume=args.resume is not '') results_path = os.path.join(save_path, 'results') results = ResultsLog(results_path, title='Training Results - %s' % args.save) logging.info("saving to %s", save_path) logging.debug("run arguments: %s", args) if 'cuda' in args.device and torch.cuda.is_available(): torch.cuda.manual_seed_all(args.seed) torch.cuda.set_device(args.device_ids[0]) cudnn.benchmark = True else: args.device_ids = None # create model logging.info("creating model %s", args.model) model = models.__dict__[args.model] model_config = {'input_size': args.input_size, 'dataset': args.dataset} if args.model_config is not '': model_config = dict(model_config, **literal_eval(args.model_config)) model = model(**model_config) logging.info("created model with configuration: %s", model_config) # optionally resume from a checkpoint if args.evaluate: if not os.path.isfile(args.evaluate): parser.error('invalid checkpoint: {}'.format(args.evaluate)) checkpoint = torch.load(args.evaluate) model.load_state_dict(checkpoint['state_dict']) logging.info("loaded checkpoint '%s' (epoch %s)", args.evaluate, checkpoint['epoch']) elif args.resume: checkpoint_file = args.resume if os.path.isdir(checkpoint_file): results.load(os.path.join(checkpoint_file, 'results.csv')) checkpoint_file = os.path.join(checkpoint_file, 'model_best.pth.tar') if os.path.isfile(checkpoint_file): logging.info("loading checkpoint '%s'", args.resume) checkpoint = torch.load(checkpoint_file) args.start_epoch = checkpoint['epoch'] - 1 best_prec1 = checkpoint['best_prec1'] model.load_state_dict(checkpoint['state_dict']) logging.info("loaded checkpoint '%s' (epoch %s)", checkpoint_file, checkpoint['epoch']) else: logging.error("no checkpoint found at '%s'", args.resume) num_parameters = sum([l.nelement() for l in model.parameters()]) logging.info("number of parameters: %d", num_parameters) # Data loading code default_transform = { 'train': get_transform(args.dataset, input_size=args.input_size, augment=True), 'eval': get_transform(args.dataset, input_size=args.input_size, augment=False) } transform = getattr(model, 'input_transform', default_transform) regime = getattr(model, 'regime', [ { 'epoch': 0, 'optimizer': args.optimizer, 'lr': args.lr, 'momentum': args.momentum, 'weight_decay': args.weight_decay }, { 'epoch': 100, 'optimizer': args.optimizer, 'lr': args.lr * 0.1, 'momentum': args.momentum, 'weight_decay': args.weight_decay }, { 'epoch': 200, 'optimizer': args.optimizer, 'lr': args.lr * 0.01, 'momentum': args.momentum, 'weight_decay': args.weight_decay }, ]) # define loss function (criterion) and optimizer criterion = getattr(model, 'criterion', nn.CrossEntropyLoss)() criterion.to(args.device, dtype) model.to(args.device, dtype) val_data = get_dataset(args.dataset, 'val', transform['eval']) val_loader = torch.utils.data.DataLoader(val_data, batch_size=args.batch_size, shuffle=False, num_workers=args.workers, pin_memory=True) if args.evaluate: validate(val_loader, model, criterion, 0) return train_data = get_dataset(args.dataset, 'train', transform['train']) train_loader = torch.utils.data.DataLoader(train_data, batch_size=args.batch_size, shuffle=True, num_workers=args.workers, pin_memory=True) optimizer = OptimRegime(model.parameters(), regime) logging.info('training regime: %s', regime) for epoch in range(args.start_epoch, args.epochs): # train for one epoch train_loss, train_prec1, train_prec5 = train(train_loader, model, criterion, epoch, optimizer, writer) # evaluate on validation set val_loss, val_prec1, val_prec5 = validate(val_loader, model, criterion, epoch, writer) # remember best prec@1 and save checkpoint is_best = val_prec1 > best_prec1 best_prec1 = max(val_prec1, best_prec1) save_checkpoint( { 'epoch': epoch + 1, 'model': args.model, 'config': args.model_config, 'state_dict': model.state_dict(), 'best_prec1': best_prec1, 'regime': regime }, is_best, path=save_path) logging.info('\n Epoch: {0}\t' 'Training Loss {train_loss:.4f} \t' 'Training Prec@1 {train_prec1:.3f} \t' 'Training Prec@5 {train_prec5:.3f} \t' 'Validation Loss {val_loss:.4f} \t' 'Validation Prec@1 {val_prec1:.3f} \t' 'Validation Prec@5 {val_prec5:.3f} \n'.format( epoch + 1, train_loss=train_loss, val_loss=val_loss, train_prec1=train_prec1, val_prec1=val_prec1, train_prec5=train_prec5, val_prec5=val_prec5)) results.add(epoch=epoch + 1, train_loss=train_loss, val_loss=val_loss, train_error1=100 - train_prec1, val_error1=100 - val_prec1, train_error5=100 - train_prec5, val_error5=100 - val_prec5) results.plot(x='epoch', y=['train_loss', 'val_loss'], legend=['training', 'validation'], title='Loss', ylabel='loss') results.plot(x='epoch', y=['train_error1', 'val_error1'], legend=['training', 'validation'], title='Error@1', ylabel='error %') results.plot(x='epoch', y=['train_error5', 'val_error5'], legend=['training', 'validation'], title='Error@5', ylabel='error %') results.save()
def main(): tracking_parser = tracker.create_tracking_parser( suppress_args=['--score-init-min', '--score-continue-min']) # Use first line of file docstring as description if it exists. parser = argparse.ArgumentParser( description=__doc__.split('\n')[0] if __doc__ else '', parents=[tracking_parser], formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('--images-dir', type=Path, required=True) parser.add_argument( '--init-detections-dir', type=Path, help=('Contains pickle files of detections for each frame.These ' 'detections are used to initialize and continue tracks.'), required=True) parser.add_argument( '--continue-detections-dir', type=Path, help=('Contains pickle files of detections for each frame.These ' 'detections are used only to continue tracks. If not specified, ' 'the continue detections are assumed to be stored in the ' 'pickles in init_detections_dir, under the "appearance_stream" ' 'key for each frame.')) parser.add_argument( '--remove-continue-overlap', type=float, default=0.1, help=('Remove detections from --continue-detections-dir if they ' 'overlap more than this threshold with a detection from ' '--init-detections-dir.')) parser.add_argument('--output-dir', type=Path, required=True) parser.add_argument( '--score-init-min', type=float, default=0.9, help=('Detection confidence threshold for starting a new track from ' '--init-detections-dir detections.')) parser.add_argument( '--score-continue-min', type=float, default=0.7, help=('Detection confidence threshold for continuing a new track from ' '--init-detections-dir or --continue-detections-dir ' 'detections.')) parser.add_argument('--fps', type=int, default=30) parser.add_argument('--extensions', nargs='*', default=IMG_EXTENSIONS) parser.add_argument( '--vis-dataset', default='objectness', choices=['coco', 'objectness'], help='Dataset to use for mapping label indices to names.') parser.add_argument('--save-images', type=parse_bool, default=False) parser.add_argument('--save-merged-detections', type=parse_bool, default=False) parser.add_argument('--save-numpy', type=parse_bool, default=False) parser.add_argument('--save-numpy-every-kth-frame', help='Save only every kth frame in numpy output.', type=int, default=1) parser.add_argument('--save-video', type=parse_bool, default=True) parser.add_argument( '--filename-format', choices=[ 'frame', 'frameN', 'sequence_frame', 'sequence-frame', 'fbms' ], default='frame', help=('Specifies how to get frame number from the filename. ' '"frame": the filename is the frame number, ' '"frameN": format <frame><number>, ' '"sequence_frame": frame number is separated by an underscore, ' '"sequence-frame": frame number is separated by a dash, ' '"fbms": assume fbms style frame numbers')) parser.add_argument('--recursive', action='store_true', help='Look recursively in --images-dir for images.') parser.add_argument('--quiet', action='store_true') tracking_params, remaining_argv = tracking_parser.parse_known_args() args = parser.parse_args(remaining_argv) tracking_params = vars(tracking_params) tracking_params['score_init_min'] = args.score_init_min tracking_params['score_continue_min'] = args.score_continue_min args.output_dir.mkdir(exist_ok=True, parents=True) output_log_file = log_utils.add_time_to_path(args.output_dir / 'tracker.log') log_utils.setup_logging(output_log_file) if args.quiet: logging.root.setLevel(logging.WARN) logging.info('Args:\n%s', pprint.pformat(vars(args))) logging.info('Tracking params:\n%s', pprint.pformat(tracking_params)) subprocess.call([ './git-state/save_git_state.sh', output_log_file.with_suffix('.git-state') ]) if args.filename_format == 'fbms': get_framenumber = fbms_utils.get_framenumber elif args.filename_format == 'frameN': def get_framenumber(x): return int(x.split('frame')[1]) elif args.filename_format == 'sequence-frame': def get_framenumber(x): return int(x.split('-')[-1]) elif args.filename_format == 'sequence_frame': def get_framenumber(x): return int(x.split('_')[-1]) elif args.filename_format == 'frame': get_framenumber = int else: raise ValueError('Unknown --filename-format: %s' % args.filename_format) args.extensions = [x if x[0] == '.' else '.' + x for x in args.extensions] track_fn = functools.partial( two_detector_track, get_framenumber=get_framenumber, tracking_params=tracking_params, remove_continue_overlap=args.remove_continue_overlap, extensions=args.extensions, output_numpy_every_kth_frame=args.save_numpy_every_kth_frame, fps=args.fps) if not args.recursive: output_merged = None if args.save_merged_detections: output_merged = args.output_dir / 'merged' assert not output_merged.exists() output_merged.mkdir() output_numpy = None if args.save_numpy: output_numpy = args.output_dir / 'results.npz' output_images_dir = None if args.save_images: output_images_dir = args.output_dir / 'images' output_images_dir.mkdir(exist_ok=True, parents=True) output_video = None if args.save_video: output_video = args.output_dir / 'video.mp4' track_fn(images_dir=args.images_dir, init_detections_dir=args.init_detections_dir, continue_detections_dir=args.continue_detections_dir, output_video=output_video, output_merged_dir=output_merged, output_numpy=output_numpy, progress=True) else: images = glob_ext(args.images_dir, args.extensions, recursive=True) image_subdirs = sorted( set(x.parent.relative_to(args.images_dir) for x in images)) for subdir in tqdm(image_subdirs): output_merged = None output_numpy = None output_images_dir = None output_video = None if args.save_merged_detections: output_merged = args.output_dir / subdir / 'merged-detections' output_merged.mkdir(exist_ok=True, parents=True) if args.save_numpy: output_numpy = args.output_dir / subdir.with_suffix('.npz') if args.save_images: output_images_dir = args.output_dir / subdir / 'images' output_images_dir.mkdir(exist_ok=True, parents=True) if args.save_video: output_video = args.output_dir / subdir.with_suffix('.mp4') if all(x is None or x.exists() for x in (output_merged, output_numpy, output_images_dir, output_video)): logging.info('%s already processed, skipping', subdir) continue init_dir = args.init_detections_dir / subdir if not init_dir.exists(): logging.warn( 'Skipping sequence %s: detections not found at %s', subdir, init_dir) continue if args.continue_detections_dir: continue_dir = args.continue_detections_dir / subdir if not continue_dir.exists(): logging.warn( 'Skipping sequence %s: detections not found at %s', subdir, continue_dir) continue if args.continue_detections_dir: continue_dir = args.continue_detections_dir / subdir else: continue_dir = None track_fn(images_dir=args.images_dir / subdir, init_detections_dir=args.init_detections_dir / subdir, continue_detections_dir=continue_dir, output_video=output_video, output_images_dir=output_images_dir, output_merged_dir=output_merged, output_numpy=output_numpy, progress=False)
def main_worker(args): global best_prec1, dtype best_prec1 = 0 dtype = torch_dtypes.get(args.dtype) torch.manual_seed(args.seed) time_stamp = datetime.now().strftime('%Y-%m-%d_%H-%M-%S') if args.evaluate: args.results_dir = '/tmp' if args.save is '': args.save = time_stamp save_path = path.join(args.results_dir, args.save) args.distributed = args.local_rank >= 0 or args.world_size > 1 if args.distributed: dist.init_process_group(backend=args.dist_backend, init_method=args.dist_init, world_size=args.world_size, rank=args.local_rank) args.local_rank = dist.get_rank() args.world_size = dist.get_world_size() if args.dist_backend == 'mpi': # If using MPI, select all visible devices args.device_ids = list(range(torch.cuda.device_count())) else: args.device_ids = [args.local_rank] if not (args.distributed and args.local_rank > 0): if not path.exists(save_path): makedirs(save_path) export_args_namespace(args, path.join(save_path, 'config.json')) setup_logging(path.join(save_path, 'log.txt'), resume=args.resume is not '', dummy=args.distributed and args.local_rank > 0) results_path = path.join(save_path, 'results') results = ResultsLog(results_path, title='Training Results - %s' % args.save) writer = SummaryWriter(log_dir=save_path, purge_step=1) logging.info("saving to %s", save_path) logging.debug("run arguments: %s", args) logging.info("creating model %s", args.model) if 'cuda' in args.device and torch.cuda.is_available(): torch.cuda.manual_seed_all(args.seed) torch.cuda.set_device(args.device_ids[0]) cudnn.benchmark = True else: args.device_ids = None # create model model = models.__dict__[args.model] model_config = {'dataset': args.dataset} if args.model_config is not '': model_config = dict(model_config, **literal_eval(args.model_config)) model = model(**model_config) logging.info("created model with configuration: %s", model_config) num_parameters = sum([l.nelement() for l in model.parameters()]) logging.info("number of parameters: %d", num_parameters) # optionally resume from a checkpoint if args.evaluate: if not path.isfile(args.evaluate): parser.error('invalid checkpoint: {}'.format(args.evaluate)) checkpoint = torch.load(args.evaluate, map_location="cpu") # Overrride configuration with checkpoint info args.model = checkpoint.get('model', args.model) args.model_config = checkpoint.get('config', args.model_config) # load checkpoint model.load_state_dict(checkpoint['state_dict']) logging.info("loaded checkpoint '%s' (epoch %s)", args.evaluate, checkpoint['epoch']) if args.resume: checkpoint_file = args.resume if path.isdir(checkpoint_file): results.load(path.join(checkpoint_file, 'results.csv')) checkpoint_file = path.join(checkpoint_file, 'model_best.pth.tar') if path.isfile(checkpoint_file): logging.info("loading checkpoint '%s'", args.resume) checkpoint = torch.load(checkpoint_file, map_location="cpu") if args.start_epoch < 0: # not explicitly set args.start_epoch = checkpoint['epoch'] best_prec1 = checkpoint['best_prec1'] model.load_state_dict(checkpoint['state_dict']) optim_state_dict = checkpoint.get('optim_state_dict', None) logging.info("loaded checkpoint '%s' (epoch %s)", checkpoint_file, checkpoint['epoch']) else: logging.error("no checkpoint found at '%s'", args.resume) else: optim_state_dict = None # define loss function (criterion) and optimizer loss_params = {} if args.label_smoothing > 0: loss_params['smooth_eps'] = args.label_smoothing criterion = getattr(model, 'criterion', CrossEntropyLoss)(**loss_params) criterion.to(args.device, dtype) model.to(args.device, dtype) # Batch-norm should always be done in float if 'half' in args.dtype: FilterModules(model, module=is_bn).to(dtype=torch.float) # optimizer configuration optim_regime = getattr(model, 'regime', [{ 'epoch': 0, 'optimizer': args.optimizer, 'lr': args.lr, 'momentum': args.momentum, 'weight_decay': args.weight_decay }]) optimizer = optim_regime if isinstance(optim_regime, OptimRegime) \ else OptimRegime(model, optim_regime, use_float_copy='half' in args.dtype) if optim_state_dict is not None: optimizer.load_state_dict(optim_state_dict) trainer = Trainer(model, criterion, optimizer, device_ids=args.device_ids, device=args.device, dtype=dtype, print_freq=args.print_freq, distributed=args.distributed, local_rank=args.local_rank, mixup=args.mixup, cutmix=args.cutmix, loss_scale=args.loss_scale, grad_clip=args.grad_clip, adapt_grad_norm=args.adapt_grad_norm) if args.tensorwatch: trainer.set_watcher(filename=path.abspath( path.join(save_path, 'tensorwatch.log')), port=args.tensorwatch_port) # Evaluation Data loading code args.eval_batch_size = args.eval_batch_size if args.eval_batch_size > 0 else args.batch_size val_data = DataRegime(getattr(model, 'data_eval_regime', None), defaults={ 'datasets_path': args.datasets_dir, 'name': args.dataset, 'split': 'val', 'augment': False, 'input_size': args.input_size, 'batch_size': args.eval_batch_size, 'shuffle': False, 'num_workers': args.workers, 'pin_memory': True, 'drop_last': False }) if args.evaluate: results = trainer.validate(val_data.get_loader()) logging.info(results) return # Training Data loading code train_data_defaults = { 'datasets_path': args.datasets_dir, 'name': args.dataset, 'split': 'train', 'augment': True, 'input_size': args.input_size, 'batch_size': args.batch_size, 'shuffle': True, 'num_workers': args.workers, 'pin_memory': True, 'drop_last': True, 'distributed': args.distributed, 'duplicates': args.duplicates, 'autoaugment': args.autoaugment, 'cutout': { 'holes': 1, 'length': 16 } if args.cutout else None } if hasattr(model, 'sampled_data_regime'): sampled_data_regime = model.sampled_data_regime probs, regime_configs = zip(*sampled_data_regime) regimes = [] for config in regime_configs: defaults = {**train_data_defaults} defaults.update(config) regimes.append(DataRegime(None, defaults=defaults)) train_data = SampledDataRegime(regimes, probs) else: train_data = DataRegime(getattr(model, 'data_regime', None), defaults=train_data_defaults) logging.info('optimization regime: %s', optim_regime) logging.info('data regime: %s', train_data) args.start_epoch = max(args.start_epoch, 0) trainer.training_steps = args.start_epoch * len(train_data) for epoch in range(args.start_epoch, args.epochs): trainer.epoch = epoch train_data.set_epoch(epoch) val_data.set_epoch(epoch) logging.info('\nStarting Epoch: {0}\n'.format(epoch + 1)) # train for one epoch train_results = trainer.train(train_data.get_loader(), chunk_batch=args.chunk_batch) # evaluate on validation set val_results = trainer.validate(val_data.get_loader()) if args.distributed and args.local_rank > 0: continue # remember best prec@1 and save checkpoint is_best = val_results['prec1'] > best_prec1 best_prec1 = max(val_results['prec1'], best_prec1) if args.drop_optim_state: optim_state_dict = None else: optim_state_dict = optimizer.state_dict() save_checkpoint( { 'epoch': epoch + 1, 'model': args.model, 'config': args.model_config, 'state_dict': model.state_dict(), 'optim_state_dict': optim_state_dict, 'best_prec1': best_prec1 }, is_best, path=save_path, save_all=args.save_all) logging.info('\nResults - Epoch: {0}\n' 'Training Loss {train[loss]:.4f} \t' 'Training Prec@1 {train[prec1]:.3f} \t' 'Training Prec@5 {train[prec5]:.3f} \t' 'Validation Loss {val[loss]:.4f} \t' 'Validation Prec@1 {val[prec1]:.3f} \t' 'Validation Prec@5 {val[prec5]:.3f} \t\n'.format( epoch + 1, train=train_results, val=val_results)) values = dict(epoch=epoch + 1, steps=trainer.training_steps) values.update({'training ' + k: v for k, v in train_results.items()}) values.update({'validation ' + k: v for k, v in val_results.items()}) results.add(**values) results.plot(x='epoch', y=['training loss', 'validation loss'], legend=['training', 'validation'], title='Loss', ylabel='loss') results.plot(x='epoch', y=['training error1', 'validation error1'], legend=['training', 'validation'], title='Error@1', ylabel='error %') results.plot(x='epoch', y=['training error5', 'validation error5'], legend=['training', 'validation'], title='Error@5', ylabel='error %') if 'grad' in train_results.keys(): results.plot(x='epoch', y=['training grad'], legend=['gradient L2 norm'], title='Gradient Norm', ylabel='value') results.save() writer.add_scalar('Loss/validation', val_results['loss'], epoch) writer.add_scalar('Loss/train', train_results['loss'], epoch) writer.add_scalar('Top1/validation', val_results['prec1'], epoch) writer.add_scalar('Top1/train', train_results['prec1'], epoch) writer.add_scalar('Top5/validation', val_results['prec5'], epoch) writer.add_scalar('Top5/train', train_results['prec5'], epoch) writer.close()
def main(): # Use first line of file docstring as description if it exists. parser = argparse.ArgumentParser( description=__doc__.split('\n')[0] if __doc__ else '', formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('--input-dir', required=True) parser.add_argument('--output-dir', required=True) parser.add_argument('--num-workers', default=8, type=int) parser.add_argument('--recursive', action='store_true') parser.add_argument( '--sequence-normalize', action='store_true', help=("Normalize all images together. If --recursive is True, " "normalize all images with the same parent directory together.")) parser.add_argument( '--diagonal-normalize', action='store_true', help='Normalize magnitude proportional to the image diagonal length.') parser.add_argument( '--min-normalize-perc', default=0, type=float, help=('Minimum magnitude normalization, as percent of image diagonal.' 'If <= 0, no minimum is used.')) parser.add_argument('--visualize-angle-only', action='store_true') args = parser.parse_args() input_root = Path(args.input_dir) output_root = Path(args.output_dir) assert input_root.exists() output_root.mkdir(exist_ok=True, parents=True) setup_logging(str(output_root / (Path(__file__).stem + '.log'))) logging.info('Args:\n%s', vars(args)) logging.info('Collecting paths') if args.recursive: flow_paths = list(tqdm(input_root.rglob('*.png'))) else: flow_paths = list(tqdm(input_root.glob('*.png'))) output_paths = { flow_path: output_root / flow_path.relative_to(input_root) for flow_path in flow_paths } for output_path in output_paths.values(): output_path.parent.mkdir(exist_ok=True, parents=True) assert not (args.diagonal_normalize and args.sequence_normalize) if args.min_normalize_perc > 0: assert not args.diagonal_normalize assert not args.sequence_normalize if not args.sequence_normalize: tasks = list(output_paths.items()) if args.diagonal_normalize: flow = load_flow_png(str(tasks[0][0])) diagonal = (flow.shape[0]**2 + flow.shape[1]**2)**0.5 magnitude_scale = diagonal / 12 else: magnitude_scale = None Parallel(n_jobs=args.num_workers)( delayed(visualize_flow)( flow_path, output_path, magnitude_scale, args.visualize_angle_only, clip_at_magnitude_scale=args.diagonal_normalize, min_normalize_percent=args.min_normalize_perc) for flow_path, output_path in tqdm(tasks)) else: # Collect flows for each parent directory flows_by_parent_dir = collections.defaultdict(list) for flow_path in flow_paths: flows_by_parent_dir[flow_path.parent].append(flow_path) with Parallel(n_jobs=args.num_workers) as parallel: for dir_flow_paths in tqdm(flows_by_parent_dir.values()): flows = parallel( delayed(load_flow_png)(str(path)) for path in dir_flow_paths) diagonal = (flows[0].shape[0]**2 + flows[0].shape[1]**2)**0.5 for flow in flows: flow[:, :, 1] = flow[:, :, 1].clip(max=diagonal) magnitude_scale = max(flow[:, :, 1].max() for flow in flows) tasks = [(flow_path, output_paths[flow_path]) for flow_path in dir_flow_paths] parallel( delayed(visualize_flow)(flow, output_path, magnitude_scale, args.visualize_angle_only) for flow, output_path in tasks)
def main(): with open(__file__, 'r') as f: _source = f.read() # Use first line of file docstring as description if it exists. parser = argparse.ArgumentParser( description=__doc__.split('\n')[0] if __doc__ else '', formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('--input-split-videos', required=True) parser.add_argument('--input-json', required=True) parser.add_argument('--output-json', required=True) args = parser.parse_args() output_json = Path(args.output_json) logging_path = args.output_json + '.log' setup_logging(logging_path) file_logger = logging.getLogger(logging_path) logging.info('Args: %s', pformat(vars(args))) with open(args.input_json, 'r') as f: data = json.load(f) if args.input_split_videos.endswith('.csv'): import csv with open(args.input_split_videos, 'r') as csvfile: reader = csv.DictReader(csvfile) valid_sequences = [row['video'] for row in reader] else: with open(args.input_split_videos, 'r') as f: valid_sequences = [x.strip() for x in f.readlines()] logging.info('Valid sequences: %s' % valid_sequences) images = [] image_ids = set() present_sequences = set() valid_sequences = set(valid_sequences) for image in data['images']: sequence = str(PurePath(image['file_name']).parent) present_sequences.add(sequence) if sequence in valid_sequences: images.append(image) image_ids.add(image['id']) new_annotations = [ x for x in data['annotations'] if x['image_id'] in image_ids ] new_images = images logging.info( 'Kept %s/%s sequences' % (len(present_sequences & valid_sequences), len(present_sequences))) logging.info('Kept %s/%s annotations, %s/%s images' % (len(new_annotations), len(data['annotations']), len(new_images), len(data['images']))) data['annotations'] = new_annotations data['images'] = new_images with open(args.output_json, 'w') as f: json.dump(data, f) file_logger.info('Source:') file_logger.info('=======') file_logger.info(_source) file_logger.info('=======')
def main(): # Use first line of file docstring as description if it exists. parser = argparse.ArgumentParser( description=__doc__.split('\n')[0] if __doc__ else '', formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument( '--detections-root', type=Path, help=('Contains subdirectory for each sequence, containing pickle ' 'files of detectron outputs for each frame.')) parser.add_argument('--output-dir', type=Path, required=True) parser.add_argument('--images-dir', type=Path, required=True) parser.add_argument('--threshold', type=float, default=0.7) parser.add_argument('--recursive', action='store_true') parser.add_argument('--extension', default='.png', help='Extension for images in --images-dir') parser.add_argument( '--duplicate-last-frame', action='store_true', help=('Whether to duplicate the last frame. This is useful if we ' 'only have predictions for n-1 frames (since flow is only ' 'computed on the first n-1 frames). NOTE: This only works if ' 'the pickle files are of the form "<frame_id>.pickle".')) args = parser.parse_args() assert args.detections_root.exists() args.output_dir.mkdir(exist_ok=True, parents=True) setup_logging( add_time_to_path(args.output_dir / (Path(__file__).name + '.log'))) logging.info('Args: %s\n', pprint.pformat(vars(args))) if args.recursive: all_pickles = args.detections_root.rglob('*.pickle') all_predictions = collections.defaultdict(list) for x in all_pickles: all_predictions[x.parent].append(x) else: all_predictions = { args.detections_root: list(args.detections_root.glob('*.pickle')) } if not all_predictions[args.detections_root]: raise ValueError("Found no .pickle files in --detections-root. " "Did you mean to specify --recursive?") all_predictions = { k: natsorted(v, alg=ns.PATH) for k, v in all_predictions.items() } # The DAVIS 2016 evaluation code really doesn't like any other files / # directories in the input directory, so we put the masks in a subdirectory # without the log file. masks_output_dir = args.output_dir / 'masks' for sequence_dir, sequence_predictions in tqdm(all_predictions.items()): relative_dir = sequence_dir.relative_to(args.detections_root) create_masks_sequence(sequence_predictions, args.images_dir / relative_dir, masks_output_dir / relative_dir, args.threshold, args.extension, duplicate_last_frame=args.duplicate_last_frame)
def main(): global args, best_psnr args = parser.parse_args() # massage args block_opts = [] block_opts = args.block_opts block_opts.append(args.block_overlap) time_stamp = datetime.now().strftime('%Y-%m-%d_%H-%M-%S') if args.save is '': args.save = time_stamp save_path = os.path.join(args.results_dir, args.save) if not os.path.exists(save_path): os.makedirs(save_path) setup_logging(os.path.join(save_path, 'log_%s.txt' % time_stamp)) results_file = os.path.join(save_path, 'results.%s') results = ResultsLog(results_file % 'csv', results_file % 'html') logging.info("saving to %s", save_path) logging.debug("run arguments: %s", args) random.seed(args.seed) np.random.seed(args.seed) torch.manual_seed(args.seed) torch.cuda.manual_seed(args.seed) if args.encoder_lr > 0: encoder_learn = True else: encoder_learn = False # create model if args.pretrained_net is not None: logging.info("=> using pre-trained model '{}'".format(args.arch)) model = models.__dict__[args.arch](block_opts, pretrained=args.pretrained_net, mask_path=args.mask_path, mean=args.mean, std=args.std, noise=args.noise, encoder_learn=encoder_learn, p=args.bernoulli_p, K=args.layers_k) else: logging.info("=> creating model '{}'".format(args.arch)) model = models.__dict__[args.arch](block_opts, mask_path=args.mask_path, mean=args.mean, std=args.std, noise=args.noise, encoder_learn=encoder_learn, p=args.bernoulli_p, K=args.layers_k) model = torch.nn.DataParallel(model, device_ids=args.gpus).cuda() # define loss function (criterion) and optimizer mseloss = loss.EuclideanDistance(args.batch_size) # annual scedule if encoder_learn: optimizer = torch.optim.SGD( [{ 'params': model.module.measurements.parameters(), 'lr': args.encoder_lr }, { 'params': model.module.reconstruction.parameters() }], args.decoder_lr, momentum=args.momentum, weight_decay=args.weight_decay) def lambda1(epoch): return 0.0 if epoch >= args.encoder_annual[2] else ( args.encoder_annual[0]**bisect_right( range(args.encoder_annual[1], args.encoder_annual[2], args.encoder_annual[1]), epoch)) def lambda2(epoch): return args.decoder_annual[0]**bisect_right( [args.decoder_annual[1]], epoch) scheduler = torch.optim.lr_scheduler.LambdaLR( optimizer, lr_lambda=[lambda1, lambda2]) else: optimizer = torch.optim.SGD( [{ 'params': model.module.reconstruction.parameters() }], args.decoder_lr, momentum=args.momentum, weight_decay=args.weight_decay) scheduler = torch.optim.lr_scheduler.MultiStepLR( optimizer, milestones=[args.decoder_annual[1]], gamma=args.decoder_annual[0]) # optionally resume from a checkpoint if args.resume: if os.path.isfile(args.resume): logging.info("=> loading checkpoint '{}'".format(args.resume)) checkpoint = torch.load(args.resume) args.start_epoch = checkpoint['epoch'] best_psnr = checkpoint['best_psnr'] model.load_state_dict(checkpoint['state_dict']) optimizer.load_state_dict(checkpoint['optimizer']) logging.info("=> loaded checkpoint '{}' (epoch {})".format( args.resume, checkpoint['epoch'])) else: logging.info("=> no checkpoint found at '{}'".format(args.resume)) cudnn.benchmark = True # Data loading code train_loader = torch.utils.data.DataLoader(datasets.videocs.VideoCS( args.data_train, args.block_opts, transforms.Compose([ transforms.ToTensor(), ]), hdf5=args.hdf5), batch_size=args.batch_size, shuffle=False, num_workers=args.workers, pin_memory=True) val_loader = torch.utils.data.DataLoader(datasets.videocs.VideoCS( args.data_val, args.block_opts, transforms.Compose([ transforms.ToTensor(), ]), hdf5=False), batch_size=1, shuffle=False, num_workers=0, pin_memory=True) # Save initial mask if encoder_learn: initial_weights = binarization( model.module.measurements.weight.clone()) perc_1 = initial_weights.mean().cpu().data.numpy()[0] logging.info('Percentage of 1: {}'.format(perc_1)) np.save(save_path + '/initial_mask.npy', model.module.measurements.weight.clone()) else: # binarize weights model.module.measurements.binarization() perc_1 = model.module.measurements.weight.clone().mean().cpu( ).data.numpy()[0] logging.info('Percentage of 1: {}'.format(perc_1)) # perform first validation validate(val_loader, model, encoder_learn) for epoch in range(args.start_epoch, args.epochs): # Annual schedule enforcement scheduler.step() logging.info(scheduler.get_lr()) if encoder_learn: save_binary_weights_before = binarization( model.module.measurements.weight.clone()) # train for one epoch train_loss = train(train_loader, model, optimizer, epoch, mseloss, encoder_learn, args.gradient_clipping) if encoder_learn: save_binary_weights_after = binarization( model.module.measurements.weight.clone()) diff = np.int( torch.abs(save_binary_weights_after - save_binary_weights_before).sum().cpu().data.numpy()) perc_1 = save_binary_weights_after.mean().cpu().data.numpy()[0] logging.info( 'Binary Weights Changed: {} - Percentage of 1: {}'.format( diff, perc_1)) else: perc1 = model.module.measurements.weight.clone().mean().cpu( ).data.numpy()[0] logging.info('Percentage of 1: {}'.format(perc_1)) # evaluate on validation set psnr = validate(val_loader, model, encoder_learn) # remember best psnr and save checkpoint is_best = psnr > best_psnr best_psnr = max(psnr, best_psnr) save_checkpoint( { 'epoch': epoch + 1, 'arch': args.arch, 'state_dict': model.state_dict(), 'best_psnr': best_psnr, 'optimizer': optimizer.state_dict(), }, is_best, path=save_path) results_add(epoch, results, train_loss, psnr) if encoder_learn: model.module.measurements.restore()
def main(): tracking_parser = tracker.create_tracking_parser( suppress_args=['--score-init-min', '--score-continue-min']) # Use first line of file docstring as description if it exists. parser = argparse.ArgumentParser( description=__doc__.split('\n')[0] if __doc__ else '', parents=[tracking_parser], formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument( '--init-detections-dir', type=Path, help=('Contains subdirectories for each FBMS sequence, each of which ' 'contain pickle files of detections for each frame. These ' 'detections are used to initialize and continue tracks.'), required=True) parser.add_argument( '--continue-detections-dir', type=Path, help=('Contains subdirectories for each FBMS sequence, each of which ' 'contain pickle files of detections for each frame. These ' 'detections are used only to continue tracks.'), required=True) parser.add_argument( '--remove-continue-overlap', type=float, default=0.1, help=('Remove detections from --continue-detections-dir if they ' 'overlap more than this threshold with a detection from ' '--init-detections-dir.')) parser.add_argument( '--fbms-split-root', type=Path, required=True, help=('Directory containing subdirectories for each sequence, each of ' 'which contains a ".dat" file of groundtruth. E.g. ' '<FBMS_ROOT>/TestSet')) parser.add_argument('--output-dir', type=Path, required=True) parser.add_argument( '--score-init-min', type=float, default=0.9, help=('Detection confidence threshold for starting a new track from ' '--init-detections-dir detections.')) parser.add_argument( '--score-continue-min', type=float, default=0.7, help=('Detection confidence threshold for continuing a new track from ' '--init-detections-dir or --continue-detections-dir ' 'detections.')) parser.add_argument('--save-video', action='store_true') parser.add_argument('--fps', type=int, default=30) parser.add_argument('--extension', default='.jpg') parser.add_argument( '--vis-dataset', default='objectness', choices=['coco', 'objectness'], help='Dataset to use for mapping label indices to names.') parser.add_argument('--save-images', action='store_true') parser.add_argument('--filter-sequences', default=[], nargs='*', type=str) parser.add_argument('--save-merged-detections', action='store_true') tracking_params, remaining_argv = tracking_parser.parse_known_args() args = parser.parse_args(remaining_argv) tracking_params = vars(tracking_params) args.output_dir.mkdir(exist_ok=True, parents=True) output_log_file = log_utils.add_time_to_path(args.output_dir / 'tracker.log') log_utils.setup_logging(output_log_file) logging.info('Args: %s', pprint.pformat(vars(args))) logging.info('Tracking params: %s', pprint.pformat(tracking_params)) subprocess.call([ './git-state/save_git_state.sh', output_log_file.with_suffix('.git-state') ]) if args.save_merged_detections: output_merged = args.output_dir / 'merged' assert not output_merged.exists() output_merged.mkdir() # We want to use init_detections with score > s_i to init tracks, and # (init_detections or continue_detections with score > s_c) to continue # tracks. However, tracker.track only wants one set of detections, so we # do some score rescaling and then merge the detections. # # Let s_i be --score-init-min and s_c be --score-continue-min. # # Detections that can init tracks: # I1: init_detections with score > s_i # # Detections that can continue tracks: # C1: init_detections with score > s_c # C2: continue_detections with score > s_c # # Set the score_init_min passed to the tracker to 1. Then, we can increase # the score of all detections in I1 to be above 1 (by adding 1.001 to the # score), and leave all other detections' scores as they are. 1.001 is # arbitrary; we just need it to be higher than any regular scoring # detection. tracking_params['score_init_min'] = 1.001 tracking_params['score_continue_min'] = args.score_continue_min def detections_loader(sequence): init_detections = tracker.load_detectron_pickles( args.init_detections_dir / sequence, fbms_utils.get_framenumber) continue_detections = tracker.load_detectron_pickles( args.continue_detections_dir / sequence, fbms_utils.get_framenumber) merged_detections = merge_detections( init_detections, continue_detections, score_init_min=args.score_init_min, score_continue_min=args.score_continue_min, remove_continue_overlap=args.remove_continue_overlap, new_score_init_min=tracking_params['score_init_min']) if args.save_merged_detections: output_merged_sequence = output_merged / sequence output_merged_sequence.mkdir() for file in merged_detections: with open(output_merged_sequence / (file + '.pickle'), 'wb') as f: pickle.dump(merged_detections[file], f) return merged_detections if not args.filter_sequences: args.filter_sequences = None track_fbms(fbms_split_root=args.fbms_split_root, detections_loader=detections_loader, output_dir=args.output_dir, tracking_params=tracking_params, frame_extension=args.extension, save_video=args.save_video, vis_dataset=args.vis_dataset, fps=args.fps, save_images=args.save_images, filter_sequences=args.filter_sequences, duplicate_last_frame=False)
def main(): # Use first line of file docstring as description if it exists. parser = argparse.ArgumentParser( description=__doc__.split('\n')[0] if __doc__ else '', formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument( '--eval-binary', help='Path to MoSegEvalAllPr binary from FBMS eval code.', required=True) parser.add_argument( '--predictions-dir', help=('Predictions directory. Should contain "all_tracks.txt" and ' '"all_shots.txt".')) parser.add_argument('--split', default='all', choices=['10', '50', '200', 'all']) parser.add_argument( '--object-threshold', default=0.75, help=('F-measure threshold for whether an object has been extracted. ' 'Default is 0.75 following {Ochs, Peter, Jitendra Malik, and ' 'Thomas Brox. "Segmentation of moving objects by long term ' 'video analysis." IEEE transactions on pattern analysis and ' 'machine intelligence 36.6 (2014): 1187-1200.}')) args = parser.parse_args() binary = args.eval_binary predictions_dir = Path(args.predictions_dir) logging_path = log_utils.add_time_to_path(predictions_dir / (Path(__file__).name + '.log')) log_utils.setup_logging(logging_path) logging.info('Args:\n%s', vars(args)) file_logger = logging.getLogger(str(logging_path)) shots_file = predictions_dir / 'all_shots.txt' tracks_file = predictions_dir / 'all_tracks.txt' command = [ str(binary), str(shots_file), args.split, str(tracks_file), str(args.object_threshold) ] logging.info('Running command:\n%s', (' '.join(command))) try: output = subprocess.check_output(command, stderr=subprocess.STDOUT) file_logger.info('Output from FBMS evaluation:\n%s', output.decode('utf-8')) except subprocess.CalledProcessError as e: logging.error('Error found when evaluating:') output = e.output.decode('utf-8') logging.exception(output) if ('Could not find "Average region density" in the file!' in output and len(args.predictions_dir) > 250): logging.info( "\n############\n" "### NOTE ###\n" "############\n" "This may be due to the very long path to --predictions-dir. " "Either move your results to have fewer characters in the " "path, OR use the modified FBMS evaluation code at " "<https://github.com/achalddave/fbms-evaluation>, or apply " "the patch at " "https://github.com/achalddave/fbms-evaluation/commit/e7df914" " to your copy of the evaluation code.") elif ('Could not find "Average Precision, Recall, F-measure" ' 'in the file!' in output): logging.info( "\n############\n" "### NOTE ###\n" "############\n" "This may be due to a very long output Numbers file from " "MoSegEvalPR. Try using the modified FBMS evaluation code " "at <https://github.com/achalddave/fbms-evaluation>, or " "apply the patch at " "https://github.com/achalddave/fbms-evaluation/commit/7fdff53f" " to your copy of the evaluation code.") import sys sys.exit(e) # Format of the output file from FBMS evaluation code. output_file = tracks_file.with_name( tracks_file.stem + '_Fgeq{:4.2f}'.format(100 * args.object_threshold) + 'Numbers.txt') if output_file.exists(): logging.info('Final results:\n') with open(output_file, 'r') as f: logging.info(f.read()) else: logging.error("Couldn't find FBMS evaluation results at path: %s" % output_file)
def main_worker(args): global best_prec1, dtype best_prec1 = 0 dtype = torch_dtypes.get(args.dtype) torch.manual_seed(args.seed) time_stamp = datetime.now().strftime('%Y-%m-%d_%H-%M-%S') if args.evaluate: args.results_dir = '/tmp' if args.save is '': args.save = time_stamp save_path = os.path.join(args.results_dir, args.save) args.distributed = args.local_rank >= 0 or args.world_size > 1 if not os.path.exists(save_path) and not (args.distributed and args.local_rank > 0): os.makedirs(save_path) setup_logging(os.path.join(save_path, 'log.txt'), resume=args.resume is not '', dummy=args.distributed and args.local_rank > 0) results_path = os.path.join(save_path, 'results') results = ResultsLog(results_path, title='Training Results - %s' % args.save) if 'cuda' in args.device and torch.cuda.is_available(): torch.cuda.manual_seed_all(args.seed) torch.cuda.set_device(args.device_ids[0]) cudnn.benchmark = True else: args.device_ids = None if not os.path.isfile(args.evaluate): parser.error('invalid checkpoint: {}'.format(args.evaluate)) checkpoint = torch.load(args.evaluate, map_location="cpu") # Overrride configuration with checkpoint info args.model = checkpoint.get('model', args.model) args.model_config = checkpoint.get('config', args.model_config) logging.info("saving to %s", save_path) logging.debug("run arguments: %s", args) logging.info("creating model %s", args.model) # create model model = models.__dict__[args.model] model_config = {'dataset': args.dataset} if args.model_config is not '': model_config = dict(model_config, **literal_eval(args.model_config)) model = model(**model_config) logging.info("created model with configuration: %s", model_config) num_parameters = sum([l.nelement() for l in model.parameters()]) logging.info("number of parameters: %d", num_parameters) # load checkpoint model.load_state_dict(checkpoint['state_dict']) logging.info("loaded checkpoint '%s' (epoch %s)", args.evaluate, checkpoint['epoch']) if args.absorb_bn: search_absorbe_bn(model, verbose=True) # define loss function (criterion) and optimizer loss_params = {} if args.label_smoothing > 0: loss_params['smooth_eps'] = args.label_smoothing criterion = getattr(model, 'criterion', nn.CrossEntropyLoss)(**loss_params) criterion.to(args.device, dtype) model.to(args.device, dtype) # Batch-norm should always be done in float if 'half' in args.dtype: FilterModules(model, module=is_bn).to(dtype=torch.float) trainer = Trainer(model, criterion, device_ids=args.device_ids, device=args.device, dtype=dtype, mixup=args.mixup, print_freq=args.print_freq) # Evaluation Data loading code val_data = DataRegime(getattr(model, 'data_eval_regime', None), defaults={ 'datasets_path': args.datasets_dir, 'name': args.dataset, 'split': 'val', 'augment': args.augment, 'input_size': args.input_size, 'batch_size': args.batch_size, 'shuffle': False, 'duplicates': args.duplicates, 'autoaugment': args.autoaugment, 'cutout': { 'holes': 1, 'length': 16 } if args.cutout else None, 'num_workers': args.workers, 'pin_memory': True, 'drop_last': False }) results = trainer.validate(val_data.get_loader(), duplicates=val_data.get('duplicates'), average_output=args.avg_out) logging.info(results) return results
def main(): tracking_parser = tracker.create_tracking_parser() # Use first line of file docstring as description if it exists. parser = argparse.ArgumentParser( description=__doc__.split('\n')[0] if __doc__ else '', parents=[tracking_parser], formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument( '--detections-dir', type=Path, help=('Contains subdirectories for each FBMS sequence, each of which ' 'contain pickle files of detections for each frame.'), required=True) parser.add_argument( '--fbms-split-root', type=Path, required=True, help=('Directory containing subdirectories for each sequence, each of ' 'which contains a ".dat" file of groundtruth. E.g. ' '<FBMS_ROOT>/TestSet')) parser.add_argument('--output-dir', type=Path, required=True) parser.add_argument('--save-images', action='store_true') parser.add_argument('--save-video', action='store_true') parser.add_argument('--fps', type=int, default=30) parser.add_argument('--extension', default='.jpg') parser.add_argument( '--vis-dataset', default='objectness', choices=['coco', 'objectness'], help='Dataset to use for mapping label indices to names.') parser.add_argument('--filter-sequences', default=[], nargs='*', type=str) parser.add_argument('--duplicate-last-frame', action='store_true') tracking_params, remaining_argv = tracking_parser.parse_known_args() args = parser.parse_args(remaining_argv) tracking_params = vars(tracking_params) args.output_dir.mkdir(exist_ok=True, parents=True) output_log_file = log_utils.add_time_to_path( args.output_dir / 'tracker.log') log_utils.setup_logging(output_log_file) logging.info('Args: %s', pprint.pformat(vars(args))) logging.info('Tracking params: %s', pprint.pformat(tracking_params)) subprocess.call([ './git-state/save_git_state.sh', output_log_file.with_suffix('.git-state') ]) detectron_input = args.detections_dir if not detectron_input.is_dir(): raise ValueError( '--detectron-dir %s is not a directory!' % args.detections_dir) for split in ['TestSet', 'TrainingSet']: if (detectron_input / split).exists(): raise ValueError( f"--detectron-dir contains a '{split}' subdirectory; it " "should just contain a subdirectory for each sequence.") def detections_loader(sequence): return tracker.load_detectron_pickles( detectron_input / sequence, frame_parser=fbms_utils.get_framenumber) if not args.filter_sequences: args.filter_sequences = None track_fbms( args.fbms_split_root, detections_loader, args.output_dir, tracking_params, args.extension, args.save_video, args.vis_dataset, args.fps, save_images=args.save_images, filter_sequences=args.filter_sequences, duplicate_last_frame=args.duplicate_last_frame)
parser = argparse.ArgumentParser( description=__doc__.split('\n')[0] if __doc__ else '', formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('--masks-dir', type=Path, required=True) parser.add_argument('--fbms-dir', required=True, type=Path) parser.add_argument( '--eval-code-dir', type=Path, default='/home/achald/research/misc/datasets/fbms/fgbg-eval-pavel/') parser.add_argument('--matlab-binary', type=Path, default='matlab') args = parser.parse_args() output_log_file = log_utils.add_time_to_path( args.masks_dir / (Path(__file__).name + '.log')) log_utils.setup_logging(output_log_file) for split in ['TrainingSet', 'TestSet']: try: command = [ 'matlab', '-nodesktop', '-nosplash', '-r', (f"evaluateAllSeqs('{args.fbms_dir}', '{args.masks_dir}', " f"{{'{split}'}}); quit") ] logging.info(f'Command:\n{" ".join(command)}') output = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=args.eval_code_dir) except subprocess.CalledProcessError as e: logging.fatal('Failed command.\nException: %s\nOutput:\n %s',
full_cmd = " ".join(cmd) msg = "{}, switch to master, Running Command [{}]".format(err_msg, full_cmd) process_utils.Alert(msg) log.logger.info(msg) self.process = subprocess.Popen( full_cmd, shell=True ) def onSlave(self, err_msg): self.master_lock_failed = 0 self.status = SLAVE msg = "{}, switch to slave, waiting {} seconds try master again".format(err_msg, self.on_slave_wait) log.logger.info(msg) process_utils.Alert(msg) self.terminate() time.sleep(self.on_slave_wait) def terminate(self): if self.process: process_utils.kill_process_tree(log.logger, None)#os.getpid()) if __name__ == "__main__": log.setup_logging("test_ha.log") server = HaServer(host='127.0.0.1') server.run()
channel_selection_fn = cs.output_select_channels_class_dependent tag += '-CDO' ## slice layers # select_layer_mode = 'slice' #'positional' # None # 'auto_group'# 'auto' # select_layer_kwargs = dict(start=0.75,stop=0.9999,stride=3,keep_last_n=2) # limit_measure = None #1000 # per class limit if terminal.ood_datasets: ood_datasets = terminal.ood_datasets # batch_size_measure=5000 return locals() # exp_ids=[] # experiments = [experiments[exp_id] for exp_id in exp_ids] setup_logging() for args in experiments: args.update(common_overwrites()) logging.info(args) if args.num_classes > 300: _USE_PERCENTILE_DEVICE = True else: _USE_PERCENTILE_DEVICE = False measure_and_eval(args, **measure_kwargs) # report_from_file('./Final_results_baseine_2020-10-08/*/*', skip_pattern=None, include_pattern=r'.*') report_from_file( os.path.join(args.results_root, f'*experiment_results-*{args.tag}*'), # skip_pattern=r'(^simes)|(^fusion)', include_pattern=r'(.*)', output=os.path.join(args.results_root, args.tag)) pass
def main(): # Use first line of file docstring as description if it exists. parser = argparse.ArgumentParser( description=__doc__.split('\n')[0] if __doc__ else '', formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument('--input-json', required=True) parser.add_argument('--fbms-root', required=True) parser.add_argument('--output-json', required=True) args = parser.parse_args() with open(__file__, 'r') as f: _source = f.read() input_path = Path(args.input_json) fbms_root = Path(args.fbms_root) output_json = Path(args.output_json) assert input_path.exists() assert fbms_root.exists() assert not output_json.exists() log_path = args.output_json + '.log' setup_logging(log_path) logging.info('Args: %s' % pformat(vars(args))) last_frames = set() sequence_paths = list((fbms_root / 'TrainingSet').iterdir()) + list( (fbms_root / 'TestSet').iterdir()) for sequence_path in sequence_paths: images = sorted( sequence_path.glob('*.jpg'), key=lambda x: fbms_utils.get_framenumber(x)) last_frames.add(images[-1].stem) logging.info('Will remove up to %s images: %s' % (len(last_frames), sorted(last_frames))) with open(input_path, 'r') as f: input_data = json.load(f) valid_images = [] valid_image_ids = set() for image in input_data['images']: image_path = Path(image['file_name']) if image_path.stem not in last_frames: valid_images.append(image) valid_image_ids.add(image['id']) else: logging.info('Removing image id: %s, path: %s' % (image['id'], image_path)) valid_annotations = [] for annotation in input_data['annotations']: if annotation['image_id'] in valid_image_ids: valid_annotations.append(annotation) logging.info('Removed %s images, %s annotations' % (len(input_data['images']) - len(valid_images), len(input_data['annotations']) - len(valid_annotations))) logging.info('Kept %s images, %s annotations' % (len(valid_images), len(valid_annotations))) input_data['images'] = valid_images input_data['annotations'] = valid_annotations with open(output_json, 'w') as f: json.dump(input_data, f) file_logger = logging.getLogger(log_path) file_logger.info('Source:') file_logger.info('=======\n%s', _source) file_logger.info('=======')
def result_summary(res_dict, args_dict, TNR_target=0.05, skip_pattern=None, include_pattern='.*', pvalue_record=None): from utils.meters import simple_auc from _collections import OrderedDict ## if not configured setup logging for external caller if not logging.getLogger('').handlers: setup_logging() in_dist = args_dict['dataset'] alphas = args_dict['alphas'] logging.info(f'Report for {args_dict["model"]} - {in_dist}') logging.info(f'Tag: {args_dict["tag"]}') result_dict = OrderedDict( model=args_dict["model"], in_dist=args_dict['dataset'], LDA=args_dict.get('LDA'), joint=args_dict['measure_joint_distribution'], tag=args_dict['tag'], channles_sellect=args_dict.get('channel_selection_fn')) # read indist results to calibrate alpha value for target TNR rows = [] accuracies = {'model': {}} for reduction_name, reduction_metrics in res_dict[in_dist].items(): accuracies[reduction_name] = {} if reduction_name.endswith('_acc'): acc = reduction_metrics.mean.cpu().numpy() std = reduction_metrics.std.cpu().numpy() acc_name = reduction_name.replace('_acc', '') if acc_name == 'model': reduction_name = 'model' if acc_name.endswith('rescaled-smx'): reduction_name = acc_name[:-13] acc_name = 'model_rescaled_smx' elif acc_name.endswith('-pval'): reduction_name = acc_name[:-5] acc_name = 'pval' accuracies[reduction_name][f'{acc_name}_t1'] = acc[0] accuracies[reduction_name][f'{acc_name}_t5'] = acc[1] accuracies[reduction_name][f'{acc_name}_std_t1'] = std[0] for reduction_name, reduction_metrics in res_dict[in_dist].items(): if skip_pattern and bool(re.match( skip_pattern, reduction_name)) or include_pattern and not bool( re.match(include_pattern, reduction_name)): continue result_dict['reduction'] = reduction_name result_dict.update(**accuracies['model']) result_dict.update(**accuracies[reduction_name]) logging.info(reduction_name) if type(reduction_metrics) != dict: # report simple metric logging.info( f'\t{reduction_metrics.mean}\t({reduction_metrics.std})') continue # report reduction specific metrics for metric_name, meter_object in reduction_metrics.items(): metric_stats = MeterDict() if not metric_name.endswith('_roc'): logging.info( f'\t{metric_name}: {meter_object.mean.numpy():0.3}') continue FPR = meter_object.mean.numpy() calibrated_alpha_id = min((FPR < TNR_target).sum() - 1, len(FPR)) if calibrated_alpha_id == -1: # all pvalues are larger than alpha fpr_under_target_alpha = meter_object.mean[0] interp_alpha = FPR[0] calibrated_alpha_id = 0 else: fpr_under_target_alpha = FPR[calibrated_alpha_id] # actual rejection threshold to use for TNR 95% interp_alpha = np.interp(0.05, FPR.squeeze(), alphas) result_dict.update( dict(metric_name=metric_name, FPR_strict=fpr_under_target_alpha, FPR_over=FPR[calibrated_alpha_id + 1], chosen_alpha=interp_alpha)) logging.info( f'\t{metric_name} - in-dist rejected: ' # f'alpha-{indist_pvalues_roc[alphas.index(TNR_target)]:0.3f} ({TNR_target:0.3f}), ' f'under-{fpr_under_target_alpha:0.3f} ({alphas[calibrated_alpha_id]:0.3f}), ' f'interp-{TNR_target:0.3f} ({interp_alpha:0.3f}), ' f'over-{FPR[calibrated_alpha_id + 1]:0.3f} ({alphas[calibrated_alpha_id + 1]})' ) if pvalue_record and reduction_name in pvalue_record[in_dist]: if metric_name.startswith( 'class_cond' ) and 'predicted_id' in pvalue_record[in_dist]: predicted_ids = pvalue_record[in_dist]['predicted_id'] in_cc_pval_pred = pvalue_record[in_dist][reduction_name][ th.arange(predicted_ids.shape[0]), predicted_ids] else: in_cc_pval_pred = pvalue_record[in_dist][ reduction_name].max(1)[0] for target_dataset_name, reduction_metrics in res_dict.items(): if target_dataset_name != in_dist and metric_name in reduction_metrics[ reduction_name]: interp_rejected = np.interp( interp_alpha, alphas, reduction_metrics[reduction_name] [metric_name].mean.numpy()) TPR = reduction_metrics[reduction_name][ metric_name].mean.numpy() raw_rejected = TPR[alphas.index(TNR_target)] auroc = simple_auc(TPR, FPR) logging.info( f'\t\t{target_dataset_name}:\traw-{raw_rejected:0.3f}\tinterp-{interp_rejected:0.3f}\tAUROC:{auroc:0.3f}' ) if pvalue_record and reduction_name in pvalue_record[ target_dataset_name]: if metric_name.startswith( 'class_cond' ) and 'predicted_id' in pvalue_record[ target_dataset_name]: predicted_ids = pvalue_record[target_dataset_name][ 'predicted_id'] out_cc_pval_pred = pvalue_record[ target_dataset_name][reduction_name][ th.arange(predicted_ids.shape[0]), predicted_ids] else: out_cc_pval_pred = pvalue_record[ target_dataset_name][reduction_name].max(1)[0] m = metric(in_cc_pval_pred.numpy(), out_cc_pval_pred.numpy()) logging.info(f'\t\t\tbenchmark metrics: {m}') result_dict.update(**m) result_dict.update( dict(out_dist=target_dataset_name, TPR95_raw=raw_rejected, TPR95_interp=interp_rejected, AUROC=auroc)) rows.append(result_dict.copy()) if in_dist.startswith( 'cifar') and target_dataset_name.startswith( 'cifar'): continue metric_stats.update( dict(TPR95_raw=th.tensor([raw_rejected]), TPR95_interp=th.tensor([interp_rejected]), AUROC=th.tensor([auroc]))) if target_dataset_name != in_dist and metric_name in reduction_metrics[ reduction_name]: result_dict['out_dist'] = 'avg' logging.info( f'\tmetric avg stats: {[k + " " + str(float(v)) for k, v in metric_stats.get_mean_dict().items()]}' ) result_dict.update(**metric_stats.get_mean_dict()) rows.append(result_dict.copy()) return rows
def initServer(logfilie, mysql_conn): # print('--------------mysql_conn:%s'%mysql_conn) db.configure_orm(mysql_conn) fd = log.setup_logging(logfilie) #kafka_utils.setup_kafka(config.G_Conf.Common.Broker) return fd
def post_main(): global args, best_prec1, dtype best_prec1 = 0 args = parser.parse_args() dtype = torch_dtypes.get(args.dtype) torch.manual_seed(args.seed) time_stamp = datetime.now().strftime('%Y-%m-%d_%H-%M-%S') if args.evaluate: args.results_dir = '/tmp' if args.save is '': args.save = time_stamp save_path = os.path.join(args.results_dir, args.save) if not os.path.exists(save_path): os.makedirs(save_path) setup_logging(os.path.join(save_path, 'log.txt'), resume=args.resume is not '') results_path = os.path.join(save_path, 'results') results = ResultsLog( results_path, title='Training Results - %s' % args.save) logging.info("saving to %s", save_path) logging.debug("run arguments: %s", args) if 'cuda' in args.device and torch.cuda.is_available(): torch.cuda.manual_seed_all(args.seed) torch.cuda.set_device(args.device_ids[0]) cudnn.benchmark = True else: args.device_ids = None # create model logging.info("creating model %s", args.model) model = models.__dict__[args.model] model_config = {'input_size': args.input_size, 'dataset': args.dataset} if args.model_config is not '': model_config = dict(model_config, **literal_eval(args.model_config)) model = model(**model_config) logging.info("created model with configuration: %s", model_config) # optionally resume from a checkpoint if args.evaluate: if not os.path.isfile(args.evaluate): parser.error('invalid checkpoint: {}'.format(args.evaluate)) checkpoint = torch.load(args.evaluate) model.load_state_dict(checkpoint['state_dict']) logging.info("loaded checkpoint '%s' (epoch %s)", args.evaluate, checkpoint['epoch']) elif args.resume: checkpoint_file = args.resume if os.path.isdir(checkpoint_file): results.load(os.path.join(checkpoint_file, 'results.csv')) checkpoint_file = os.path.join( checkpoint_file, 'model_best.pth.tar') if os.path.isfile(checkpoint_file): logging.info("loading checkpoint '%s'", args.resume) checkpoint = torch.load(checkpoint_file) args.start_epoch = checkpoint['epoch'] - 1 best_prec1 = checkpoint['best_prec1'] model.load_state_dict(checkpoint['state_dict']) logging.info("loaded checkpoint '%s' (epoch %s)", checkpoint_file, checkpoint['epoch']) else: logging.error("no checkpoint found at '%s'", args.resume) num_parameters = sum([l.nelement() for l in model.parameters()]) logging.info("number of parameters: %d", num_parameters) # Data loading code default_transform = { 'train': get_transform(args.dataset, input_size=args.input_size, augment=True), 'eval': get_transform(args.dataset, input_size=args.input_size, augment=False) } transform = getattr(model, 'input_transform', default_transform) regime = getattr(model, 'regime', [{'epoch': 0, 'optimizer': args.optimizer, 'lr': args.lr, 'momentum': args.momentum, 'weight_decay': args.weight_decay}]) # define loss function (criterion) and optimizer criterion = getattr(model, 'criterion', nn.CrossEntropyLoss)() criterion.to(args.device, dtype) model.to(args.device, dtype) val_data = get_dataset(args.dataset, 'val', transform['eval']) val_loader = torch.utils.data.DataLoader( val_data, batch_size=args.batch_size, shuffle=False, num_workers=args.workers, pin_memory=True) if args.evaluate: validate(val_loader, model, criterion, 0) return train_data = get_dataset(args.dataset, 'train', transform['train']) train_loader = torch.utils.data.DataLoader( train_data, batch_size=args.batch_size, shuffle=True, num_workers=args.workers, pin_memory=True) optimizer = OptimRegime(model.parameters(), regime) logging.info('training regime: %s', regime) for i in range(len(get_layer_list(model))): sketch_layer(model, i) fix_layer(model, i) model.cuda() print(str(i)+'-Sketch: ') val_loss, val_prec1, val_prec5 = validate( val_loader, model, criterion, 0) print('Start Retraining ...') for epoch in range(1, 200+1): # train for one epoch train_loss, train_prec1, train_prec5 = train( train_loader, model, criterion, epoch, optimizer) val_loss, val_prec1, val_prec5 = validate(val_loader, model, criterion, 0)