def main():
# Use first line of file docstring as description if it exists.
parser = argparse.ArgumentParser(
description=__doc__.split('\n')[0] if __doc__ else '',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument(
'--dat-dir',
type=Path,
help=('Contains a .dat file for each sequence, unless '
'--dat-files-in-subdirs is specified, in which case it should '
'contain a subdirectory for each sequence, which in turn has a '
'single dat file.'))
parser.add_argument('--dat-file', type=Path)
parser.add_argument(
'--output',
help=('Output directory. If --dat-file is specified, this refers to '
'the output file. Default: {{dat_file_without_extension}}.npy '
'if --dat-file is specified, else '
'{{dat_dir}}/numpy-predictions if --dat-dir is specified.'))
parser.add_argument(
'--label-size-space-separated',
action='store_true',
help=('The FBMS code requires that there is a newline between <track '
'label> and <track size>, but one of the outputs I downloaded '
'from a method has the two fields separated by a space. This '
'boolean allows parsing such files.'))
parser.add_argument(
'--dat-files-in-subdirs',
action='store_true',
help=('See --dat-dir help for information. Ignored if --dat-file is '
'specified.'))
args = parser.parse_args()
assert (args.dat_file is None) != (args.dat_dir is None), (
'Exactly one of --dat-file or --dat-dir must be specified.')
if args.dat_file:
if not args.output:
args.output = '{dat_file_without_extension}.npy'
args.output = Path(
args.output.format(
dat_file_without_extension=args.dat_file.with_suffix('')))
else:
if not args.output:
args.output = '{dat_dir}/numpy-predictions'
args.output = Path(args.output.format(dat_dir=args.dat_dir))
assert not args.output.exists()
if args.dat_dir is not None:
args.output.mkdir(parents=True)
setup_logging(args.output / (Path(__file__).name + '.log'))
else:
setup_logging(args.output + '.log')
logging.info('Args:\n%s' % args)
if args.dat_dir is not None:
sequences = []
inputs = []
if args.dat_files_in_subdirs:
for sequence_dir in args.dat_dir.iterdir():
if sequence_dir.is_dir():
dat_file = list(sequence_dir.glob('*.dat'))
if len(dat_file) != 1:
logging.error(
'Found %s (!= 1) dat files in %s, skipping...' %
(len(dat_file), sequence_dir))
continue
inputs.append(dat_file[0])
sequences.append(sequence_dir.name)
else:
for sequence_dat in args.dat_dir.glob('*.dat'):
sequences.append(sequence_dat.stem)
inputs.append(sequence_dat)
outputs = [args.output / (x + '.npy') for x in sequences]
else:
inputs = [args.dat_file]
outputs = [args.output]
single_input = len(inputs) == 1
for input_path, output_path in zip(tqdm(inputs, disable=single_input),
outputs):
with open(input_path, 'r') as f:
tracks_txt = f.read()
width, height = Image.open(next(input_path.parent.glob('*.ppm'))).size
segmentation = parse_tracks(
tracks_txt,
image_shape=(height, width),
track_label_size_space_separated=args.label_size_space_separated,
progress=True)
np.save(output_path, segmentation)
def main():
tracking_parser = create_tracking_parser()
# Use first line of file docstring as description if it exists.
parser = argparse.ArgumentParser(
description=__doc__.split('\n')[0] if __doc__ else '',
parents=[tracking_parser],
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('--detectron-dir', type=Path, required=True)
parser.add_argument('--images-dir', type=Path, required=True)
parser.add_argument('--output-images-dir', type=Path)
parser.add_argument('--output-video', type=Path)
parser.add_argument('--output-video-fps', default=30, type=float)
parser.add_argument(
'--output-track-file',
type=Path,
help='Optional; path to output MOT17 style tracking output.')
parser.add_argument('--extensions', nargs='*', default=IMG_EXTENSIONS)
parser.add_argument('--dataset',
default='coco',
choices=['coco', 'objectness'])
parser.add_argument(
'--filename-format',
choices=[
'frame', 'frameN', 'sequence_frame', 'sequence-frame', 'fbms'
],
default='frame',
help=('Specifies how to get frame number from the filename. '
'"frame": the filename is the frame number, '
'"frameN": format <frame><number>, '
'"sequence_frame": frame number is separated by an underscore, '
'"sequence-frame": frame number is separated by a dash, '
'"fbms": assume fbms style frame numbers'))
tracking_params, remaining_argv = tracking_parser.parse_known_args()
args = parser.parse_args(remaining_argv)
tracking_params = vars(tracking_params)
assert (
args.output_images_dir is not None or args.output_video is not None
or args.output_track_file is not None), (
'One of --output-dir, --output-video, or --output-track-file must '
'be specified.')
if args.output_track_file is not None:
output_log_file = args.output_track_file.with_name(
args.output_track_file.stem + '-tracker.log')
elif args.output_video is not None:
output_log_file = args.output_video.with_name(args.output_video.stem +
'-tracker.log')
elif args.output_images_dir is not None:
output_log_file = args.output_images_dir / 'tracker.log'
output_log_file.parent.mkdir(exist_ok=True, parents=True)
setup_logging(output_log_file)
logging.info('Printing source code to logging file')
with open(__file__, 'r') as f:
logging.debug(f.read())
logging.info('Args: %s', pprint.pformat(vars(args)))
logging.info('Tracking params: %s', pprint.pformat(tracking_params))
subprocess.call([
'./git-state/save_git_state.sh',
output_log_file.with_suffix('.git-state')
])
detectron_input = args.detectron_dir
if not detectron_input.is_dir():
raise ValueError('--detectron-dir %s is not a directory!' %
args.detectron_dir)
if args.filename_format == 'fbms':
from utils.fbms.utils import get_framenumber
elif args.filename_format == 'frameN':
def get_framenumber(x):
return int(x.split('frame')[1])
elif args.filename_format == 'sequence-frame':
def get_framenumber(x):
return int(x.split('-')[-1])
elif args.filename_format == 'sequence_frame':
def get_framenumber(x):
return int(x.split('_')[-1])
elif args.filename_format == 'frame':
get_framenumber = int
else:
raise ValueError('Unknown --filename-format: %s' %
args.filename_format)
detection_results = load_detectron_pickles(args.detectron_dir,
frame_parser=get_framenumber)
track_and_visualize(detection_results,
args.images_dir,
tracking_params,
get_framenumber,
args.extensions,
vis_dataset=args.dataset,
output_images_dir=args.output_images_dir,
output_video=args.output_video,
output_video_fps=args.output_video_fps,
output_track_file=args.output_track_file)
def main_worker(args):
global best_prec1, dtype
best_prec1 = 0
dtype = torch_dtypes.get(args.dtype)
torch.manual_seed(args.seed)
time_stamp = datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
if args.eval_path:
args.results_dir = os.path.join(args.results_dir, 'evaluating_results')
else:
args.results_dir = os.path.join(args.results_dir, 'training_results')
if not os.path.exists(args.results_dir):
os.mkdir(args.results_dir)
if args.save is '':
args.save = time_stamp
save_path = os.path.join(args.results_dir, args.save)
args.distributed = args.local_rank >= 0 or args.world_size > 1
if not os.path.exists(save_path) and not (args.distributed
and args.local_rank > 0):
os.makedirs(save_path)
setup_logging(os.path.join(save_path, 'log.txt'),
resume=args.resume is not '',
dummy=args.distributed and args.local_rank > 0)
results_path = os.path.join(save_path, 'results')
results = ResultsLog(results_path,
title='Training Results - %s' % args.save)
if 'cuda' in args.device and torch.cuda.is_available():
torch.cuda.manual_seed_all(args.seed)
torch.cuda.set_device(args.device_ids[0])
cudnn.benchmark = True
else:
args.device_ids = None
dump_args(args, os.path.join(save_path, 'args.txt'))
# Pruning and evaluating
if len(args.pruning_percs) > 0:
results = []
checkpoint = torch.load(args.eval_path, map_location="cpu")
for perc in args.pruning_percs:
model, criterion = create_model_and_criterion(args)
model = register_stats_collectors(model)
model = prune_model(model,
checkpoint['state_dict'],
prune_perc=perc)
res = eval_checkpoint(args, model, criterion)['prec1']
results.append(res)
# After we gathered min/max statistics we can gather also histograms.
if args.gather_histograms == True:
model = register_hist_collectors(model)
_ = eval_checkpoint(args, model, criterion)['prec1']
dump_buffers(model, save_path)
for perc, res in zip(args.pruning_percs, results):
print("prune%:", perc, " acc:", res)
def main():
global args, best_psnr
args = parser.parse_args()
# massage args
block_opts = []
block_opts = args.block_opts
block_opts.append(args.block_overlap)
time_stamp = datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
if args.save is '':
args.save = time_stamp
save_path = os.path.join(args.results_dir, args.save)
if not os.path.exists(save_path):
os.makedirs(save_path)
setup_logging(os.path.join(save_path, 'log_%s.txt' % time_stamp))
results_file = os.path.join(save_path, 'results.%s')
results = ResultsLog(results_file % 'csv', results_file % 'html')
logging.info("saving to %s", save_path)
logging.debug("run arguments: %s", args)
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
torch.cuda.manual_seed(args.seed)
if args.encoder_lr > 0:
encoder_learn = True
else:
encoder_learn = False
# create model
if args.pretrained_net is not None:
logging.info("=> using pre-trained model '{}'".format(args.arch))
model = models.__dict__[args.arch](
block_opts, pretrained=args.pretrained_net, mask_path=args.mask_path, mean=args.mean, std=args.std,
noise=args.noise, encoder_learn=encoder_learn, p=args.bernoulli_p, K=args.layers_k)
else:
logging.info("=> creating model '{}'".format(args.arch))
model = models.__dict__[args.arch](
block_opts, mask_path=args.mask_path, mean=args.mean, std=args.std,
noise=args.noise, encoder_learn=encoder_learn, p=args.bernoulli_p, K=args.layers_k)
model = torch.nn.DataParallel(model, device_ids=args.gpus).cuda()
# define loss function (criterion) and optimizer
mseloss = loss.EuclideanDistance(args.batch_size)
# annual scedule
if encoder_learn:
optimizer = torch.optim.SGD([
{'params': model.module.measurements.parameters(), 'lr': args.encoder_lr},
{'params': model.module.reconstruction.parameters()}],
args.decoder_lr, momentum=args.momentum, weight_decay=args.weight_decay)
def lambda1(epoch): return 0.0 if epoch >= args.encoder_annual[2] else (
args.encoder_annual[0] ** bisect_right(range(args.encoder_annual[1], args.encoder_annual[2], args.encoder_annual[1]), epoch))
def lambda2(
epoch): return args.decoder_annual[0] ** bisect_right([args.decoder_annual[1]], epoch)
scheduler = torch.optim.lr_scheduler.LambdaLR(
optimizer, lr_lambda=[lambda1, lambda2])
else:
optimizer = torch.optim.SGD([
{'params': model.module.reconstruction.parameters()}],
args.decoder_lr, momentum=args.momentum, weight_decay=args.weight_decay)
scheduler = torch.optim.lr_scheduler.MultiStepLR(
optimizer, milestones=[args.decoder_annual[1]], gamma=args.decoder_annual[0])
# optionally resume from a checkpoint
if args.resume:
if os.path.isfile(args.resume):
logging.info("=> loading checkpoint '{}'".format(args.resume))
checkpoint = torch.load(args.resume)
args.start_epoch = checkpoint['epoch']
best_psnr = checkpoint['best_psnr']
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
logging.info("=> loaded checkpoint '{}' (epoch {})"
.format(args.resume, checkpoint['epoch']))
else:
logging.info("=> no checkpoint found at '{}'".format(args.resume))
cudnn.benchmark = True
# Data loading code
train_loader = torch.utils.data.DataLoader(
datasets.videocs.VideoCS(args.data_train, args.block_opts, transforms.Compose([
transforms.ToTensor(),
]), hdf5=args.hdf5),
batch_size=args.batch_size, shuffle=False,
num_workers=args.workers, pin_memory=True)
val_loader = torch.utils.data.DataLoader(
datasets.videocs.VideoCS(args.data_val, args.block_opts, transforms.Compose([
transforms.ToTensor(),
]), hdf5=False),
batch_size=1, shuffle=False,
num_workers=0, pin_memory=True)
# Save initial mask
if encoder_learn:
initial_weights = binarization(
model.module.measurements.weight.clone())
perc_1 = initial_weights.mean().cpu().data.numpy()[0]
logging.info('Percentage of 1: {}'.format(perc_1))
np.save(save_path + '/initial_mask.npy',
model.module.measurements.weight.clone())
else:
# binarize weights
model.module.measurements.binarization()
perc_1 = model.module.measurements.weight.clone().mean().cpu().data.numpy()[
0]
logging.info('Percentage of 1: {}'.format(perc_1))
# perform first validation
validate(val_loader, model, encoder_learn)
for epoch in range(args.start_epoch, args.epochs):
# Annual schedule enforcement
scheduler.step()
logging.info(scheduler.get_lr())
if encoder_learn:
save_binary_weights_before = binarization(
model.module.measurements.weight.clone())
# train for one epoch
train_loss = train(train_loader, model, optimizer, epoch,
mseloss, encoder_learn, args.gradient_clipping)
if encoder_learn:
save_binary_weights_after = binarization(
model.module.measurements.weight.clone())
diff = np.int(torch.abs(save_binary_weights_after -
save_binary_weights_before).sum().cpu().data.numpy())
perc_1 = save_binary_weights_after.mean().cpu().data.numpy()[0]
logging.info(
'Binary Weights Changed: {} - Percentage of 1: {}'.format(diff, perc_1))
else:
perc1 = model.module.measurements.weight.clone().mean().cpu().data.numpy()[0]
logging.info('Percentage of 1: {}'.format(perc_1))
# evaluate on validation set
psnr = validate(val_loader, model, encoder_learn)
# remember best psnr and save checkpoint
is_best = psnr > best_psnr
best_psnr = max(psnr, best_psnr)
save_checkpoint({
'epoch': epoch + 1,
'arch': args.arch,
'state_dict': model.state_dict(),
'best_psnr': best_psnr,
'optimizer': optimizer.state_dict(),
}, is_best, path=save_path)
results_add(epoch, results, train_loss, psnr)
if encoder_learn:
model.module.measurements.restore()
def main():
# Use first line of file docstring as description if it exists.
parser = argparse.ArgumentParser(
description=__doc__.split('\n')[0] if __doc__ else '',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument(
'--input-dir',
help=('Contains numpy file of predictions for each sequence. Each '
'file should be named {sequence}.npy and contain a (height, '
'width, num_frames) numpy array. Also see '
'--predictions-in-subdir for alternative directory structure.'))
parser.add_argument(
'--predictions-pickle',
help=('Use pickle file containing dictionary mapping image_id to '
'prediction numpy array, instead of --input-dir. This can also '
'be used to resume from a previous run of this script, as this '
'script dumps an "predictions.pkl" file in the output dir. '
'Mutually exclusive with --input-dir.'))
parser.add_argument(
'--filename-format',
choices=['frame', 'sequence_frame', 'sequence-frame', 'fbms'],
default='frame',
help=('Specifies how to get frame number from the "file_name" stored '
'in COCO annotations. '
'"frame": the filename is the frame number, '
'"sequence_frame": frame number is separated by an underscore, '
'"sequence-frame": frame number is separated by a dash, '
'"fbms": assume fbms style frame numbers'))
parser.add_argument('--annotations-json', required=True)
parser.add_argument('--output-dir', required=True)
parser.add_argument('--visualize',
action='store_true',
help='Whether to visualize masks.')
parser.add_argument(
'--no-remove-largest-mask',
dest='remove_largest_mask',
action='store_false',
help=('By default, we remove the mask with the largest area, making '
'the assumption that it is background. Setting this flag '
'avoids this behavior (i.e. keeps the largest area mask).'))
parser.add_argument(
'--min-mask-portion',
default=0.001,
type=float,
help='Remove masks that are less than this portion of image size.')
parser.add_argument(
'--predictions-in-subdir',
action='store_true',
help=('If true, assume --input-dir contains a subdirectory for each '
'sequence, each of which contains a "results.npy" file.'))
args = parser.parse_args()
assert (args.input_dir is None) != (args.predictions_pickle is None), (
'Exactly one of --input-dir or --predictions-pickle required.')
if args.input_dir:
input_dir = Path(args.input_dir)
assert input_dir.exists()
output_dir = Path(args.output_dir)
output_dir.mkdir(exist_ok=True, parents=True)
_source_path = Path(__file__)
logging_path = output_dir / (_source_path.stem + _source_path.suffix +
'.log')
setup_logging(str(logging_path))
logging.info('Args:\n%s', vars(args))
groundtruth = COCO(args.annotations_json)
if args.filename_format == 'fbms':
def parse_frame_offset(path):
return get_frameoffset(path.parent.stem, get_framenumber(path))
elif args.filename_format == 'sequence-frame':
def parse_frame_offset(x):
return int(x.split('-')[-1])
elif args.filename_format == 'sequence_frame':
def parse_frame_offset(x):
return int(x.split('_')[-1])
elif args.filename_format == 'frame':
parse_frame_offset = int
else:
raise ValueError('Unknown --filename-format: %s' %
args.filename_format)
if args.input_dir:
image_to_predictions_numpy = load_numpy_annotations(
input_dir,
groundtruth,
parse_frame_offset,
predictions_in_results_npy=args.predictions_in_subdir)
with open(output_dir / 'predictions.pkl', 'wb') as f:
pickle.dump(image_to_predictions_numpy, f)
else:
# Use to continue from a previous run.
with open(args.predictions_pickle, 'rb') as f:
image_to_predictions_numpy = pickle.load(f)
if args.visualize:
from PIL import Image
max_masks_per_image = max(
len(np.unique(x)) for x in image_to_predictions_numpy.values())
colors = (np.random.rand(max_masks_per_image, 3) * 256).round()
vis_dir = output_dir / 'vis'
vis_dir.mkdir()
annotations = []
for image_id, annotation_np in tqdm(image_to_predictions_numpy.items()):
ids = sorted(np.unique(annotation_np))
masks = [annotation_np == object_id for object_id in ids]
# Sort masks by area
masks = sorted(masks, key=lambda mask: mask.sum())
if args.remove_largest_mask:
masks = masks[:-1] # Remove mask with largest area (background)
if args.visualize:
vis_mask = np.zeros(
(annotation_np.shape[0], annotation_np.shape[1], 3),
dtype=np.uint8)
for i, mask in enumerate(masks):
vis_mask[mask] = colors[i]
Image.fromarray(vis_mask).save(vis_dir / ('%s.png' % image_id))
if not masks:
continue
image_area = masks[0].shape[0] * masks[0].shape[1]
masks_np = np.array(masks, dtype=np.uint8).transpose(1, 2, 0)
rle_masks = mask_util.encode(np.asfortranarray(masks_np))
for rle_mask in rle_masks:
# See https://github.com/cocodataset/cocoapi/issues/70
rle_mask['counts'] = rle_mask['counts'].decode('ascii')
area = mask_util.area(rle_mask).item()
ratio = area / image_area
score = 0.75 # ratio * 0.3 + 0.7 # Map to (0.7, 1) range
if ratio < args.min_mask_portion:
continue
annotations.append({
'image_id': image_id,
'segmentation': rle_mask,
'category_id': 1,
'score': score,
'area': ratio,
'bbox': mask_util.toBbox(rle_mask).tolist()
})
for ann_id, ann in enumerate(annotations):
ann['id'] = ann_id + 1
ann['iscrowd'] = 0
with open(output_dir / 'results.json', 'w') as f:
json.dump(annotations, f)
predictions = groundtruth.loadRes(annotations)
coco_eval = COCOeval(groundtruth, predictions, 'bbox')
coco_eval.evaluate()
coco_eval.accumulate()
summary_f = StringIO()
with redirect_stdout(summary_f):
coco_eval.summarize()
logging.info('Detection evaluation summary:\n%s', summary_f.getvalue())
coco_eval = COCOeval(groundtruth, predictions, 'segm')
coco_eval.evaluate()
coco_eval.accumulate()
summary_f = StringIO()
with redirect_stdout(summary_f):
coco_eval.summarize()
logging.info('Segmentation evaluation summary:\n%s', summary_f.getvalue())
def main_worker(args):
global best_prec1, dtype
acc = -1
loss = -1
best_prec1 = 0
dtype = torch_dtypes.get(args.dtype)
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
time_stamp = datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
if args.evaluate:
args.results_dir = '/tmp'
if args.save is '':
args.save = time_stamp
save_path = os.path.join(args.results_dir, args.save)
args.distributed = args.local_rank >= 0 or args.world_size > 1
if args.distributed:
dist.init_process_group(backend=args.dist_backend,
init_method=args.dist_init,
world_size=args.world_size,
rank=args.local_rank)
args.local_rank = dist.get_rank()
args.world_size = dist.get_world_size()
if args.dist_backend == 'mpi':
# If using MPI, select all visible devices
args.device_ids = list(range(torch.cuda.device_count()))
else:
args.device_ids = [args.local_rank]
if not os.path.exists(save_path) and not (args.distributed
and args.local_rank > 0):
os.makedirs(save_path)
setup_logging(os.path.join(save_path, 'log.txt'),
resume=args.resume is not '',
dummy=args.distributed and args.local_rank > 0)
results_path = os.path.join(save_path, 'results')
results = ResultsLog(results_path,
title='Training Results - %s' % args.save)
logging.info("saving to %s", save_path)
logging.debug("run arguments: %s", args)
logging.info("creating model %s", args.model)
if 'cuda' in args.device and torch.cuda.is_available():
torch.cuda.manual_seed_all(args.seed)
torch.cuda.set_device(args.device_ids[0])
cudnn.benchmark = True
else:
args.device_ids = None
# create model
model = models.__dict__[args.model]
dataset_type = 'imagenet' if args.dataset == 'imagenet_calib' else args.dataset
model_config = {'dataset': dataset_type}
if args.model_config is not '':
if isinstance(args.model_config, dict):
for k, v in args.model_config.items():
if k not in model_config.keys():
model_config[k] = v
else:
args_dict = literal_eval(args.model_config)
for k, v in args_dict.items():
model_config[k] = v
if (args.absorb_bn or args.load_from_vision
or args.pretrained) and not args.batch_norn_tuning:
if args.load_from_vision:
import torchvision
exec_lfv_str = 'torchvision.models.' + args.load_from_vision + '(pretrained=True)'
model = eval(exec_lfv_str)
if 'pytcv' in args.model:
from pytorchcv.model_provider import get_model as ptcv_get_model
exec_lfv_str = 'ptcv_get_model("' + args.load_from_vision + '", pretrained=True)'
model_pytcv = eval(exec_lfv_str)
model = convert_pytcv_model(model, model_pytcv)
else:
if not os.path.isfile(args.absorb_bn):
parser.error('invalid checkpoint: {}'.format(args.evaluate))
model = model(**model_config)
checkpoint = torch.load(args.absorb_bn,
map_location=lambda storage, loc: storage)
checkpoint = checkpoint[
'state_dict'] if 'state_dict' in checkpoint.keys(
) else checkpoint
model.load_state_dict(checkpoint, strict=False)
if 'batch_norm' in model_config and not model_config['batch_norm']:
logging.info('Creating absorb_bn state dict')
search_absorbe_bn(model)
filename_ab = args.absorb_bn + '.absorb_bn' if args.absorb_bn else save_path + '/' + args.model + '.absorb_bn'
torch.save(model.state_dict(), filename_ab)
else:
filename_bn = save_path + '/' + args.model + '.with_bn'
torch.save(model.state_dict(), filename_bn)
if (args.load_from_vision
or args.absorb_bn) and not args.evaluate_init_configuration:
return
if 'inception' in args.model:
model = model(init_weights=False, **model_config)
else:
model = model(**model_config)
logging.info("created model with configuration: %s", model_config)
num_parameters = sum([l.nelement() for l in model.parameters()])
logging.info("number of parameters: %d", num_parameters)
# optionally resume from a checkpoint
if args.evaluate:
if not os.path.isfile(args.evaluate):
parser.error('invalid checkpoint: {}'.format(args.evaluate))
checkpoint = torch.load(args.evaluate, map_location="cpu")
# Overrride configuration with checkpoint info
args.model = checkpoint.get('model', args.model)
args.model_config = checkpoint.get('config', args.model_config)
if not model_config['batch_norm']:
search_absorbe_fake_bn(model)
# load checkpoint
if 'state_dict' in checkpoint.keys():
model.load_state_dict(checkpoint['state_dict'])
logging.info("loaded checkpoint '%s'", args.evaluate)
else:
model.load_state_dict(checkpoint, strict=False)
logging.info("loaded checkpoint '%s'", args.evaluate)
if args.resume:
checkpoint_file = args.resume
if os.path.isdir(checkpoint_file):
results.load(os.path.join(checkpoint_file, 'results.csv'))
checkpoint_file = os.path.join(checkpoint_file,
'model_best.pth.tar')
if os.path.isfile(checkpoint_file):
logging.info("loading checkpoint '%s'", args.resume)
checkpoint = torch.load(checkpoint_file)
if args.start_epoch < 0: # not explicitly set
args.start_epoch = checkpoint[
'epoch'] - 1 if 'epoch' in checkpoint.keys() else 0
best_prec1 = checkpoint[
'best_prec1'] if 'best_prec1' in checkpoint.keys() else -1
sd = checkpoint['state_dict'] if 'state_dict' in checkpoint.keys(
) else checkpoint
model.load_state_dict(sd, strict=False)
logging.info("loaded checkpoint '%s' (epoch %s)", checkpoint_file,
args.start_epoch)
else:
logging.error("no checkpoint found at '%s'", args.resume)
# define loss function (criterion) and optimizer
loss_params = {}
if args.label_smoothing > 0:
loss_params['smooth_eps'] = args.label_smoothing
criterion = getattr(model, 'criterion', CrossEntropyLoss)(**loss_params)
if args.kld_loss:
criterion = nn.KLDivLoss(reduction='mean')
criterion.to(args.device, dtype)
model.to(args.device, dtype)
# Batch-norm should always be done in float
if 'half' in args.dtype:
FilterModules(model, module=is_bn).to(dtype=torch.float)
# optimizer configuration
optim_regime = getattr(model, 'regime', [{
'epoch': 0,
'optimizer': args.optimizer,
'lr': args.lr,
'momentum': args.momentum,
'weight_decay': args.weight_decay
}])
if args.fine_tune or args.prune:
if not args.resume: args.start_epoch = 0
if args.update_only_th:
#optim_regime = [
# {'epoch': 0, 'optimizer': 'Adam', 'lr': 1e-4}]
optim_regime = [{
'epoch': 0,
'optimizer': 'SGD',
'lr': 1e-1
}, {
'epoch': 10,
'lr': 1e-2
}, {
'epoch': 15,
'lr': 1e-3
}]
else:
optim_regime = [{
'epoch': 0,
'optimizer': 'SGD',
'lr': 1e-4,
'momentum': 0.9
}, {
'epoch': 2,
'lr': 1e-5,
'momentum': 0.9
}, {
'epoch': 10,
'lr': 1e-6,
'momentum': 0.9
}]
optimizer = optim_regime if isinstance(optim_regime, OptimRegime) \
else OptimRegime(model, optim_regime, use_float_copy='half' in args.dtype)
# Training Data loading code
train_data = DataRegime(getattr(model, 'data_regime', None),
defaults={
'datasets_path': args.datasets_dir,
'name': args.dataset,
'split': 'train',
'augment': False,
'input_size': args.input_size,
'batch_size': args.batch_size,
'shuffle': not args.seq_adaquant,
'num_workers': args.workers,
'pin_memory': True,
'drop_last': True,
'distributed': args.distributed,
'duplicates': args.duplicates,
'autoaugment': args.autoaugment,
'cutout': {
'holes': 1,
'length': 16
} if args.cutout else None,
'inception_prep': 'inception' in args.model
})
if args.names_sp_layers is None and args.layers_precision_dict is None:
args.names_sp_layers = [
key[:-7] for key in model.state_dict().keys()
if 'weight' in key and 'running' not in key and (
'conv' in key or 'downsample.0' in key or 'fc' in key)
]
if args.keep_first_last:
args.names_sp_layers = [
name for name in args.names_sp_layers if name != 'conv1'
and name != 'fc' and name != 'Conv2d_1a_3x3.conv'
]
args.names_sp_layers = [
k for k in args.names_sp_layers if 'downsample' not in k
] if args.ignore_downsample else args.names_sp_layers
if args.num_sp_layers == 0 and not args.keep_first_last:
args.names_sp_layers = []
if args.layers_precision_dict is not None:
print(args.layers_precision_dict)
prunner = None
trainer = Trainer(model,
prunner,
criterion,
optimizer,
device_ids=args.device_ids,
device=args.device,
dtype=dtype,
distributed=args.distributed,
local_rank=args.local_rank,
mixup=args.mixup,
loss_scale=args.loss_scale,
grad_clip=args.grad_clip,
print_freq=args.print_freq,
adapt_grad_norm=args.adapt_grad_norm,
epoch=args.start_epoch,
update_only_th=args.update_only_th,
optimize_rounding=args.optimize_rounding)
# Evaluation Data loading code
args.eval_batch_size = args.eval_batch_size if args.eval_batch_size > 0 else args.batch_size
dataset_type = 'imagenet' if args.dataset == 'imagenet_calib' else args.dataset
val_data = DataRegime(getattr(model, 'data_eval_regime', None),
defaults={
'datasets_path': args.datasets_dir,
'name': dataset_type,
'split': 'val',
'augment': False,
'input_size': args.input_size,
'batch_size': args.eval_batch_size,
'shuffle': True,
'num_workers': args.workers,
'pin_memory': True,
'drop_last': False
})
if args.evaluate or args.resume:
from utils.layer_sensativity import search_replace_layer, extract_save_quant_state_dict, search_replace_layer_from_dict
if args.layers_precision_dict is not None:
model = search_replace_layer_from_dict(
model, ast.literal_eval(args.layers_precision_dict))
else:
model = search_replace_layer(model,
args.names_sp_layers,
num_bits_activation=args.nbits_act,
num_bits_weight=args.nbits_weight)
cached_input_output = {}
quant_keys = [
'.weight', '.bias', '.equ_scale', '.quantize_input.running_zero_point',
'.quantize_input.running_range', '.quantize_weight.running_zero_point',
'.quantize_weight.running_range',
'.quantize_input1.running_zero_point', '.quantize_input1.running_range'
'.quantize_input2.running_zero_point', '.quantize_input2.running_range'
]
if args.adaquant:
def Qhook(name, module, input, output):
if module not in cached_qinput:
cached_qinput[module] = []
# Meanwhile store data in the RAM.
cached_qinput[module].append(input[0].detach().cpu())
# print(name)
def hook(name, module, input, output):
if module not in cached_input_output:
cached_input_output[module] = []
# Meanwhile store data in the RAM.
cached_input_output[module].append(
(input[0].detach().cpu(), output.detach().cpu()))
# print(name)
from models.modules.quantize import QConv2d, QLinear
handlers = []
count = 0
for name, m in model.named_modules():
if isinstance(m, QConv2d) or isinstance(m, QLinear):
#if isinstance(m, QConv2d) or isinstance(m, QLinear):
# if isinstance(m, QConv2d):
m.quantize = False
if count < 1000:
# if (isinstance(m, QConv2d) and m.groups == 1) or isinstance(m, QLinear):
handlers.append(
m.register_forward_hook(partial(hook, name)))
count += 1
# Store input/output for all quantizable layers
trainer.validate(train_data.get_loader())
print("Input/outputs cached")
for handler in handlers:
handler.remove()
for m in model.modules():
if isinstance(m, QConv2d) or isinstance(m, QLinear):
m.quantize = True
mse_df = pd.DataFrame(
index=np.arange(len(cached_input_output)),
columns=['name', 'bit', 'shape', 'mse_before', 'mse_after'])
print_freq = 100
for i, layer in enumerate(cached_input_output):
if i > 0 and args.seq_adaquant:
count = 0
cached_qinput = {}
for name, m in model.named_modules():
if layer.name == name:
if count < 1000:
handler = m.register_forward_hook(
partial(Qhook, name))
count += 1
# Store input/output for all quantizable layers
trainer.validate(train_data.get_loader())
print("cashed quant Input%s" % layer.name)
cached_input_output[layer][0] = (
cached_qinput[layer][0], cached_input_output[layer][0][1])
handler.remove()
print("\nOptimize {}:{} for {} bit of shape {}".format(
i, layer.name, layer.num_bits, layer.weight.shape))
mse_before, mse_after, snr_before, snr_after, kurt_in, kurt_w = \
optimize_layer(layer, cached_input_output[layer], args.optimize_weights, batch_size=args.batch_size, model_name=args.model)
print("\nMSE before optimization: {}".format(mse_before))
print("MSE after optimization: {}".format(mse_after))
mse_df.loc[i, 'name'] = layer.name
mse_df.loc[i, 'bit'] = layer.num_bits
mse_df.loc[i, 'shape'] = str(layer.weight.shape)
mse_df.loc[i, 'mse_before'] = mse_before
mse_df.loc[i, 'mse_after'] = mse_after
mse_df.loc[i, 'snr_before'] = snr_before
mse_df.loc[i, 'snr_after'] = snr_after
mse_df.loc[i, 'kurt_in'] = kurt_in
mse_df.loc[i, 'kurt_w'] = kurt_w
mse_csv = args.evaluate + '.mse.csv'
mse_df.to_csv(mse_csv)
filename = args.evaluate + '.adaquant'
torch.save(model.state_dict(), filename)
train_data = None
cached_input_output = None
val_results = trainer.validate(val_data.get_loader())
logging.info(val_results)
if args.res_log is not None:
if not os.path.exists(args.res_log):
df = pd.DataFrame()
else:
df = pd.read_csv(args.res_log, index_col=0)
ckp = ntpath.basename(args.evaluate)
if args.cmp is not None:
ckp += '_{}'.format(args.cmp)
adaquant_type = 'adaquant_seq' if args.seq_adaquant else 'adaquant_parallel'
df.loc[ckp, 'acc_' + adaquant_type] = val_results['prec1']
df.to_csv(args.res_log)
# print(df)
elif args.per_layer:
# Store input/output for all quantizable layers
calib_all_8_results = trainer.validate(train_data.get_loader())
print('########## All 8bit results ###########', calib_all_8_results)
int8_opt_model_state_dict = torch.load(args.int8_opt_model_path)
int4_opt_model_state_dict = torch.load(args.int4_opt_model_path)
per_layer_results = {}
args.names_sp_layers = [
key[:-7] for key in model.state_dict().keys()
if 'weight' in key and 'running' not in key and 'quantize' not in
key and ('conv' in key or 'downsample.0' in key or 'fc' in key)
]
for layer_idx, layer in enumerate(args.names_sp_layers):
model.load_state_dict(int8_opt_model_state_dict, strict=False)
model = search_replace_layer(model, [layer],
num_bits_activation=args.nbits_act,
num_bits_weight=args.nbits_weight)
layer_keys = [
key for key in int8_opt_model_state_dict
for qpkey in quant_keys if layer + qpkey == key
]
for key in layer_keys:
model.state_dict()[key].copy_(int4_opt_model_state_dict[key])
calib_results = trainer.validate(train_data.get_loader())
model = search_replace_layer(model, [layer],
num_bits_activation=8,
num_bits_weight=8)
print('finished %d out of %d' %
(layer_idx, len(args.names_sp_layers)))
logging.info(layer)
logging.info(calib_results)
per_layer_results[layer] = {
'base precision':
8,
'replaced precision':
args.nbits_act,
'replaced layer':
layer,
'accuracy':
calib_results['prec1'],
'loss':
calib_results['loss'],
'Parameters Size [Elements]':
model.state_dict()[layer + '.weight'].numel(),
'MACs':
'-'
}
torch.save(
per_layer_results, args.evaluate + '.per_layer_accuracy.A' +
str(args.nbits_act) + '.W' + str(args.nbits_weight))
all_8_dict = {
'base precision': 8,
'replaced precision': args.nbits_act,
'replaced layer': '-',
'accuracy': calib_all_8_results['prec1'],
'loss': calib_all_8_results['loss'],
'Parameters Size [Elements]': '-',
'MACs': '-'
}
columns = [key for key in all_8_dict]
with open(
args.evaluate + '.per_layer_accuracy.A' + str(args.nbits_act) +
'.W' + str(args.nbits_weight) + '.csv', "w") as f:
f.write(",".join(columns) + "\n")
col = [str(all_8_dict[c]) for c in all_8_dict.keys()]
f.write(",".join(col) + "\n")
for layer in per_layer_results:
r = per_layer_results[layer]
col = [str(r[c]) for c in r.keys()]
f.write(",".join(col) + "\n")
elif args.mixed_builder:
if isinstance(args.names_sp_layers, list):
print('loading int8 model" ', args.int8_opt_model_path)
int8_opt_model_state_dict = torch.load(args.int8_opt_model_path)
print('loading int4 model" ', args.int4_opt_model_path)
int4_opt_model_state_dict = torch.load(args.int4_opt_model_path)
model.load_state_dict(int8_opt_model_state_dict, strict=False)
model = search_replace_layer(model,
args.names_sp_layers,
num_bits_activation=args.nbits_act,
num_bits_weight=args.nbits_weight)
for layer_idx, layer in enumerate(args.names_sp_layers):
layer_keys = [
key for key in int8_opt_model_state_dict
for qpkey in quant_keys if layer + qpkey == key
]
for key in layer_keys:
model.state_dict()[key].copy_(
int4_opt_model_state_dict[key])
print('switched layer %s to 4 bit' % (layer))
elif isinstance(args.names_sp_layers, dict):
quant_models = {}
base_precision = args.precisions[0]
for m, prec in zip(args.opt_model_paths, args.precisions):
print('For precision={}, loading {}'.format(prec, m))
quant_models[prec] = torch.load(m)
model.load_state_dict(quant_models[base_precision], strict=False)
for layer_name, nbits_list in args.names_sp_layers.items():
model = search_replace_layer(model, [layer_name],
num_bits_activation=nbits_list[0],
num_bits_weight=nbits_list[0])
layer_keys = [
key for key in quant_models[base_precision]
for qpkey in quant_keys if layer_name + qpkey == key
]
for key in layer_keys:
model.state_dict()[key].copy_(
quant_models[nbits_list[0]][key])
print('switched layer {} to {} bit'.format(
layer_name, nbits_list[0]))
if os.environ.get('DEBUG') == 'True':
from utils.layer_sensativity import check_quantized_model
fp_names = check_quantized_model(trainer.model)
if len(fp_names) > 0:
logging.info('Found FP32 layers in the model:')
logging.info(fp_names)
if args.eval_on_train:
mixedIP_results = trainer.validate(train_data.get_loader())
else:
mixedIP_results = trainer.validate(val_data.get_loader())
torch.save(
{
'state_dict': model.state_dict(),
'config-ip': args.names_sp_layers
}, args.evaluate + '.mixed-ip-results.' + args.suffix)
logging.info(mixedIP_results)
acc = mixedIP_results['prec1']
loss = mixedIP_results['loss']
elif args.batch_norn_tuning:
from utils.layer_sensativity import search_replace_layer, extract_save_quant_state_dict, search_replace_layer_from_dict
from models.modules.quantize import QConv2d
if args.layers_precision_dict is not None:
model = search_replace_layer_from_dict(
model, literal_eval(args.layers_precision_dict))
else:
model = search_replace_layer(model,
args.names_sp_layers,
num_bits_activation=args.nbits_act,
num_bits_weight=args.nbits_weight)
exec_lfv_str = 'torchvision.models.' + args.load_from_vision + '(pretrained=True)'
model_orig = eval(exec_lfv_str)
model_orig.to(args.device, dtype)
search_copy_bn_params(model_orig)
layers_orig = dict([(n, m) for n, m in model_orig.named_modules()
if isinstance(m, nn.Conv2d)])
layers_q = dict([(n, m) for n, m in model.named_modules()
if isinstance(m, QConv2d)])
for l in layers_orig:
conv_orig = layers_orig[l]
conv_q = layers_q[l]
conv_q.register_parameter('gamma',
nn.Parameter(conv_orig.gamma.clone()))
conv_q.register_parameter('beta',
nn.Parameter(conv_orig.beta.clone()))
del model_orig
search_add_bn(model)
print("Run BN tuning")
for tt in range(args.tuning_iter):
print(tt)
trainer.cal_bn_stats(train_data.get_loader())
search_absorbe_tuning_bn(model)
filename = args.evaluate + '.bn_tuning'
print("Save model to: {}".format(filename))
torch.save(model.state_dict(), filename)
val_results = trainer.validate(val_data.get_loader())
logging.info(val_results)
if args.res_log is not None:
if not os.path.exists(args.res_log):
df = pd.DataFrame()
else:
df = pd.read_csv(args.res_log, index_col=0)
ckp = ntpath.basename(args.evaluate)
df.loc[ckp, 'acc_bn_tuning'] = val_results['prec1']
df.loc[ckp, 'loss_bn_tuning'] = val_results['loss']
df.to_csv(args.res_log)
# print(df)
elif args.bias_tuning:
for epoch in range(args.epochs):
trainer.epoch = epoch
train_data.set_epoch(epoch)
val_data.set_epoch(epoch)
logging.info('\nStarting Epoch: {0}\n'.format(epoch + 1))
# train for one epoch
repeat_train = 20 if args.update_only_th else 1
for tt in range(repeat_train):
print(tt)
train_results = trainer.train(
train_data.get_loader(),
duplicates=train_data.get('duplicates'),
chunk_batch=args.chunk_batch)
logging.info(train_results)
val_results = trainer.validate(val_data.get_loader())
logging.info(val_results)
if args.res_log is not None:
if not os.path.exists(args.res_log):
df = pd.DataFrame()
else:
df = pd.read_csv(args.res_log, index_col=0)
ckp = ntpath.basename(args.evaluate)
if 'bn_tuning' in ckp:
ckp = ckp.replace('.bn_tuning', '')
df.loc[ckp, 'acc_bias_tuning'] = val_results['prec1']
df.to_csv(args.res_log)
# import pdb; pdb.set_trace()
else:
#print('Please Choose one of the following ....')
if model_config['measure']:
results = trainer.validate(train_data.get_loader(), rec=args.rec)
# results = trainer.validate(val_data.get_loader())
# print(results)
else:
if args.evaluate_init_configuration:
results = trainer.validate(val_data.get_loader())
if args.res_log is not None:
if not os.path.exists(args.res_log):
df = pd.DataFrame()
else:
df = pd.read_csv(args.res_log, index_col=0)
ckp = ntpath.basename(args.evaluate)
if args.cmp is not None:
ckp += '_{}'.format(args.cmp)
df.loc[ckp, 'acc_base'] = results['prec1']
df.to_csv(args.res_log)
if args.extract_bias_mean:
file_name = 'bias_mean_measure' if model_config[
'measure'] else 'bias_mean_quant'
torch.save(trainer.bias_mean, file_name)
if model_config['measure']:
filename = args.evaluate + '.measure'
if 'perC' in args.model_config: filename += '_perC'
torch.save(model.state_dict(), filename)
logging.info(results)
else:
if args.evaluate_init_configuration:
logging.info(results)
return acc, loss
def main():
global args, best_prec1, dtype
best_prec1 = 0
args = parser.parse_args()
dtype = torch_dtypes.get(args.dtype)
torch.manual_seed(args.seed)
time_stamp = datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
if args.evaluate:
args.results_dir = '/tmp'
if args.save is '':
args.save = time_stamp
if args.log_dir is None:
import time
writer = SummaryWriter(log_dir="runs/time-%d" % int(time.time()))
else:
writer = SummaryWriter(log_dir="runs/%s" % (args.log_dir))
save_path = os.path.join(args.results_dir, args.save)
if not os.path.exists(save_path):
os.makedirs(save_path)
setup_logging(os.path.join(save_path, 'log.txt'),
resume=args.resume is not '')
results_path = os.path.join(save_path, 'results')
results = ResultsLog(results_path,
title='Training Results - %s' % args.save)
logging.info("saving to %s", save_path)
logging.debug("run arguments: %s", args)
if 'cuda' in args.device and torch.cuda.is_available():
torch.cuda.manual_seed_all(args.seed)
torch.cuda.set_device(args.device_ids[0])
cudnn.benchmark = True
else:
args.device_ids = None
# create model
logging.info("creating model %s", args.model)
model = models.__dict__[args.model]
model_config = {'input_size': args.input_size, 'dataset': args.dataset}
if args.model_config is not '':
model_config = dict(model_config, **literal_eval(args.model_config))
model = model(**model_config)
logging.info("created model with configuration: %s", model_config)
# optionally resume from a checkpoint
if args.evaluate:
if not os.path.isfile(args.evaluate):
parser.error('invalid checkpoint: {}'.format(args.evaluate))
checkpoint = torch.load(args.evaluate)
model.load_state_dict(checkpoint['state_dict'])
logging.info("loaded checkpoint '%s' (epoch %s)", args.evaluate,
checkpoint['epoch'])
elif args.resume:
checkpoint_file = args.resume
if os.path.isdir(checkpoint_file):
results.load(os.path.join(checkpoint_file, 'results.csv'))
checkpoint_file = os.path.join(checkpoint_file,
'model_best.pth.tar')
if os.path.isfile(checkpoint_file):
logging.info("loading checkpoint '%s'", args.resume)
checkpoint = torch.load(checkpoint_file)
args.start_epoch = checkpoint['epoch'] - 1
best_prec1 = checkpoint['best_prec1']
model.load_state_dict(checkpoint['state_dict'])
logging.info("loaded checkpoint '%s' (epoch %s)", checkpoint_file,
checkpoint['epoch'])
else:
logging.error("no checkpoint found at '%s'", args.resume)
num_parameters = sum([l.nelement() for l in model.parameters()])
logging.info("number of parameters: %d", num_parameters)
# Data loading code
default_transform = {
'train':
get_transform(args.dataset, input_size=args.input_size, augment=True),
'eval':
get_transform(args.dataset, input_size=args.input_size, augment=False)
}
transform = getattr(model, 'input_transform', default_transform)
regime = getattr(model, 'regime', [
{
'epoch': 0,
'optimizer': args.optimizer,
'lr': args.lr,
'momentum': args.momentum,
'weight_decay': args.weight_decay
},
{
'epoch': 100,
'optimizer': args.optimizer,
'lr': args.lr * 0.1,
'momentum': args.momentum,
'weight_decay': args.weight_decay
},
{
'epoch': 200,
'optimizer': args.optimizer,
'lr': args.lr * 0.01,
'momentum': args.momentum,
'weight_decay': args.weight_decay
},
])
# define loss function (criterion) and optimizer
criterion = getattr(model, 'criterion', nn.CrossEntropyLoss)()
criterion.to(args.device, dtype)
model.to(args.device, dtype)
val_data = get_dataset(args.dataset, 'val', transform['eval'])
val_loader = torch.utils.data.DataLoader(val_data,
batch_size=args.batch_size,
shuffle=False,
num_workers=args.workers,
pin_memory=True)
if args.evaluate:
validate(val_loader, model, criterion, 0)
return
train_data = get_dataset(args.dataset, 'train', transform['train'])
train_loader = torch.utils.data.DataLoader(train_data,
batch_size=args.batch_size,
shuffle=True,
num_workers=args.workers,
pin_memory=True)
optimizer = OptimRegime(model.parameters(), regime)
logging.info('training regime: %s', regime)
for epoch in range(args.start_epoch, args.epochs):
# train for one epoch
train_loss, train_prec1, train_prec5 = train(train_loader, model,
criterion, epoch,
optimizer, writer)
# evaluate on validation set
val_loss, val_prec1, val_prec5 = validate(val_loader, model, criterion,
epoch, writer)
# remember best prec@1 and save checkpoint
is_best = val_prec1 > best_prec1
best_prec1 = max(val_prec1, best_prec1)
save_checkpoint(
{
'epoch': epoch + 1,
'model': args.model,
'config': args.model_config,
'state_dict': model.state_dict(),
'best_prec1': best_prec1,
'regime': regime
},
is_best,
path=save_path)
logging.info('\n Epoch: {0}\t'
'Training Loss {train_loss:.4f} \t'
'Training Prec@1 {train_prec1:.3f} \t'
'Training Prec@5 {train_prec5:.3f} \t'
'Validation Loss {val_loss:.4f} \t'
'Validation Prec@1 {val_prec1:.3f} \t'
'Validation Prec@5 {val_prec5:.3f} \n'.format(
epoch + 1,
train_loss=train_loss,
val_loss=val_loss,
train_prec1=train_prec1,
val_prec1=val_prec1,
train_prec5=train_prec5,
val_prec5=val_prec5))
results.add(epoch=epoch + 1,
train_loss=train_loss,
val_loss=val_loss,
train_error1=100 - train_prec1,
val_error1=100 - val_prec1,
train_error5=100 - train_prec5,
val_error5=100 - val_prec5)
results.plot(x='epoch',
y=['train_loss', 'val_loss'],
legend=['training', 'validation'],
title='Loss',
ylabel='loss')
results.plot(x='epoch',
y=['train_error1', 'val_error1'],
legend=['training', 'validation'],
title='Error@1',
ylabel='error %')
results.plot(x='epoch',
y=['train_error5', 'val_error5'],
legend=['training', 'validation'],
title='Error@5',
ylabel='error %')
results.save()
def main():
tracking_parser = tracker.create_tracking_parser(
suppress_args=['--score-init-min', '--score-continue-min'])
# Use first line of file docstring as description if it exists.
parser = argparse.ArgumentParser(
description=__doc__.split('\n')[0] if __doc__ else '',
parents=[tracking_parser],
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('--images-dir', type=Path, required=True)
parser.add_argument(
'--init-detections-dir',
type=Path,
help=('Contains pickle files of detections for each frame.These '
'detections are used to initialize and continue tracks.'),
required=True)
parser.add_argument(
'--continue-detections-dir',
type=Path,
help=('Contains pickle files of detections for each frame.These '
'detections are used only to continue tracks. If not specified, '
'the continue detections are assumed to be stored in the '
'pickles in init_detections_dir, under the "appearance_stream" '
'key for each frame.'))
parser.add_argument(
'--remove-continue-overlap',
type=float,
default=0.1,
help=('Remove detections from --continue-detections-dir if they '
'overlap more than this threshold with a detection from '
'--init-detections-dir.'))
parser.add_argument('--output-dir', type=Path, required=True)
parser.add_argument(
'--score-init-min',
type=float,
default=0.9,
help=('Detection confidence threshold for starting a new track from '
'--init-detections-dir detections.'))
parser.add_argument(
'--score-continue-min',
type=float,
default=0.7,
help=('Detection confidence threshold for continuing a new track from '
'--init-detections-dir or --continue-detections-dir '
'detections.'))
parser.add_argument('--fps', type=int, default=30)
parser.add_argument('--extensions', nargs='*', default=IMG_EXTENSIONS)
parser.add_argument(
'--vis-dataset',
default='objectness',
choices=['coco', 'objectness'],
help='Dataset to use for mapping label indices to names.')
parser.add_argument('--save-images', type=parse_bool, default=False)
parser.add_argument('--save-merged-detections',
type=parse_bool,
default=False)
parser.add_argument('--save-numpy', type=parse_bool, default=False)
parser.add_argument('--save-numpy-every-kth-frame',
help='Save only every kth frame in numpy output.',
type=int,
default=1)
parser.add_argument('--save-video', type=parse_bool, default=True)
parser.add_argument(
'--filename-format',
choices=[
'frame', 'frameN', 'sequence_frame', 'sequence-frame', 'fbms'
],
default='frame',
help=('Specifies how to get frame number from the filename. '
'"frame": the filename is the frame number, '
'"frameN": format <frame><number>, '
'"sequence_frame": frame number is separated by an underscore, '
'"sequence-frame": frame number is separated by a dash, '
'"fbms": assume fbms style frame numbers'))
parser.add_argument('--recursive',
action='store_true',
help='Look recursively in --images-dir for images.')
parser.add_argument('--quiet', action='store_true')
tracking_params, remaining_argv = tracking_parser.parse_known_args()
args = parser.parse_args(remaining_argv)
tracking_params = vars(tracking_params)
tracking_params['score_init_min'] = args.score_init_min
tracking_params['score_continue_min'] = args.score_continue_min
args.output_dir.mkdir(exist_ok=True, parents=True)
output_log_file = log_utils.add_time_to_path(args.output_dir /
'tracker.log')
log_utils.setup_logging(output_log_file)
if args.quiet:
logging.root.setLevel(logging.WARN)
logging.info('Args:\n%s', pprint.pformat(vars(args)))
logging.info('Tracking params:\n%s', pprint.pformat(tracking_params))
subprocess.call([
'./git-state/save_git_state.sh',
output_log_file.with_suffix('.git-state')
])
if args.filename_format == 'fbms':
get_framenumber = fbms_utils.get_framenumber
elif args.filename_format == 'frameN':
def get_framenumber(x):
return int(x.split('frame')[1])
elif args.filename_format == 'sequence-frame':
def get_framenumber(x):
return int(x.split('-')[-1])
elif args.filename_format == 'sequence_frame':
def get_framenumber(x):
return int(x.split('_')[-1])
elif args.filename_format == 'frame':
get_framenumber = int
else:
raise ValueError('Unknown --filename-format: %s' %
args.filename_format)
args.extensions = [x if x[0] == '.' else '.' + x for x in args.extensions]
track_fn = functools.partial(
two_detector_track,
get_framenumber=get_framenumber,
tracking_params=tracking_params,
remove_continue_overlap=args.remove_continue_overlap,
extensions=args.extensions,
output_numpy_every_kth_frame=args.save_numpy_every_kth_frame,
fps=args.fps)
if not args.recursive:
output_merged = None
if args.save_merged_detections:
output_merged = args.output_dir / 'merged'
assert not output_merged.exists()
output_merged.mkdir()
output_numpy = None
if args.save_numpy:
output_numpy = args.output_dir / 'results.npz'
output_images_dir = None
if args.save_images:
output_images_dir = args.output_dir / 'images'
output_images_dir.mkdir(exist_ok=True, parents=True)
output_video = None
if args.save_video:
output_video = args.output_dir / 'video.mp4'
track_fn(images_dir=args.images_dir,
init_detections_dir=args.init_detections_dir,
continue_detections_dir=args.continue_detections_dir,
output_video=output_video,
output_merged_dir=output_merged,
output_numpy=output_numpy,
progress=True)
else:
images = glob_ext(args.images_dir, args.extensions, recursive=True)
image_subdirs = sorted(
set(x.parent.relative_to(args.images_dir) for x in images))
for subdir in tqdm(image_subdirs):
output_merged = None
output_numpy = None
output_images_dir = None
output_video = None
if args.save_merged_detections:
output_merged = args.output_dir / subdir / 'merged-detections'
output_merged.mkdir(exist_ok=True, parents=True)
if args.save_numpy:
output_numpy = args.output_dir / subdir.with_suffix('.npz')
if args.save_images:
output_images_dir = args.output_dir / subdir / 'images'
output_images_dir.mkdir(exist_ok=True, parents=True)
if args.save_video:
output_video = args.output_dir / subdir.with_suffix('.mp4')
if all(x is None or x.exists()
for x in (output_merged, output_numpy, output_images_dir,
output_video)):
logging.info('%s already processed, skipping', subdir)
continue
init_dir = args.init_detections_dir / subdir
if not init_dir.exists():
logging.warn(
'Skipping sequence %s: detections not found at %s', subdir,
init_dir)
continue
if args.continue_detections_dir:
continue_dir = args.continue_detections_dir / subdir
if not continue_dir.exists():
logging.warn(
'Skipping sequence %s: detections not found at %s',
subdir, continue_dir)
continue
if args.continue_detections_dir:
continue_dir = args.continue_detections_dir / subdir
else:
continue_dir = None
track_fn(images_dir=args.images_dir / subdir,
init_detections_dir=args.init_detections_dir / subdir,
continue_detections_dir=continue_dir,
output_video=output_video,
output_images_dir=output_images_dir,
output_merged_dir=output_merged,
output_numpy=output_numpy,
progress=False)
def main_worker(args):
global best_prec1, dtype
best_prec1 = 0
dtype = torch_dtypes.get(args.dtype)
torch.manual_seed(args.seed)
time_stamp = datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
if args.evaluate:
args.results_dir = '/tmp'
if args.save is '':
args.save = time_stamp
save_path = path.join(args.results_dir, args.save)
args.distributed = args.local_rank >= 0 or args.world_size > 1
if args.distributed:
dist.init_process_group(backend=args.dist_backend,
init_method=args.dist_init,
world_size=args.world_size,
rank=args.local_rank)
args.local_rank = dist.get_rank()
args.world_size = dist.get_world_size()
if args.dist_backend == 'mpi':
# If using MPI, select all visible devices
args.device_ids = list(range(torch.cuda.device_count()))
else:
args.device_ids = [args.local_rank]
if not (args.distributed and args.local_rank > 0):
if not path.exists(save_path):
makedirs(save_path)
export_args_namespace(args, path.join(save_path, 'config.json'))
setup_logging(path.join(save_path, 'log.txt'),
resume=args.resume is not '',
dummy=args.distributed and args.local_rank > 0)
results_path = path.join(save_path, 'results')
results = ResultsLog(results_path,
title='Training Results - %s' % args.save)
writer = SummaryWriter(log_dir=save_path, purge_step=1)
logging.info("saving to %s", save_path)
logging.debug("run arguments: %s", args)
logging.info("creating model %s", args.model)
if 'cuda' in args.device and torch.cuda.is_available():
torch.cuda.manual_seed_all(args.seed)
torch.cuda.set_device(args.device_ids[0])
cudnn.benchmark = True
else:
args.device_ids = None
# create model
model = models.__dict__[args.model]
model_config = {'dataset': args.dataset}
if args.model_config is not '':
model_config = dict(model_config, **literal_eval(args.model_config))
model = model(**model_config)
logging.info("created model with configuration: %s", model_config)
num_parameters = sum([l.nelement() for l in model.parameters()])
logging.info("number of parameters: %d", num_parameters)
# optionally resume from a checkpoint
if args.evaluate:
if not path.isfile(args.evaluate):
parser.error('invalid checkpoint: {}'.format(args.evaluate))
checkpoint = torch.load(args.evaluate, map_location="cpu")
# Overrride configuration with checkpoint info
args.model = checkpoint.get('model', args.model)
args.model_config = checkpoint.get('config', args.model_config)
# load checkpoint
model.load_state_dict(checkpoint['state_dict'])
logging.info("loaded checkpoint '%s' (epoch %s)", args.evaluate,
checkpoint['epoch'])
if args.resume:
checkpoint_file = args.resume
if path.isdir(checkpoint_file):
results.load(path.join(checkpoint_file, 'results.csv'))
checkpoint_file = path.join(checkpoint_file, 'model_best.pth.tar')
if path.isfile(checkpoint_file):
logging.info("loading checkpoint '%s'", args.resume)
checkpoint = torch.load(checkpoint_file, map_location="cpu")
if args.start_epoch < 0: # not explicitly set
args.start_epoch = checkpoint['epoch']
best_prec1 = checkpoint['best_prec1']
model.load_state_dict(checkpoint['state_dict'])
optim_state_dict = checkpoint.get('optim_state_dict', None)
logging.info("loaded checkpoint '%s' (epoch %s)", checkpoint_file,
checkpoint['epoch'])
else:
logging.error("no checkpoint found at '%s'", args.resume)
else:
optim_state_dict = None
# define loss function (criterion) and optimizer
loss_params = {}
if args.label_smoothing > 0:
loss_params['smooth_eps'] = args.label_smoothing
criterion = getattr(model, 'criterion', CrossEntropyLoss)(**loss_params)
criterion.to(args.device, dtype)
model.to(args.device, dtype)
# Batch-norm should always be done in float
if 'half' in args.dtype:
FilterModules(model, module=is_bn).to(dtype=torch.float)
# optimizer configuration
optim_regime = getattr(model, 'regime', [{
'epoch': 0,
'optimizer': args.optimizer,
'lr': args.lr,
'momentum': args.momentum,
'weight_decay': args.weight_decay
}])
optimizer = optim_regime if isinstance(optim_regime, OptimRegime) \
else OptimRegime(model, optim_regime, use_float_copy='half' in args.dtype)
if optim_state_dict is not None:
optimizer.load_state_dict(optim_state_dict)
trainer = Trainer(model,
criterion,
optimizer,
device_ids=args.device_ids,
device=args.device,
dtype=dtype,
print_freq=args.print_freq,
distributed=args.distributed,
local_rank=args.local_rank,
mixup=args.mixup,
cutmix=args.cutmix,
loss_scale=args.loss_scale,
grad_clip=args.grad_clip,
adapt_grad_norm=args.adapt_grad_norm)
if args.tensorwatch:
trainer.set_watcher(filename=path.abspath(
path.join(save_path, 'tensorwatch.log')),
port=args.tensorwatch_port)
# Evaluation Data loading code
args.eval_batch_size = args.eval_batch_size if args.eval_batch_size > 0 else args.batch_size
val_data = DataRegime(getattr(model, 'data_eval_regime', None),
defaults={
'datasets_path': args.datasets_dir,
'name': args.dataset,
'split': 'val',
'augment': False,
'input_size': args.input_size,
'batch_size': args.eval_batch_size,
'shuffle': False,
'num_workers': args.workers,
'pin_memory': True,
'drop_last': False
})
if args.evaluate:
results = trainer.validate(val_data.get_loader())
logging.info(results)
return
# Training Data loading code
train_data_defaults = {
'datasets_path': args.datasets_dir,
'name': args.dataset,
'split': 'train',
'augment': True,
'input_size': args.input_size,
'batch_size': args.batch_size,
'shuffle': True,
'num_workers': args.workers,
'pin_memory': True,
'drop_last': True,
'distributed': args.distributed,
'duplicates': args.duplicates,
'autoaugment': args.autoaugment,
'cutout': {
'holes': 1,
'length': 16
} if args.cutout else None
}
if hasattr(model, 'sampled_data_regime'):
sampled_data_regime = model.sampled_data_regime
probs, regime_configs = zip(*sampled_data_regime)
regimes = []
for config in regime_configs:
defaults = {**train_data_defaults}
defaults.update(config)
regimes.append(DataRegime(None, defaults=defaults))
train_data = SampledDataRegime(regimes, probs)
else:
train_data = DataRegime(getattr(model, 'data_regime', None),
defaults=train_data_defaults)
logging.info('optimization regime: %s', optim_regime)
logging.info('data regime: %s', train_data)
args.start_epoch = max(args.start_epoch, 0)
trainer.training_steps = args.start_epoch * len(train_data)
for epoch in range(args.start_epoch, args.epochs):
trainer.epoch = epoch
train_data.set_epoch(epoch)
val_data.set_epoch(epoch)
logging.info('\nStarting Epoch: {0}\n'.format(epoch + 1))
# train for one epoch
train_results = trainer.train(train_data.get_loader(),
chunk_batch=args.chunk_batch)
# evaluate on validation set
val_results = trainer.validate(val_data.get_loader())
if args.distributed and args.local_rank > 0:
continue
# remember best prec@1 and save checkpoint
is_best = val_results['prec1'] > best_prec1
best_prec1 = max(val_results['prec1'], best_prec1)
if args.drop_optim_state:
optim_state_dict = None
else:
optim_state_dict = optimizer.state_dict()
save_checkpoint(
{
'epoch': epoch + 1,
'model': args.model,
'config': args.model_config,
'state_dict': model.state_dict(),
'optim_state_dict': optim_state_dict,
'best_prec1': best_prec1
},
is_best,
path=save_path,
save_all=args.save_all)
logging.info('\nResults - Epoch: {0}\n'
'Training Loss {train[loss]:.4f} \t'
'Training Prec@1 {train[prec1]:.3f} \t'
'Training Prec@5 {train[prec5]:.3f} \t'
'Validation Loss {val[loss]:.4f} \t'
'Validation Prec@1 {val[prec1]:.3f} \t'
'Validation Prec@5 {val[prec5]:.3f} \t\n'.format(
epoch + 1, train=train_results, val=val_results))
values = dict(epoch=epoch + 1, steps=trainer.training_steps)
values.update({'training ' + k: v for k, v in train_results.items()})
values.update({'validation ' + k: v for k, v in val_results.items()})
results.add(**values)
results.plot(x='epoch',
y=['training loss', 'validation loss'],
legend=['training', 'validation'],
title='Loss',
ylabel='loss')
results.plot(x='epoch',
y=['training error1', 'validation error1'],
legend=['training', 'validation'],
title='Error@1',
ylabel='error %')
results.plot(x='epoch',
y=['training error5', 'validation error5'],
legend=['training', 'validation'],
title='Error@5',
ylabel='error %')
if 'grad' in train_results.keys():
results.plot(x='epoch',
y=['training grad'],
legend=['gradient L2 norm'],
title='Gradient Norm',
ylabel='value')
results.save()
writer.add_scalar('Loss/validation', val_results['loss'], epoch)
writer.add_scalar('Loss/train', train_results['loss'], epoch)
writer.add_scalar('Top1/validation', val_results['prec1'], epoch)
writer.add_scalar('Top1/train', train_results['prec1'], epoch)
writer.add_scalar('Top5/validation', val_results['prec5'], epoch)
writer.add_scalar('Top5/train', train_results['prec5'], epoch)
writer.close()
def main():
# Use first line of file docstring as description if it exists.
parser = argparse.ArgumentParser(
description=__doc__.split('\n')[0] if __doc__ else '',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('--input-dir', required=True)
parser.add_argument('--output-dir', required=True)
parser.add_argument('--num-workers', default=8, type=int)
parser.add_argument('--recursive', action='store_true')
parser.add_argument(
'--sequence-normalize',
action='store_true',
help=("Normalize all images together. If --recursive is True, "
"normalize all images with the same parent directory together."))
parser.add_argument(
'--diagonal-normalize',
action='store_true',
help='Normalize magnitude proportional to the image diagonal length.')
parser.add_argument(
'--min-normalize-perc',
default=0,
type=float,
help=('Minimum magnitude normalization, as percent of image diagonal.'
'If <= 0, no minimum is used.'))
parser.add_argument('--visualize-angle-only', action='store_true')
args = parser.parse_args()
input_root = Path(args.input_dir)
output_root = Path(args.output_dir)
assert input_root.exists()
output_root.mkdir(exist_ok=True, parents=True)
setup_logging(str(output_root / (Path(__file__).stem + '.log')))
logging.info('Args:\n%s', vars(args))
logging.info('Collecting paths')
if args.recursive:
flow_paths = list(tqdm(input_root.rglob('*.png')))
else:
flow_paths = list(tqdm(input_root.glob('*.png')))
output_paths = {
flow_path: output_root / flow_path.relative_to(input_root)
for flow_path in flow_paths
}
for output_path in output_paths.values():
output_path.parent.mkdir(exist_ok=True, parents=True)
assert not (args.diagonal_normalize and args.sequence_normalize)
if args.min_normalize_perc > 0:
assert not args.diagonal_normalize
assert not args.sequence_normalize
if not args.sequence_normalize:
tasks = list(output_paths.items())
if args.diagonal_normalize:
flow = load_flow_png(str(tasks[0][0]))
diagonal = (flow.shape[0]**2 + flow.shape[1]**2)**0.5
magnitude_scale = diagonal / 12
else:
magnitude_scale = None
Parallel(n_jobs=args.num_workers)(
delayed(visualize_flow)(
flow_path,
output_path,
magnitude_scale,
args.visualize_angle_only,
clip_at_magnitude_scale=args.diagonal_normalize,
min_normalize_percent=args.min_normalize_perc)
for flow_path, output_path in tqdm(tasks))
else:
# Collect flows for each parent directory
flows_by_parent_dir = collections.defaultdict(list)
for flow_path in flow_paths:
flows_by_parent_dir[flow_path.parent].append(flow_path)
with Parallel(n_jobs=args.num_workers) as parallel:
for dir_flow_paths in tqdm(flows_by_parent_dir.values()):
flows = parallel(
delayed(load_flow_png)(str(path))
for path in dir_flow_paths)
diagonal = (flows[0].shape[0]**2 + flows[0].shape[1]**2)**0.5
for flow in flows:
flow[:, :, 1] = flow[:, :, 1].clip(max=diagonal)
magnitude_scale = max(flow[:, :, 1].max() for flow in flows)
tasks = [(flow_path, output_paths[flow_path])
for flow_path in dir_flow_paths]
parallel(
delayed(visualize_flow)(flow, output_path, magnitude_scale,
args.visualize_angle_only)
for flow, output_path in tasks)
def main():
with open(__file__, 'r') as f:
_source = f.read()
# Use first line of file docstring as description if it exists.
parser = argparse.ArgumentParser(
description=__doc__.split('\n')[0] if __doc__ else '',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('--input-split-videos', required=True)
parser.add_argument('--input-json', required=True)
parser.add_argument('--output-json', required=True)
args = parser.parse_args()
output_json = Path(args.output_json)
logging_path = args.output_json + '.log'
setup_logging(logging_path)
file_logger = logging.getLogger(logging_path)
logging.info('Args: %s', pformat(vars(args)))
with open(args.input_json, 'r') as f:
data = json.load(f)
if args.input_split_videos.endswith('.csv'):
import csv
with open(args.input_split_videos, 'r') as csvfile:
reader = csv.DictReader(csvfile)
valid_sequences = [row['video'] for row in reader]
else:
with open(args.input_split_videos, 'r') as f:
valid_sequences = [x.strip() for x in f.readlines()]
logging.info('Valid sequences: %s' % valid_sequences)
images = []
image_ids = set()
present_sequences = set()
valid_sequences = set(valid_sequences)
for image in data['images']:
sequence = str(PurePath(image['file_name']).parent)
present_sequences.add(sequence)
if sequence in valid_sequences:
images.append(image)
image_ids.add(image['id'])
new_annotations = [
x for x in data['annotations'] if x['image_id'] in image_ids
]
new_images = images
logging.info(
'Kept %s/%s sequences' %
(len(present_sequences & valid_sequences), len(present_sequences)))
logging.info('Kept %s/%s annotations, %s/%s images' %
(len(new_annotations), len(data['annotations']),
len(new_images), len(data['images'])))
data['annotations'] = new_annotations
data['images'] = new_images
with open(args.output_json, 'w') as f:
json.dump(data, f)
file_logger.info('Source:')
file_logger.info('=======')
file_logger.info(_source)
file_logger.info('=======')
def main():
# Use first line of file docstring as description if it exists.
parser = argparse.ArgumentParser(
description=__doc__.split('\n')[0] if __doc__ else '',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument(
'--detections-root',
type=Path,
help=('Contains subdirectory for each sequence, containing pickle '
'files of detectron outputs for each frame.'))
parser.add_argument('--output-dir', type=Path, required=True)
parser.add_argument('--images-dir', type=Path, required=True)
parser.add_argument('--threshold', type=float, default=0.7)
parser.add_argument('--recursive', action='store_true')
parser.add_argument('--extension',
default='.png',
help='Extension for images in --images-dir')
parser.add_argument(
'--duplicate-last-frame',
action='store_true',
help=('Whether to duplicate the last frame. This is useful if we '
'only have predictions for n-1 frames (since flow is only '
'computed on the first n-1 frames). NOTE: This only works if '
'the pickle files are of the form "<frame_id>.pickle".'))
args = parser.parse_args()
assert args.detections_root.exists()
args.output_dir.mkdir(exist_ok=True, parents=True)
setup_logging(
add_time_to_path(args.output_dir / (Path(__file__).name + '.log')))
logging.info('Args: %s\n', pprint.pformat(vars(args)))
if args.recursive:
all_pickles = args.detections_root.rglob('*.pickle')
all_predictions = collections.defaultdict(list)
for x in all_pickles:
all_predictions[x.parent].append(x)
else:
all_predictions = {
args.detections_root: list(args.detections_root.glob('*.pickle'))
}
if not all_predictions[args.detections_root]:
raise ValueError("Found no .pickle files in --detections-root. "
"Did you mean to specify --recursive?")
all_predictions = {
k: natsorted(v, alg=ns.PATH)
for k, v in all_predictions.items()
}
# The DAVIS 2016 evaluation code really doesn't like any other files /
# directories in the input directory, so we put the masks in a subdirectory
# without the log file.
masks_output_dir = args.output_dir / 'masks'
for sequence_dir, sequence_predictions in tqdm(all_predictions.items()):
relative_dir = sequence_dir.relative_to(args.detections_root)
create_masks_sequence(sequence_predictions,
args.images_dir / relative_dir,
masks_output_dir / relative_dir,
args.threshold,
args.extension,
duplicate_last_frame=args.duplicate_last_frame)
def main():
global args, best_psnr
args = parser.parse_args()
# massage args
block_opts = []
block_opts = args.block_opts
block_opts.append(args.block_overlap)
time_stamp = datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
if args.save is '':
args.save = time_stamp
save_path = os.path.join(args.results_dir, args.save)
if not os.path.exists(save_path):
os.makedirs(save_path)
setup_logging(os.path.join(save_path, 'log_%s.txt' % time_stamp))
results_file = os.path.join(save_path, 'results.%s')
results = ResultsLog(results_file % 'csv', results_file % 'html')
logging.info("saving to %s", save_path)
logging.debug("run arguments: %s", args)
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
torch.cuda.manual_seed(args.seed)
if args.encoder_lr > 0:
encoder_learn = True
else:
encoder_learn = False
# create model
if args.pretrained_net is not None:
logging.info("=> using pre-trained model '{}'".format(args.arch))
model = models.__dict__[args.arch](block_opts,
pretrained=args.pretrained_net,
mask_path=args.mask_path,
mean=args.mean,
std=args.std,
noise=args.noise,
encoder_learn=encoder_learn,
p=args.bernoulli_p,
K=args.layers_k)
else:
logging.info("=> creating model '{}'".format(args.arch))
model = models.__dict__[args.arch](block_opts,
mask_path=args.mask_path,
mean=args.mean,
std=args.std,
noise=args.noise,
encoder_learn=encoder_learn,
p=args.bernoulli_p,
K=args.layers_k)
model = torch.nn.DataParallel(model, device_ids=args.gpus).cuda()
# define loss function (criterion) and optimizer
mseloss = loss.EuclideanDistance(args.batch_size)
# annual scedule
if encoder_learn:
optimizer = torch.optim.SGD(
[{
'params': model.module.measurements.parameters(),
'lr': args.encoder_lr
}, {
'params': model.module.reconstruction.parameters()
}],
args.decoder_lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
def lambda1(epoch):
return 0.0 if epoch >= args.encoder_annual[2] else (
args.encoder_annual[0]**bisect_right(
range(args.encoder_annual[1], args.encoder_annual[2],
args.encoder_annual[1]), epoch))
def lambda2(epoch):
return args.decoder_annual[0]**bisect_right(
[args.decoder_annual[1]], epoch)
scheduler = torch.optim.lr_scheduler.LambdaLR(
optimizer, lr_lambda=[lambda1, lambda2])
else:
optimizer = torch.optim.SGD(
[{
'params': model.module.reconstruction.parameters()
}],
args.decoder_lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
scheduler = torch.optim.lr_scheduler.MultiStepLR(
optimizer,
milestones=[args.decoder_annual[1]],
gamma=args.decoder_annual[0])
# optionally resume from a checkpoint
if args.resume:
if os.path.isfile(args.resume):
logging.info("=> loading checkpoint '{}'".format(args.resume))
checkpoint = torch.load(args.resume)
args.start_epoch = checkpoint['epoch']
best_psnr = checkpoint['best_psnr']
model.load_state_dict(checkpoint['state_dict'])
optimizer.load_state_dict(checkpoint['optimizer'])
logging.info("=> loaded checkpoint '{}' (epoch {})".format(
args.resume, checkpoint['epoch']))
else:
logging.info("=> no checkpoint found at '{}'".format(args.resume))
cudnn.benchmark = True
# Data loading code
train_loader = torch.utils.data.DataLoader(datasets.videocs.VideoCS(
args.data_train,
args.block_opts,
transforms.Compose([
transforms.ToTensor(),
]),
hdf5=args.hdf5),
batch_size=args.batch_size,
shuffle=False,
num_workers=args.workers,
pin_memory=True)
val_loader = torch.utils.data.DataLoader(datasets.videocs.VideoCS(
args.data_val,
args.block_opts,
transforms.Compose([
transforms.ToTensor(),
]),
hdf5=False),
batch_size=1,
shuffle=False,
num_workers=0,
pin_memory=True)
# Save initial mask
if encoder_learn:
initial_weights = binarization(
model.module.measurements.weight.clone())
perc_1 = initial_weights.mean().cpu().data.numpy()[0]
logging.info('Percentage of 1: {}'.format(perc_1))
np.save(save_path + '/initial_mask.npy',
model.module.measurements.weight.clone())
else:
# binarize weights
model.module.measurements.binarization()
perc_1 = model.module.measurements.weight.clone().mean().cpu(
).data.numpy()[0]
logging.info('Percentage of 1: {}'.format(perc_1))
# perform first validation
validate(val_loader, model, encoder_learn)
for epoch in range(args.start_epoch, args.epochs):
# Annual schedule enforcement
scheduler.step()
logging.info(scheduler.get_lr())
if encoder_learn:
save_binary_weights_before = binarization(
model.module.measurements.weight.clone())
# train for one epoch
train_loss = train(train_loader, model, optimizer, epoch, mseloss,
encoder_learn, args.gradient_clipping)
if encoder_learn:
save_binary_weights_after = binarization(
model.module.measurements.weight.clone())
diff = np.int(
torch.abs(save_binary_weights_after -
save_binary_weights_before).sum().cpu().data.numpy())
perc_1 = save_binary_weights_after.mean().cpu().data.numpy()[0]
logging.info(
'Binary Weights Changed: {} - Percentage of 1: {}'.format(
diff, perc_1))
else:
perc1 = model.module.measurements.weight.clone().mean().cpu(
).data.numpy()[0]
logging.info('Percentage of 1: {}'.format(perc_1))
# evaluate on validation set
psnr = validate(val_loader, model, encoder_learn)
# remember best psnr and save checkpoint
is_best = psnr > best_psnr
best_psnr = max(psnr, best_psnr)
save_checkpoint(
{
'epoch': epoch + 1,
'arch': args.arch,
'state_dict': model.state_dict(),
'best_psnr': best_psnr,
'optimizer': optimizer.state_dict(),
},
is_best,
path=save_path)
results_add(epoch, results, train_loss, psnr)
if encoder_learn:
model.module.measurements.restore()
def main():
tracking_parser = tracker.create_tracking_parser(
suppress_args=['--score-init-min', '--score-continue-min'])
# Use first line of file docstring as description if it exists.
parser = argparse.ArgumentParser(
description=__doc__.split('\n')[0] if __doc__ else '',
parents=[tracking_parser],
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument(
'--init-detections-dir',
type=Path,
help=('Contains subdirectories for each FBMS sequence, each of which '
'contain pickle files of detections for each frame. These '
'detections are used to initialize and continue tracks.'),
required=True)
parser.add_argument(
'--continue-detections-dir',
type=Path,
help=('Contains subdirectories for each FBMS sequence, each of which '
'contain pickle files of detections for each frame. These '
'detections are used only to continue tracks.'),
required=True)
parser.add_argument(
'--remove-continue-overlap',
type=float,
default=0.1,
help=('Remove detections from --continue-detections-dir if they '
'overlap more than this threshold with a detection from '
'--init-detections-dir.'))
parser.add_argument(
'--fbms-split-root',
type=Path,
required=True,
help=('Directory containing subdirectories for each sequence, each of '
'which contains a ".dat" file of groundtruth. E.g. '
'<FBMS_ROOT>/TestSet'))
parser.add_argument('--output-dir', type=Path, required=True)
parser.add_argument(
'--score-init-min',
type=float,
default=0.9,
help=('Detection confidence threshold for starting a new track from '
'--init-detections-dir detections.'))
parser.add_argument(
'--score-continue-min',
type=float,
default=0.7,
help=('Detection confidence threshold for continuing a new track from '
'--init-detections-dir or --continue-detections-dir '
'detections.'))
parser.add_argument('--save-video', action='store_true')
parser.add_argument('--fps', type=int, default=30)
parser.add_argument('--extension', default='.jpg')
parser.add_argument(
'--vis-dataset',
default='objectness',
choices=['coco', 'objectness'],
help='Dataset to use for mapping label indices to names.')
parser.add_argument('--save-images', action='store_true')
parser.add_argument('--filter-sequences', default=[], nargs='*', type=str)
parser.add_argument('--save-merged-detections', action='store_true')
tracking_params, remaining_argv = tracking_parser.parse_known_args()
args = parser.parse_args(remaining_argv)
tracking_params = vars(tracking_params)
args.output_dir.mkdir(exist_ok=True, parents=True)
output_log_file = log_utils.add_time_to_path(args.output_dir /
'tracker.log')
log_utils.setup_logging(output_log_file)
logging.info('Args: %s', pprint.pformat(vars(args)))
logging.info('Tracking params: %s', pprint.pformat(tracking_params))
subprocess.call([
'./git-state/save_git_state.sh',
output_log_file.with_suffix('.git-state')
])
if args.save_merged_detections:
output_merged = args.output_dir / 'merged'
assert not output_merged.exists()
output_merged.mkdir()
# We want to use init_detections with score > s_i to init tracks, and
# (init_detections or continue_detections with score > s_c) to continue
# tracks. However, tracker.track only wants one set of detections, so we
# do some score rescaling and then merge the detections.
#
# Let s_i be --score-init-min and s_c be --score-continue-min.
#
# Detections that can init tracks:
# I1: init_detections with score > s_i
#
# Detections that can continue tracks:
# C1: init_detections with score > s_c
# C2: continue_detections with score > s_c
#
# Set the score_init_min passed to the tracker to 1. Then, we can increase
# the score of all detections in I1 to be above 1 (by adding 1.001 to the
# score), and leave all other detections' scores as they are. 1.001 is
# arbitrary; we just need it to be higher than any regular scoring
# detection.
tracking_params['score_init_min'] = 1.001
tracking_params['score_continue_min'] = args.score_continue_min
def detections_loader(sequence):
init_detections = tracker.load_detectron_pickles(
args.init_detections_dir / sequence, fbms_utils.get_framenumber)
continue_detections = tracker.load_detectron_pickles(
args.continue_detections_dir / sequence,
fbms_utils.get_framenumber)
merged_detections = merge_detections(
init_detections,
continue_detections,
score_init_min=args.score_init_min,
score_continue_min=args.score_continue_min,
remove_continue_overlap=args.remove_continue_overlap,
new_score_init_min=tracking_params['score_init_min'])
if args.save_merged_detections:
output_merged_sequence = output_merged / sequence
output_merged_sequence.mkdir()
for file in merged_detections:
with open(output_merged_sequence / (file + '.pickle'),
'wb') as f:
pickle.dump(merged_detections[file], f)
return merged_detections
if not args.filter_sequences:
args.filter_sequences = None
track_fbms(fbms_split_root=args.fbms_split_root,
detections_loader=detections_loader,
output_dir=args.output_dir,
tracking_params=tracking_params,
frame_extension=args.extension,
save_video=args.save_video,
vis_dataset=args.vis_dataset,
fps=args.fps,
save_images=args.save_images,
filter_sequences=args.filter_sequences,
duplicate_last_frame=False)
def main():
# Use first line of file docstring as description if it exists.
parser = argparse.ArgumentParser(
description=__doc__.split('\n')[0] if __doc__ else '',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument(
'--eval-binary',
help='Path to MoSegEvalAllPr binary from FBMS eval code.',
required=True)
parser.add_argument(
'--predictions-dir',
help=('Predictions directory. Should contain "all_tracks.txt" and '
'"all_shots.txt".'))
parser.add_argument('--split',
default='all',
choices=['10', '50', '200', 'all'])
parser.add_argument(
'--object-threshold',
default=0.75,
help=('F-measure threshold for whether an object has been extracted. '
'Default is 0.75 following {Ochs, Peter, Jitendra Malik, and '
'Thomas Brox. "Segmentation of moving objects by long term '
'video analysis." IEEE transactions on pattern analysis and '
'machine intelligence 36.6 (2014): 1187-1200.}'))
args = parser.parse_args()
binary = args.eval_binary
predictions_dir = Path(args.predictions_dir)
logging_path = log_utils.add_time_to_path(predictions_dir /
(Path(__file__).name + '.log'))
log_utils.setup_logging(logging_path)
logging.info('Args:\n%s', vars(args))
file_logger = logging.getLogger(str(logging_path))
shots_file = predictions_dir / 'all_shots.txt'
tracks_file = predictions_dir / 'all_tracks.txt'
command = [
str(binary),
str(shots_file), args.split,
str(tracks_file),
str(args.object_threshold)
]
logging.info('Running command:\n%s', (' '.join(command)))
try:
output = subprocess.check_output(command, stderr=subprocess.STDOUT)
file_logger.info('Output from FBMS evaluation:\n%s',
output.decode('utf-8'))
except subprocess.CalledProcessError as e:
logging.error('Error found when evaluating:')
output = e.output.decode('utf-8')
logging.exception(output)
if ('Could not find "Average region density" in the file!' in output
and len(args.predictions_dir) > 250):
logging.info(
"\n############\n"
"### NOTE ###\n"
"############\n"
"This may be due to the very long path to --predictions-dir. "
"Either move your results to have fewer characters in the "
"path, OR use the modified FBMS evaluation code at "
"<https://github.com/achalddave/fbms-evaluation>, or apply "
"the patch at "
"https://github.com/achalddave/fbms-evaluation/commit/e7df914"
" to your copy of the evaluation code.")
elif ('Could not find "Average Precision, Recall, F-measure" '
'in the file!' in output):
logging.info(
"\n############\n"
"### NOTE ###\n"
"############\n"
"This may be due to a very long output Numbers file from "
"MoSegEvalPR. Try using the modified FBMS evaluation code "
"at <https://github.com/achalddave/fbms-evaluation>, or "
"apply the patch at "
"https://github.com/achalddave/fbms-evaluation/commit/7fdff53f"
" to your copy of the evaluation code.")
import sys
sys.exit(e)
# Format of the output file from FBMS evaluation code.
output_file = tracks_file.with_name(
tracks_file.stem + '_Fgeq{:4.2f}'.format(100 * args.object_threshold) +
'Numbers.txt')
if output_file.exists():
logging.info('Final results:\n')
with open(output_file, 'r') as f:
logging.info(f.read())
else:
logging.error("Couldn't find FBMS evaluation results at path: %s" %
output_file)
def main_worker(args):
global best_prec1, dtype
best_prec1 = 0
dtype = torch_dtypes.get(args.dtype)
torch.manual_seed(args.seed)
time_stamp = datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
if args.evaluate:
args.results_dir = '/tmp'
if args.save is '':
args.save = time_stamp
save_path = os.path.join(args.results_dir, args.save)
args.distributed = args.local_rank >= 0 or args.world_size > 1
if not os.path.exists(save_path) and not (args.distributed
and args.local_rank > 0):
os.makedirs(save_path)
setup_logging(os.path.join(save_path, 'log.txt'),
resume=args.resume is not '',
dummy=args.distributed and args.local_rank > 0)
results_path = os.path.join(save_path, 'results')
results = ResultsLog(results_path,
title='Training Results - %s' % args.save)
if 'cuda' in args.device and torch.cuda.is_available():
torch.cuda.manual_seed_all(args.seed)
torch.cuda.set_device(args.device_ids[0])
cudnn.benchmark = True
else:
args.device_ids = None
if not os.path.isfile(args.evaluate):
parser.error('invalid checkpoint: {}'.format(args.evaluate))
checkpoint = torch.load(args.evaluate, map_location="cpu")
# Overrride configuration with checkpoint info
args.model = checkpoint.get('model', args.model)
args.model_config = checkpoint.get('config', args.model_config)
logging.info("saving to %s", save_path)
logging.debug("run arguments: %s", args)
logging.info("creating model %s", args.model)
# create model
model = models.__dict__[args.model]
model_config = {'dataset': args.dataset}
if args.model_config is not '':
model_config = dict(model_config, **literal_eval(args.model_config))
model = model(**model_config)
logging.info("created model with configuration: %s", model_config)
num_parameters = sum([l.nelement() for l in model.parameters()])
logging.info("number of parameters: %d", num_parameters)
# load checkpoint
model.load_state_dict(checkpoint['state_dict'])
logging.info("loaded checkpoint '%s' (epoch %s)", args.evaluate,
checkpoint['epoch'])
if args.absorb_bn:
search_absorbe_bn(model, verbose=True)
# define loss function (criterion) and optimizer
loss_params = {}
if args.label_smoothing > 0:
loss_params['smooth_eps'] = args.label_smoothing
criterion = getattr(model, 'criterion', nn.CrossEntropyLoss)(**loss_params)
criterion.to(args.device, dtype)
model.to(args.device, dtype)
# Batch-norm should always be done in float
if 'half' in args.dtype:
FilterModules(model, module=is_bn).to(dtype=torch.float)
trainer = Trainer(model,
criterion,
device_ids=args.device_ids,
device=args.device,
dtype=dtype,
mixup=args.mixup,
print_freq=args.print_freq)
# Evaluation Data loading code
val_data = DataRegime(getattr(model, 'data_eval_regime', None),
defaults={
'datasets_path': args.datasets_dir,
'name': args.dataset,
'split': 'val',
'augment': args.augment,
'input_size': args.input_size,
'batch_size': args.batch_size,
'shuffle': False,
'duplicates': args.duplicates,
'autoaugment': args.autoaugment,
'cutout': {
'holes': 1,
'length': 16
} if args.cutout else None,
'num_workers': args.workers,
'pin_memory': True,
'drop_last': False
})
results = trainer.validate(val_data.get_loader(),
duplicates=val_data.get('duplicates'),
average_output=args.avg_out)
logging.info(results)
return results
def main():
tracking_parser = tracker.create_tracking_parser()
# Use first line of file docstring as description if it exists.
parser = argparse.ArgumentParser(
description=__doc__.split('\n')[0] if __doc__ else '',
parents=[tracking_parser],
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument(
'--detections-dir',
type=Path,
help=('Contains subdirectories for each FBMS sequence, each of which '
'contain pickle files of detections for each frame.'),
required=True)
parser.add_argument(
'--fbms-split-root',
type=Path,
required=True,
help=('Directory containing subdirectories for each sequence, each of '
'which contains a ".dat" file of groundtruth. E.g. '
'<FBMS_ROOT>/TestSet'))
parser.add_argument('--output-dir', type=Path, required=True)
parser.add_argument('--save-images', action='store_true')
parser.add_argument('--save-video', action='store_true')
parser.add_argument('--fps', type=int, default=30)
parser.add_argument('--extension', default='.jpg')
parser.add_argument(
'--vis-dataset',
default='objectness',
choices=['coco', 'objectness'],
help='Dataset to use for mapping label indices to names.')
parser.add_argument('--filter-sequences', default=[], nargs='*', type=str)
parser.add_argument('--duplicate-last-frame', action='store_true')
tracking_params, remaining_argv = tracking_parser.parse_known_args()
args = parser.parse_args(remaining_argv)
tracking_params = vars(tracking_params)
args.output_dir.mkdir(exist_ok=True, parents=True)
output_log_file = log_utils.add_time_to_path(
args.output_dir / 'tracker.log')
log_utils.setup_logging(output_log_file)
logging.info('Args: %s', pprint.pformat(vars(args)))
logging.info('Tracking params: %s', pprint.pformat(tracking_params))
subprocess.call([
'./git-state/save_git_state.sh',
output_log_file.with_suffix('.git-state')
])
detectron_input = args.detections_dir
if not detectron_input.is_dir():
raise ValueError(
'--detectron-dir %s is not a directory!' % args.detections_dir)
for split in ['TestSet', 'TrainingSet']:
if (detectron_input / split).exists():
raise ValueError(
f"--detectron-dir contains a '{split}' subdirectory; it "
"should just contain a subdirectory for each sequence.")
def detections_loader(sequence):
return tracker.load_detectron_pickles(
detectron_input / sequence,
frame_parser=fbms_utils.get_framenumber)
if not args.filter_sequences:
args.filter_sequences = None
track_fbms(
args.fbms_split_root,
detections_loader,
args.output_dir,
tracking_params,
args.extension,
args.save_video,
args.vis_dataset,
args.fps,
save_images=args.save_images,
filter_sequences=args.filter_sequences,
duplicate_last_frame=args.duplicate_last_frame)
parser = argparse.ArgumentParser(
description=__doc__.split('\n')[0] if __doc__ else '',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('--masks-dir', type=Path, required=True)
parser.add_argument('--fbms-dir', required=True, type=Path)
parser.add_argument(
'--eval-code-dir',
type=Path,
default='/home/achald/research/misc/datasets/fbms/fgbg-eval-pavel/')
parser.add_argument('--matlab-binary', type=Path, default='matlab')
args = parser.parse_args()
output_log_file = log_utils.add_time_to_path(
args.masks_dir / (Path(__file__).name + '.log'))
log_utils.setup_logging(output_log_file)
for split in ['TrainingSet', 'TestSet']:
try:
command = [
'matlab', '-nodesktop', '-nosplash', '-r',
(f"evaluateAllSeqs('{args.fbms_dir}', '{args.masks_dir}', "
f"{{'{split}'}}); quit")
]
logging.info(f'Command:\n{" ".join(command)}')
output = subprocess.run(command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd=args.eval_code_dir)
except subprocess.CalledProcessError as e:
logging.fatal('Failed command.\nException: %s\nOutput:\n %s',
full_cmd = " ".join(cmd)
msg = "{}, switch to master, Running Command [{}]".format(err_msg, full_cmd)
process_utils.Alert(msg)
log.logger.info(msg)
self.process = subprocess.Popen(
full_cmd,
shell=True
)
def onSlave(self, err_msg):
self.master_lock_failed = 0
self.status = SLAVE
msg = "{}, switch to slave, waiting {} seconds try master again".format(err_msg, self.on_slave_wait)
log.logger.info(msg)
process_utils.Alert(msg)
self.terminate()
time.sleep(self.on_slave_wait)
def terminate(self):
if self.process:
process_utils.kill_process_tree(log.logger, None)#os.getpid())
if __name__ == "__main__":
log.setup_logging("test_ha.log")
server = HaServer(host='127.0.0.1')
server.run()
channel_selection_fn = cs.output_select_channels_class_dependent
tag += '-CDO'
## slice layers
# select_layer_mode = 'slice' #'positional' # None # 'auto_group'# 'auto'
# select_layer_kwargs = dict(start=0.75,stop=0.9999,stride=3,keep_last_n=2)
# limit_measure = None #1000 # per class limit
if terminal.ood_datasets:
ood_datasets = terminal.ood_datasets
# batch_size_measure=5000
return locals()
# exp_ids=[]
# experiments = [experiments[exp_id] for exp_id in exp_ids]
setup_logging()
for args in experiments:
args.update(common_overwrites())
logging.info(args)
if args.num_classes > 300:
_USE_PERCENTILE_DEVICE = True
else:
_USE_PERCENTILE_DEVICE = False
measure_and_eval(args, **measure_kwargs)
# report_from_file('./Final_results_baseine_2020-10-08/*/*', skip_pattern=None, include_pattern=r'.*')
report_from_file(
os.path.join(args.results_root, f'*experiment_results-*{args.tag}*'),
# skip_pattern=r'(^simes)|(^fusion)',
include_pattern=r'(.*)',
output=os.path.join(args.results_root, args.tag))
pass
def main():
# Use first line of file docstring as description if it exists.
parser = argparse.ArgumentParser(
description=__doc__.split('\n')[0] if __doc__ else '',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('--input-json', required=True)
parser.add_argument('--fbms-root', required=True)
parser.add_argument('--output-json', required=True)
args = parser.parse_args()
with open(__file__, 'r') as f:
_source = f.read()
input_path = Path(args.input_json)
fbms_root = Path(args.fbms_root)
output_json = Path(args.output_json)
assert input_path.exists()
assert fbms_root.exists()
assert not output_json.exists()
log_path = args.output_json + '.log'
setup_logging(log_path)
logging.info('Args: %s' % pformat(vars(args)))
last_frames = set()
sequence_paths = list((fbms_root / 'TrainingSet').iterdir()) + list(
(fbms_root / 'TestSet').iterdir())
for sequence_path in sequence_paths:
images = sorted(
sequence_path.glob('*.jpg'),
key=lambda x: fbms_utils.get_framenumber(x))
last_frames.add(images[-1].stem)
logging.info('Will remove up to %s images: %s' % (len(last_frames),
sorted(last_frames)))
with open(input_path, 'r') as f:
input_data = json.load(f)
valid_images = []
valid_image_ids = set()
for image in input_data['images']:
image_path = Path(image['file_name'])
if image_path.stem not in last_frames:
valid_images.append(image)
valid_image_ids.add(image['id'])
else:
logging.info('Removing image id: %s, path: %s' % (image['id'], image_path))
valid_annotations = []
for annotation in input_data['annotations']:
if annotation['image_id'] in valid_image_ids:
valid_annotations.append(annotation)
logging.info('Removed %s images, %s annotations' %
(len(input_data['images']) - len(valid_images),
len(input_data['annotations']) - len(valid_annotations)))
logging.info('Kept %s images, %s annotations' %
(len(valid_images), len(valid_annotations)))
input_data['images'] = valid_images
input_data['annotations'] = valid_annotations
with open(output_json, 'w') as f:
json.dump(input_data, f)
file_logger = logging.getLogger(log_path)
file_logger.info('Source:')
file_logger.info('=======\n%s', _source)
file_logger.info('=======')
def result_summary(res_dict,
args_dict,
TNR_target=0.05,
skip_pattern=None,
include_pattern='.*',
pvalue_record=None):
from utils.meters import simple_auc
from _collections import OrderedDict
## if not configured setup logging for external caller
if not logging.getLogger('').handlers:
setup_logging()
in_dist = args_dict['dataset']
alphas = args_dict['alphas']
logging.info(f'Report for {args_dict["model"]} - {in_dist}')
logging.info(f'Tag: {args_dict["tag"]}')
result_dict = OrderedDict(
model=args_dict["model"],
in_dist=args_dict['dataset'],
LDA=args_dict.get('LDA'),
joint=args_dict['measure_joint_distribution'],
tag=args_dict['tag'],
channles_sellect=args_dict.get('channel_selection_fn'))
# read indist results to calibrate alpha value for target TNR
rows = []
accuracies = {'model': {}}
for reduction_name, reduction_metrics in res_dict[in_dist].items():
accuracies[reduction_name] = {}
if reduction_name.endswith('_acc'):
acc = reduction_metrics.mean.cpu().numpy()
std = reduction_metrics.std.cpu().numpy()
acc_name = reduction_name.replace('_acc', '')
if acc_name == 'model':
reduction_name = 'model'
if acc_name.endswith('rescaled-smx'):
reduction_name = acc_name[:-13]
acc_name = 'model_rescaled_smx'
elif acc_name.endswith('-pval'):
reduction_name = acc_name[:-5]
acc_name = 'pval'
accuracies[reduction_name][f'{acc_name}_t1'] = acc[0]
accuracies[reduction_name][f'{acc_name}_t5'] = acc[1]
accuracies[reduction_name][f'{acc_name}_std_t1'] = std[0]
for reduction_name, reduction_metrics in res_dict[in_dist].items():
if skip_pattern and bool(re.match(
skip_pattern, reduction_name)) or include_pattern and not bool(
re.match(include_pattern, reduction_name)):
continue
result_dict['reduction'] = reduction_name
result_dict.update(**accuracies['model'])
result_dict.update(**accuracies[reduction_name])
logging.info(reduction_name)
if type(reduction_metrics) != dict:
# report simple metric
logging.info(
f'\t{reduction_metrics.mean}\t({reduction_metrics.std})')
continue
# report reduction specific metrics
for metric_name, meter_object in reduction_metrics.items():
metric_stats = MeterDict()
if not metric_name.endswith('_roc'):
logging.info(
f'\t{metric_name}: {meter_object.mean.numpy():0.3}')
continue
FPR = meter_object.mean.numpy()
calibrated_alpha_id = min((FPR < TNR_target).sum() - 1, len(FPR))
if calibrated_alpha_id == -1:
# all pvalues are larger than alpha
fpr_under_target_alpha = meter_object.mean[0]
interp_alpha = FPR[0]
calibrated_alpha_id = 0
else:
fpr_under_target_alpha = FPR[calibrated_alpha_id]
# actual rejection threshold to use for TNR 95%
interp_alpha = np.interp(0.05, FPR.squeeze(), alphas)
result_dict.update(
dict(metric_name=metric_name,
FPR_strict=fpr_under_target_alpha,
FPR_over=FPR[calibrated_alpha_id + 1],
chosen_alpha=interp_alpha))
logging.info(
f'\t{metric_name} - in-dist rejected: '
# f'alpha-{indist_pvalues_roc[alphas.index(TNR_target)]:0.3f} ({TNR_target:0.3f}), '
f'under-{fpr_under_target_alpha:0.3f} ({alphas[calibrated_alpha_id]:0.3f}), '
f'interp-{TNR_target:0.3f} ({interp_alpha:0.3f}), '
f'over-{FPR[calibrated_alpha_id + 1]:0.3f} ({alphas[calibrated_alpha_id + 1]})'
)
if pvalue_record and reduction_name in pvalue_record[in_dist]:
if metric_name.startswith(
'class_cond'
) and 'predicted_id' in pvalue_record[in_dist]:
predicted_ids = pvalue_record[in_dist]['predicted_id']
in_cc_pval_pred = pvalue_record[in_dist][reduction_name][
th.arange(predicted_ids.shape[0]), predicted_ids]
else:
in_cc_pval_pred = pvalue_record[in_dist][
reduction_name].max(1)[0]
for target_dataset_name, reduction_metrics in res_dict.items():
if target_dataset_name != in_dist and metric_name in reduction_metrics[
reduction_name]:
interp_rejected = np.interp(
interp_alpha, alphas, reduction_metrics[reduction_name]
[metric_name].mean.numpy())
TPR = reduction_metrics[reduction_name][
metric_name].mean.numpy()
raw_rejected = TPR[alphas.index(TNR_target)]
auroc = simple_auc(TPR, FPR)
logging.info(
f'\t\t{target_dataset_name}:\traw-{raw_rejected:0.3f}\tinterp-{interp_rejected:0.3f}\tAUROC:{auroc:0.3f}'
)
if pvalue_record and reduction_name in pvalue_record[
target_dataset_name]:
if metric_name.startswith(
'class_cond'
) and 'predicted_id' in pvalue_record[
target_dataset_name]:
predicted_ids = pvalue_record[target_dataset_name][
'predicted_id']
out_cc_pval_pred = pvalue_record[
target_dataset_name][reduction_name][
th.arange(predicted_ids.shape[0]),
predicted_ids]
else:
out_cc_pval_pred = pvalue_record[
target_dataset_name][reduction_name].max(1)[0]
m = metric(in_cc_pval_pred.numpy(),
out_cc_pval_pred.numpy())
logging.info(f'\t\t\tbenchmark metrics: {m}')
result_dict.update(**m)
result_dict.update(
dict(out_dist=target_dataset_name,
TPR95_raw=raw_rejected,
TPR95_interp=interp_rejected,
AUROC=auroc))
rows.append(result_dict.copy())
if in_dist.startswith(
'cifar') and target_dataset_name.startswith(
'cifar'):
continue
metric_stats.update(
dict(TPR95_raw=th.tensor([raw_rejected]),
TPR95_interp=th.tensor([interp_rejected]),
AUROC=th.tensor([auroc])))
if target_dataset_name != in_dist and metric_name in reduction_metrics[
reduction_name]:
result_dict['out_dist'] = 'avg'
logging.info(
f'\tmetric avg stats: {[k + " " + str(float(v)) for k, v in metric_stats.get_mean_dict().items()]}'
)
result_dict.update(**metric_stats.get_mean_dict())
rows.append(result_dict.copy())
return rows
def initServer(logfilie, mysql_conn):
# print('--------------mysql_conn:%s'%mysql_conn)
db.configure_orm(mysql_conn)
fd = log.setup_logging(logfilie)
#kafka_utils.setup_kafka(config.G_Conf.Common.Broker)
return fd
def post_main():
global args, best_prec1, dtype
best_prec1 = 0
args = parser.parse_args()
dtype = torch_dtypes.get(args.dtype)
torch.manual_seed(args.seed)
time_stamp = datetime.now().strftime('%Y-%m-%d_%H-%M-%S')
if args.evaluate:
args.results_dir = '/tmp'
if args.save is '':
args.save = time_stamp
save_path = os.path.join(args.results_dir, args.save)
if not os.path.exists(save_path):
os.makedirs(save_path)
setup_logging(os.path.join(save_path, 'log.txt'),
resume=args.resume is not '')
results_path = os.path.join(save_path, 'results')
results = ResultsLog(
results_path, title='Training Results - %s' % args.save)
logging.info("saving to %s", save_path)
logging.debug("run arguments: %s", args)
if 'cuda' in args.device and torch.cuda.is_available():
torch.cuda.manual_seed_all(args.seed)
torch.cuda.set_device(args.device_ids[0])
cudnn.benchmark = True
else:
args.device_ids = None
# create model
logging.info("creating model %s", args.model)
model = models.__dict__[args.model]
model_config = {'input_size': args.input_size, 'dataset': args.dataset}
if args.model_config is not '':
model_config = dict(model_config, **literal_eval(args.model_config))
model = model(**model_config)
logging.info("created model with configuration: %s", model_config)
# optionally resume from a checkpoint
if args.evaluate:
if not os.path.isfile(args.evaluate):
parser.error('invalid checkpoint: {}'.format(args.evaluate))
checkpoint = torch.load(args.evaluate)
model.load_state_dict(checkpoint['state_dict'])
logging.info("loaded checkpoint '%s' (epoch %s)",
args.evaluate, checkpoint['epoch'])
elif args.resume:
checkpoint_file = args.resume
if os.path.isdir(checkpoint_file):
results.load(os.path.join(checkpoint_file, 'results.csv'))
checkpoint_file = os.path.join(
checkpoint_file, 'model_best.pth.tar')
if os.path.isfile(checkpoint_file):
logging.info("loading checkpoint '%s'", args.resume)
checkpoint = torch.load(checkpoint_file)
args.start_epoch = checkpoint['epoch'] - 1
best_prec1 = checkpoint['best_prec1']
model.load_state_dict(checkpoint['state_dict'])
logging.info("loaded checkpoint '%s' (epoch %s)",
checkpoint_file, checkpoint['epoch'])
else:
logging.error("no checkpoint found at '%s'", args.resume)
num_parameters = sum([l.nelement() for l in model.parameters()])
logging.info("number of parameters: %d", num_parameters)
# Data loading code
default_transform = {
'train': get_transform(args.dataset,
input_size=args.input_size, augment=True),
'eval': get_transform(args.dataset,
input_size=args.input_size, augment=False)
}
transform = getattr(model, 'input_transform', default_transform)
regime = getattr(model, 'regime', [{'epoch': 0,
'optimizer': args.optimizer,
'lr': args.lr,
'momentum': args.momentum,
'weight_decay': args.weight_decay}])
# define loss function (criterion) and optimizer
criterion = getattr(model, 'criterion', nn.CrossEntropyLoss)()
criterion.to(args.device, dtype)
model.to(args.device, dtype)
val_data = get_dataset(args.dataset, 'val', transform['eval'])
val_loader = torch.utils.data.DataLoader(
val_data,
batch_size=args.batch_size, shuffle=False,
num_workers=args.workers, pin_memory=True)
if args.evaluate:
validate(val_loader, model, criterion, 0)
return
train_data = get_dataset(args.dataset, 'train', transform['train'])
train_loader = torch.utils.data.DataLoader(
train_data,
batch_size=args.batch_size, shuffle=True,
num_workers=args.workers, pin_memory=True)
optimizer = OptimRegime(model.parameters(), regime)
logging.info('training regime: %s', regime)
for i in range(len(get_layer_list(model))):
sketch_layer(model, i)
fix_layer(model, i)
model.cuda()
print(str(i)+'-Sketch: ')
val_loss, val_prec1, val_prec5 = validate(
val_loader, model, criterion, 0)
print('Start Retraining ...')
for epoch in range(1, 200+1):
# train for one epoch
train_loss, train_prec1, train_prec5 = train(
train_loader, model, criterion, epoch, optimizer)
val_loss, val_prec1, val_prec5 = validate(val_loader, model, criterion, 0)
