def add_optimizer(model): stepsz = int(30 * args.epoch_size / total_batch_size / num_shards) if args.float16_compute: # TODO: merge with multi-precision optimizer opt = optimizer.build_fp16_sgd( model, args.base_learning_rate, momentum=0.9, nesterov=1, weight_decay=args.weight_decay, # weight decay included policy="step", stepsize=stepsz, gamma=0.1 ) else: optimizer.add_weight_decay(model, args.weight_decay) opt = optimizer.build_multi_precision_sgd( model, args.base_learning_rate, momentum=0.9, nesterov=1, policy="step", stepsize=stepsz, gamma=0.1 ) return opt
def add_optimizer(model): stepsz = int(30 * args.epoch_size / total_batch_size / num_shards) if args.float16_compute: # TODO: merge with multi-prceision optimizer opt = optimizer.build_fp16_sgd( model, args.base_learning_rate, momentum=0.9, nesterov=1, weight_decay=args.weight_decay, # weight decay included policy="step", stepsize=stepsz, gamma=0.1 ) else: optimizer.add_weight_decay(model, args.weight_decay) opt = optimizer.build_multi_precision_sgd( model, args.base_learning_rate, momentum=0.9, nesterov=1, policy="step", stepsize=stepsz, gamma=0.1 ) return opt
def add_optimizer(model): # stepsz = int(30 * args.epoch_size / total_batch_size / num_shards) stepsz = 1 if args.dtype == 'float16': opt = optimizer.build_fp16_sgd( model, args.base_learning_rate, momentum=0.9, nesterov=1, weight_decay=args.weight_decay, # weight decay included policy="step", stepsize=stepsz, gamma=0.9999 ) else: optimizer.add_weight_decay(model, args.weight_decay) opt = optimizer.build_multi_precision_sgd( model, args.base_learning_rate, momentum=0.9, nesterov=1, policy="step", stepsize=stepsz, gamma=0.9999 ) return opt
def build_optimizer(model, float16_compute = False): if False: # float16_compute: # A newwer versions of Caffe support this print("[INFO] Building FP16 SGD optimizer.") opt = optimizer.build_fp16_sgd( model, 0.1, momentum=0.9, policy='step', gamma=0.1, weight_decay=1e-4 ) else: print("[INFO] Building Multi-precision SGD optimizer.") optimizer.add_weight_decay(model, 1e-4) #opt = optimizer.build_sgd( opt = optimizer.build_multi_precision_sgd( model, 0.1, momentum=0.9, policy='fixed', gamma=0.1 ) return opt
def build_optimizer(model, float16_compute=False): if False: # float16_compute: # A newwer versions of Caffe support this print("[INFO] Building FP16 SGD optimizer.") opt = optimizer.build_fp16_sgd( model, 0.1, momentum=0.9, policy='step', gamma=0.1, weight_decay=1e-4 ) else: optimizer.add_weight_decay(model, 1e-4) try: opt_builder = optimizer.build_multi_precision_sgd print("[INFO] Building Multi-precision SGD optimizer.") except AttributeError: opt_builder = optimizer.build_sgd print("[INFO] Building SGD optimizer (Multi-precision SGD is not available).") opt = opt_builder( model, 0.1, momentum=0.9, policy='fixed', gamma=0.1 ) return opt