def add_optimizer(model):
stepsz = int(30 * args.epoch_size / total_batch_size / num_shards)
if args.float16_compute:
# TODO: merge with multi-precision optimizer
opt = optimizer.build_fp16_sgd(
model,
args.base_learning_rate,
momentum=0.9,
nesterov=1,
weight_decay=args.weight_decay, # weight decay included
policy="step",
stepsize=stepsz,
gamma=0.1
)
else:
optimizer.add_weight_decay(model, args.weight_decay)
opt = optimizer.build_multi_precision_sgd(
model,
args.base_learning_rate,
momentum=0.9,
nesterov=1,
policy="step",
stepsize=stepsz,
gamma=0.1
)
return opt
def add_optimizer(model):
stepsz = int(30 * args.epoch_size / total_batch_size / num_shards)
if args.float16_compute:
# TODO: merge with multi-prceision optimizer
opt = optimizer.build_fp16_sgd(
model,
args.base_learning_rate,
momentum=0.9,
nesterov=1,
weight_decay=args.weight_decay, # weight decay included
policy="step",
stepsize=stepsz,
gamma=0.1
)
else:
optimizer.add_weight_decay(model, args.weight_decay)
opt = optimizer.build_multi_precision_sgd(
model,
args.base_learning_rate,
momentum=0.9,
nesterov=1,
policy="step",
stepsize=stepsz,
gamma=0.1
)
return opt
def add_optimizer(model):
# stepsz = int(30 * args.epoch_size / total_batch_size / num_shards)
stepsz = 1
if args.dtype == 'float16':
opt = optimizer.build_fp16_sgd(
model,
args.base_learning_rate,
momentum=0.9,
nesterov=1,
weight_decay=args.weight_decay, # weight decay included
policy="step",
stepsize=stepsz,
gamma=0.9999
)
else:
optimizer.add_weight_decay(model, args.weight_decay)
opt = optimizer.build_multi_precision_sgd(
model,
args.base_learning_rate,
momentum=0.9,
nesterov=1,
policy="step",
stepsize=stepsz,
gamma=0.9999
)
return opt
def build_optimizer(model, float16_compute = False):
if False: # float16_compute: # A newwer versions of Caffe support this
print("[INFO] Building FP16 SGD optimizer.")
opt = optimizer.build_fp16_sgd(
model, 0.1, momentum=0.9, policy='step', gamma=0.1, weight_decay=1e-4
)
else:
print("[INFO] Building Multi-precision SGD optimizer.")
optimizer.add_weight_decay(model, 1e-4)
#opt = optimizer.build_sgd(
opt = optimizer.build_multi_precision_sgd(
model, 0.1, momentum=0.9, policy='fixed', gamma=0.1
)
return opt
def build_optimizer(model, float16_compute=False):
if False: # float16_compute: # A newwer versions of Caffe support this
print("[INFO] Building FP16 SGD optimizer.")
opt = optimizer.build_fp16_sgd(
model, 0.1, momentum=0.9, policy='step', gamma=0.1, weight_decay=1e-4
)
else:
optimizer.add_weight_decay(model, 1e-4)
try:
opt_builder = optimizer.build_multi_precision_sgd
print("[INFO] Building Multi-precision SGD optimizer.")
except AttributeError:
opt_builder = optimizer.build_sgd
print("[INFO] Building SGD optimizer (Multi-precision SGD is not available).")
opt = opt_builder(
model, 0.1, momentum=0.9, policy='fixed', gamma=0.1
)
return opt
