def dict_config(model, optimizer, sched_dict, scheduler=None):
app_cfg_logger.debug('Schedule contents:\n' +
json.dumps(sched_dict, indent=2))
if scheduler is None:
scheduler = distiller.CompressionScheduler(model)
pruners = __factory('pruners', model, sched_dict)
regularizers = __factory('regularizers', model, sched_dict)
quantizers = __factory('quantizers',
model,
sched_dict,
optimizer=optimizer)
if len(quantizers) > 1:
raise ValueError("\nError: Multiple Quantizers not supported")
extensions = __factory('extensions', model, sched_dict)
try:
lr_policies = []
for policy_def in sched_dict['policies']:
policy = None
if 'pruner' in policy_def:
try:
instance_name, args = __policy_params(policy_def, 'pruner')
except TypeError as e:
print(
'\n\nFatal Error: a policy is defined with a null pruner'
)
print(
'Here\'s the policy definition for your reference:\n{}'
.format(json.dumps(policy_def, indent=1)))
raise
assert instance_name in pruners, "Pruner {} was not defined in the list of pruners".format(
instance_name)
pruner = pruners[instance_name]
policy = distiller.PruningPolicy(pruner, args)
elif 'regularizer' in policy_def:
instance_name, args = __policy_params(policy_def,
'regularizer')
assert instance_name in regularizers, "Regularizer {} was not defined in the list of regularizers".format(
instance_name)
regularizer = regularizers[instance_name]
if args is None:
policy = distiller.RegularizationPolicy(regularizer)
else:
policy = distiller.RegularizationPolicy(
regularizer, **args)
elif 'quantizer' in policy_def:
instance_name, args = __policy_params(policy_def, 'quantizer')
assert instance_name in quantizers, "Quantizer {} was not defined in the list of quantizers".format(
instance_name)
quantizer = quantizers[instance_name]
policy = distiller.QuantizationPolicy(quantizer)
elif 'lr_scheduler' in policy_def:
# LR schedulers take an optimizer in their CTOR, so postpone handling until we're certain
# a quantization policy was initialized (if exists)
lr_policies.append(policy_def)
continue
elif 'extension' in policy_def:
instance_name, args = __policy_params(policy_def, 'extension')
assert instance_name in extensions, "Extension {} was not defined in the list of extensions".format(
instance_name)
extension = extensions[instance_name]
policy = extension
else:
raise ValueError(
"\nFATAL Parsing error while parsing the pruning schedule - unknown policy [%s]"
.format(policy_def))
add_policy_to_scheduler(policy, policy_def, scheduler)
# Any changes to the optmizer caused by a quantizer have occured by now, so safe to create LR schedulers
lr_schedulers = __factory('lr_schedulers',
model,
sched_dict,
optimizer=optimizer)
for policy_def in lr_policies:
instance_name, args = __policy_params(policy_def, 'lr_scheduler')
assert instance_name in lr_schedulers, "LR-scheduler {} was not defined in the list of lr-schedulers".format(
instance_name)
lr_scheduler = lr_schedulers[instance_name]
policy = distiller.LRPolicy(lr_scheduler)
add_policy_to_scheduler(policy, policy_def, scheduler)
except AssertionError:
# propagate the assertion information
raise
except Exception as exception:
print("\nFATAL Parsing error!\n%s" % json.dumps(policy_def, indent=1))
print("Exception: %s %s" % (type(exception), exception))
raise
return scheduler
def dictConfig(model, optimizer, schedule, sched_dict, logger):
logger.debug(json.dumps(sched_dict, indent=1))
pruners = __factory('pruners', model, sched_dict)
regularizers = __factory('regularizers', model, sched_dict)
lr_schedulers = __factory('lr_schedulers',
model,
sched_dict,
optimizer=optimizer)
extensions = __factory('extensions', model, sched_dict)
try:
for policy_def in sched_dict['policies']:
policy = None
if 'pruner' in policy_def:
try:
instance_name, args = __policy_params(policy_def, 'pruner')
except TypeError as e:
print(
'\n\nFatal Error: a policy is defined with a null pruner'
)
print(
'Here\'s the policy definition for your reference:\n{}'
.format(json.dumps(policy_def, indent=1)))
exit(1)
assert instance_name in pruners, "Pruner {} was not defined in the list of pruners".format(
instance_name)
pruner = pruners[instance_name]
policy = distiller.PruningPolicy(pruner, args)
elif 'regularizer' in policy_def:
instance_name, args = __policy_params(policy_def,
'regularizer')
assert instance_name in regularizers, "Regularizer {} was not defined in the list of regularizers".format(
instance_name)
regularizer = regularizers[instance_name]
if args is None:
policy = distiller.RegularizationPolicy(regularizer)
else:
policy = distiller.RegularizationPolicy(
regularizer, **args)
elif 'lr_scheduler' in policy_def:
instance_name, args = __policy_params(policy_def,
'lr_scheduler')
assert instance_name in lr_schedulers, "LR-scheduler {} was not defined in the list of lr-schedulers".format(
instance_name)
lr_scheduler = lr_schedulers[instance_name]
policy = distiller.LRPolicy(lr_scheduler)
elif 'extension' in policy_def:
instance_name, args = __policy_params(policy_def, 'extension')
assert instance_name in extensions, "Extension {} was not defined in the list of extensions".format(
instance_name)
extension = extensions[instance_name]
policy = extension
else:
print(
"\nFATAL Parsing error while parsing the pruning schedule - unknown policy [%s]"
% policy_def)
exit(1)
if 'epochs' in policy_def:
schedule.add_policy(policy, epochs=policy_def['epochs'])
else:
schedule.add_policy(
policy,
starting_epoch=policy_def['starting_epoch'],
ending_epoch=policy_def['ending_epoch'],
frequency=policy_def['frequency'])
except AssertionError:
# propagate the assertion information
raise
except Exception as exception:
print("\nFATAL Parsing error!\n%s" % json.dumps(policy_def, indent=1))
print("Exception: %s %s" % (type(exception), exception))
exit(1)
return schedule
def objective(space):
global model
global count
global global_min_score
#Explore new model
model = create_model(False, args.dataset, args.arch, device_ids=args.gpus)
count += 1
# Objective function: F(Acc, Lat) = (1 - Acc.) + (alpha * Sparsity)
accuracy = 0
alpha = 0.3 # Super-parameter: the importance of inference time
latency = 0.0
sparsity = 0.0
# Training hyperparameter
if args.resume:
model, compression_scheduler, start_epoch = apputils.load_checkpoint(
model, chkpt_file=args.resume)
print('resume mode: {}'.format(args.resume))
print(global_min_score)
criterion = nn.CrossEntropyLoss().cuda()
optimizer = torch.optim.SGD(model.parameters(), lr=args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
"""
distiller/distiller/config.py
# Element-wise sparsity
sparsity_levels = {net_param: sparsity_level}
pruner = distiller.pruning.SparsityLevelParameterPruner(name='sensitivity', levels=sparsity_levels)
policy = distiller.PruningPolicy(pruner, pruner_args=None)
scheduler = distiller.CompressionScheduler(model)
scheduler.add_policy(policy, epochs=[0, 2, 4])
# Local search
add multiple pruner for each layer
"""
sparsity_levels = {}
for key, value in space.items():
sparsity_levels[key] = value
#print(sparsity_levels)
pruner = distiller.pruning.SparsityLevelParameterPruner(name='sensitivity', levels=sparsity_levels) # for SparsityLevelParameterPruner
# pruner = distiller.pruning.SensitivityPruner(name='sensitivity', sensitivities=sparsity_levels) # for SensitivityPruner
policy = distiller.PruningPolicy(pruner, pruner_args=None)
lrpolicy = distiller.LRPolicy(torch.optim.lr_scheduler.StepLR(optimizer, step_size=6, gamma=0.1))
compression_scheduler = distiller.CompressionScheduler(model)
compression_scheduler.add_policy(policy, epochs=[PrunerEpoch])
# compression_scheduler.add_policy(policy, starting_epoch=0, ending_epoch=38, frequency=2)
compression_scheduler.add_policy(lrpolicy, starting_epoch=0, ending_epoch=50, frequency=1)
"""
distiller/example/classifier_compression/compress_classifier.py
For each epoch:
compression_scheduler.on_epoch_begin(epoch)
train()
save_checkpoint()
compression_scheduler.on_epoch_end(epoch)
train():
For each training step:
compression_scheduler.on_minibatch_begin(epoch)
output = model(input)
loss = criterion(output, target)
compression_scheduler.before_backward_pass(epoch)
loss.backward()
optimizer.step()
compression_scheduler.on_minibatch_end(epoch)
"""
local_min_score = 2.
for i in range(args.epochs):
compression_scheduler.on_epoch_begin(i)
train_accuracy = train(i,criterion, optimizer, compression_scheduler)
val_accuracy = validate() # Validate hyperparameter setting
t, sparsity = distiller.weights_sparsity_tbl_summary(model, return_total_sparsity=True)
compression_scheduler.on_epoch_end(i, optimizer)
apputils.save_checkpoint(i, args.arch, model, optimizer, compression_scheduler, train_accuracy, False,
'hyperopt', './')
print('Epoch: {}, train_acc: {:.4f}, val_acc: {:.4f}, sparsity: {:.4f}'.format(i, train_accuracy, val_accuracy, sparsity))
score = (1-(val_accuracy/100.)) + (alpha * (1-sparsity/100.)) # objective funtion here
if(score < global_min_score):
global_min_score = score
apputils.save_checkpoint(i, args.arch, model, optimizer, compression_scheduler, train_accuracy, True, 'best', './')
if(score < local_min_score):
local_min_score = score
if (PrunerConstraint == True and i >= PrunerEpoch and (sparsity < Expected_Sparsity_Level_Low or sparsity > Expected_Sparsity_Level_High)):
break
test_accuracy = test() # Validate hyperparameter setting
print('{} trials: score: {:.4f}, train_acc:{:.4f}, val_acc:{:.4f}, test_acc:{:.4f}, sparsity:{:.4f}'.format(count,
local_min_score,
train_accuracy,
val_accuracy,
test_accuracy,
sparsity))
return local_min_score
def objective(space):
global model
global count
global best_dict
#Explore new model
model = create_model(False, args.dataset, args.arch, device_ids=args.gpus)
if args.resume:
model, _, _ = apputils.load_checkpoint(
model, chkpt_file=args.resume)
count += 1
print('{} trial starting...'.format(count))
# Objective function: F(Acc, Lat) = (1 - Acc.) + (alpha * Sparsity)
accuracy = 0
#alpha = 0.2 # Super-parameter: the importance of inference time
alpha = 1.0 # Super-parameter: the importance of inference time
sparsity = 0.0
# Training hyperparameter
criterion = nn.CrossEntropyLoss().cuda()
optimizer = torch.optim.SGD(model.parameters(), lr=args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
"""
distiller/distiller/config.py
# Element-wise sparsity
sparsity_levels = {net_param: sparsity_level}
pruner = distiller.pruning.SparsityLevelParameterPruner(name='sensitivity', levels=sparsity_levels)
policy = distiller.PruningPolicy(pruner, pruner_args=None)
scheduler = distiller.CompressionScheduler(model)
scheduler.add_policy(policy, epochs=[0, 2, 4])
# Local search
add multiple pruner for each layer
"""
sparsity_levels = {}
for key, value in space.items():
sparsity_levels[key] = value
pruner = distiller.pruning.SparsityLevelParameterPruner(name='sensitivity', levels=sparsity_levels)
policy = distiller.PruningPolicy(pruner, pruner_args=None)
lrpolicy = distiller.LRPolicy(torch.optim.lr_scheduler.StepLR(optimizer, step_size=30, gamma=0.1))
compression_scheduler = distiller.CompressionScheduler(model)
#compression_scheduler.add_policy(policy, epochs=[90])
compression_scheduler.add_policy(policy, epochs=[0])
compression_scheduler.add_policy(lrpolicy, starting_epoch=0, ending_epoch=90, frequency=1)
"""
distiller/example/classifier_compression/compress_classifier.py
For each epoch:
compression_scheduler.on_epoch_begin(epoch)
train()
save_checkpoint()
compression_scheduler.on_epoch_end(epoch)
train():
For each training step:
compression_scheduler.on_minibatch_begin(epoch)
output = model(input)
loss = criterion(output, target)
compression_scheduler.before_backward_pass(epoch)
loss.backward()
optimizer.step()
compression_scheduler.on_minibatch_end(epoch)
"""
for i in range(args.epochs):
compression_scheduler.on_epoch_begin(i)
train_accuracy = train(i,criterion, optimizer, compression_scheduler)
val_accuracy = validate() # Validate hyperparameter setting
t, sparsity = distiller.weights_sparsity_tbl_summary(model, return_total_sparsity=True)
compression_scheduler.on_epoch_end(i, optimizer)
apputils.save_checkpoint(i, args.arch, model, optimizer, compression_scheduler, train_accuracy, False,
'hyperopt', './')
print('{} epochs => train acc:{:.2f}%, val acc:{:.2f}%'.format(i, train_accuracy, val_accuracy))
test_accuracy = validate(test_loader) # Validate hyperparameter setting
#score = (1-(val_accuracy/100.)) + (alpha * (1-sparsity/100.)) # objective funtion here
# objective funtion here
# accuracy: 98~90%, sparsity: 80%~50%
score = -((val_accuracy/100.)**2-0.9**2 + alpha * ((sparsity/100.)**2-0.5**2))
print('{} trials: score: {:.2f}\ttrain acc:{:.2f}%\tval acc:{:.2f}%\ttest acc:{:.2f}%\tsparsity:{:.2f}%'.format(count,
score,
train_accuracy,
val_accuracy,
test_accuracy,
sparsity))
if score < best_dict['score']:
best_dict['trial'] = count
best_dict['score'] = score
best_dict['tr_acc'] = train_accuracy
best_dict['v_acc'] = val_accuracy
best_dict['te_acc'] = test_accuracy
best_dict['sparsity'] = sparsity
best_dict['model_best'] = copy.deepcopy(model)
return score
