def find_node(self,
ia_op_exec_context: InputAgnosticOperationExecutionContext,
tensor_metas: List[TensorMeta],
tm_comparators: List[TensorMetaComparator]) -> NNCFNode:
op_exec_context = OperationExecutionContext(
ia_op_exec_context.operator_name,
ia_op_exec_context.scope_in_model,
ia_op_exec_context.call_order,
tensor_metas,
tm_comparators=tm_comparators)
nncf_node_candidates = []
node_candidates = self._find_nodes_with_matching_context_and_inputs(
op_exec_context)
if not node_candidates:
node_candidates = self._find_nodes_with_matching_context_among_inputless(
op_exec_context)
for nx_node in node_candidates.values():
nncf_node_candidates.append(
NNCFNode(nx_node[NNCFGraph.ID_NODE_ATTR], op_exec_context))
result = None
if len(nncf_node_candidates) == 1:
result = nncf_node_candidates[0]
if len(nncf_node_candidates) > 1:
nncf_logger.warning("More than one node matches input")
result = nncf_node_candidates[0]
return result
def apply_minmax_init(self,
min_values,
max_values,
log_module_name: str = None):
if self.initialized:
nncf_logger.debug(
"Skipped initializing {} - loaded from checkpoint".format(
log_module_name))
return
if torch.any(torch.eq(min_values, np.inf)) or torch.any(
torch.eq(max_values, -np.inf)):
raise AttributeError(
'Statistics is not collected for {}'.format(log_module_name))
sign = torch.any(torch.lt(min_values, 0))
if self.signedness_to_force is not None and sign != self.signedness_to_force:
nncf_logger.warning("Forcing signed to {} for module {}".format(
self.signedness_to_force, log_module_name))
sign = self.signedness_to_force
self.signed = int(sign)
abs_max = torch.max(torch.abs(max_values), torch.abs(min_values))
SCALE_LOWER_THRESHOLD = 0.1
self.scale.fill_(SCALE_LOWER_THRESHOLD)
self.scale.masked_scatter_(torch.gt(abs_max, SCALE_LOWER_THRESHOLD),
abs_max)
nncf_logger.info("Set sign: {} and scale: {} for {}".format(
self.signed, get_flat_tensor_contents_string(self.scale),
log_module_name))
def run_batchnorm_adaptation(self, config):
initializer_params = config.get("initializer", {})
init_bn_adapt_config = initializer_params.get('batchnorm_adaptation', {})
num_bn_adaptation_samples = init_bn_adapt_config.get('num_bn_adaptation_samples', 0)
num_bn_forget_samples = init_bn_adapt_config.get('num_bn_forget_samples', 0)
try:
bn_adaptation_args = config.get_extra_struct(BNAdaptationInitArgs)
has_bn_adapt_init_args = True
except KeyError:
has_bn_adapt_init_args = False
if not init_bn_adapt_config:
if has_bn_adapt_init_args:
nncf_logger.warning("Enabling quantization batch norm adaptation with default parameters.")
num_bn_adaptation_samples = 2000
num_bn_forget_samples = 1000
if num_bn_adaptation_samples < 0:
raise AttributeError('Number of adaptation samples must be >= 0')
if num_bn_adaptation_samples > 0:
if not has_bn_adapt_init_args:
nncf_logger.info(
'Could not run batchnorm adaptation '
'as the adaptation data loader is not provided as an extra struct. '
'Refer to `NNCFConfig.register_extra_structs` and the `BNAdaptationInitArgs` class')
return
batch_size = bn_adaptation_args.data_loader.batch_size
num_bn_forget_steps = numpy.ceil(num_bn_forget_samples / batch_size)
num_bn_adaptation_steps = numpy.ceil(num_bn_adaptation_samples / batch_size)
bn_adaptation_runner = DataLoaderBNAdaptationRunner(self._model, bn_adaptation_args.device,
num_bn_forget_steps)
bn_adaptation_runner.run(bn_adaptation_args.data_loader, num_bn_adaptation_steps)
def _apply_minmax_init(self,
min_values,
max_values,
log_module_name: str = None):
if torch.any(torch.eq(min_values, np.inf)) or torch.any(
torch.eq(max_values, -np.inf)):
raise AttributeError(
'Statistics is not collected for {}'.format(log_module_name))
sign = torch.any(torch.lt(min_values, 0))
if self.signedness_to_force is not None and sign != self.signedness_to_force:
nncf_logger.warning("Forcing signed to {} for module {}".format(
self.signedness_to_force, log_module_name))
sign = self.signedness_to_force
self.signed = int(sign)
abs_max = torch.max(torch.abs(max_values), torch.abs(min_values))
SCALE_LOWER_THRESHOLD = 0.1
mask = torch.gt(abs_max, SCALE_LOWER_THRESHOLD)
self._scale_param_storage.data = torch.where(
mask, abs_max,
SCALE_LOWER_THRESHOLD * torch.ones_like(self._scale_param_storage))
if self._is_using_log_scale_storage:
self._scale_param_storage.data.log_()
nncf_logger.info("Set sign: {} and scale: {} for {}".format(
self.signed, get_flat_tensor_contents_string(self.scale),
log_module_name))
def find_node(self,
ia_op_exec_context: InputAgnosticOperationExecutionContext,
tensor_metas: List[TensorMeta],
tm_comparators: List[TensorMetaComparator]) -> NNCFNode:
nncf_node_candidates = []
iter_scopes = self._get_iteration_scopes(
ia_op_exec_context.scope_in_model)
# compare meta information about first input nodes during the matching. During the iteration some nodes may
# change number of inputs, e.g. on concat of hidden outputs
input_matcher = FirstInputsMatcher()
op_exec_context = OperationExecutionContext(
ia_op_exec_context.operator_name,
ia_op_exec_context.scope_in_model,
ia_op_exec_context.call_order,
tensor_metas,
input_matcher=input_matcher,
tm_comparators=tm_comparators)
node_candidates = self._find_nodes_with_matching_context_and_inputs(
op_exec_context)
if not node_candidates:
op_exec_context = OperationExecutionContext(
ia_op_exec_context.operator_name,
ia_op_exec_context.scope_in_model,
ia_op_exec_context.call_order,
tensor_metas,
tm_comparators=tm_comparators)
node_candidates = self._find_nodes_with_matching_context_among_inputless(
op_exec_context)
if not node_candidates and iter_scopes:
# ignore information about node creator and index of input
comparators = tm_comparators + [
ShapeOnlyTensorMetaComparator()
]
op_exec_context = OperationExecutionContext(
ia_op_exec_context.operator_name,
ia_op_exec_context.scope_in_model,
ia_op_exec_context.call_order,
tensor_metas,
tm_comparators=comparators)
# match with starting points of iteration
iter_nodes = self._match_first_iteration_nodes(
op_exec_context, iter_scopes)
for node in iter_nodes.items():
nncf_node_candidates.append(node[1])
for nx_node in node_candidates.values():
nncf_node_candidates.append(
NNCFNode(nx_node[NNCFGraph.ID_NODE_ATTR], op_exec_context))
result = None
if len(nncf_node_candidates) == 1:
result = nncf_node_candidates[0]
if len(nncf_node_candidates) > 1:
nncf_logger.warning("More than one node matches input")
result = nncf_node_candidates[0]
return result
def check_parameter_size(key, saved_value, num_loaded_layers):
saved_size = saved_value.size()
size = state_dict[key].size()
if saved_size == size:
new_dict[key] = saved_value
return num_loaded_layers + 1
nncf_logger.warning("Different size of value of '{}' in dictionary ({}) and in resuming model ({})"
.format(key, saved_size, size, ))
skipped_keys.append(key)
return num_loaded_layers
def __init__(self, sparsity_algo, params=None):
super().__init__(sparsity_algo, params)
self.power = self._params.get('power', 0.9)
self.concave = self._params.get('concave', False)
self._update_per_optimizer_step = self._params.get('update_per_optimizer_step', False)
if self._update_per_optimizer_step:
self._steps_per_epoch = self._params.get('steps_per_epoch')
if self._steps_per_epoch is None:
logger.warning("Optimizer set to update sparsity level per optimizer step,"
"but steps_per_epoch was not set in config. Will only start updating "
"sparsity level after measuring the actual steps per epoch as signaled "
"by a .epoch_step() call.")
def visualize_graph(self, path):
out_graph = self._get_graph_for_visualization()
nx.drawing.nx_pydot.write_dot(out_graph, path)
try:
A = to_agraph(out_graph)
A.layout('dot')
png_path = os.path.splitext(path)[0]+'.png'
A.draw(png_path)
except ImportError:
nncf_logger.warning("Graphviz is not installed - only the .dot model visualization format will be used. "
"Install pygraphviz into your Python environment and graphviz system-wide to enable "
"PNG rendering.")
def get_model_size(self, per_quantizer_bw: Dict[QuantizerId,
int]) -> np.int64:
model_size = 0
for qid, nparam in self._nparam_map.items():
if qid in per_quantizer_bw:
model_size += nparam * per_quantizer_bw[qid]
else:
logger.warning(
"[ModelSizeCalculator] Missing Bitwidth of QID: {}, using {} bits"
.format(str(qid), ModelSizeCalculator.FLOAT_BITWIDTH))
model_size += nparam * ModelSizeCalculator.FLOAT_BITWIDTH
return model_size
def get_class_by_type_name(type_name):
"""
Return class of metaop that corresponds to type_name type.
"""
cls = PRUNING_OPERATOR_METATYPES.get_operator_metatype_by_op_name(
type_name)
if cls is None:
nncf_logger.warning(
"Layer {} is not pruneable - will not propagate pruned filters through it"
.format(type_name))
cls = StopMaskForwardOps
return cls
def check_parameter_size(key, value_to_load, num_loaded_layers):
size_of_value_to_load = value_to_load.size()
size = model_state_dict[key].size()
if size_of_value_to_load == size:
new_dict[key] = value_to_load
return num_loaded_layers + 1
nncf_logger.warning(
"Different size of value of '{}' in resuming dictionary ({}) and in model ({})"
.format(
key,
size_of_value_to_load,
size,
))
skipped_keys.append(key)
return num_loaded_layers
def process_problematic_keys(is_resume, issues, is_all_saved_loaded):
error_msgs = []
def add_error_msg(name, keys):
error_msgs.insert(
0, '{} key(s):\n{}. '.format(name,
',\n'.join('\t\t"{}"'.format(k) for k in keys)))
for name, keys in issues.items():
is_missing = name == 'Missing'
if keys and (not is_missing or is_missing and (is_resume or not is_all_saved_loaded)):
add_error_msg(name, keys)
if error_msgs:
error_msg = 'Error(s) when loading model parameters:\n\t{}'.format("\n\t".join(error_msgs))
if is_resume:
raise RuntimeError(error_msg)
nncf_logger.warning(error_msg)
def epoch_step(self, next_epoch=None):
if self._update_per_optimizer_step:
if self.current_epoch == 0 and self._steps_in_current_epoch > 0 and self._steps_per_epoch is None:
self._steps_per_epoch = self._steps_in_current_epoch
# Reset step and epoch step counters
next_epoch = -1
if self._steps_in_current_epoch != self._steps_per_epoch and self._steps_in_current_epoch > 0:
self._steps_per_epoch = self._steps_in_current_epoch
logger.warning("Actual optimizer steps per epoch is different than what is "
"specified by scheduler parameters! Scheduling may be incorrect. "
"Setting scheduler's global step count to (current epoch) * "
"(actual steps per epoch)")
self.step(self._steps_per_epoch * (self.current_epoch - 1))
self._steps_in_current_epoch = 0
super().epoch_step(next_epoch)
def wrap_inputs(self, model_args, model_kwargs):
bound_model_params = self._fwd_signature.bind(*model_args,
**model_kwargs)
for param_name in self._fwd_params_to_input_infos_odict:
param_kind = self._fwd_signature.parameters[param_name].kind
if param_kind is Parameter.VAR_POSITIONAL or param_kind is Parameter.VAR_KEYWORD:
nncf_logger.warning(
"An input_info tensor was bound to a *args or **kwargs variadic parameter in the"
"forward's signature! This is currently unsupported by NNCF. Input compression may "
"be incorrect.")
# Currently won't support input info mapping to *args or **kwargs-mapped parameters
continue
if param_name not in bound_model_params.arguments:
nncf_logger.warning(
"A call to a compressed model's forward occured without one of the params"
"specified in input_infos! Input compression may be incorrect. Trying to recover "
"by wrapping the default value for the parameter.")
bound_model_params.apply_defaults()
potential_tensor = bound_model_params.arguments[param_name]
if potential_tensor is not None:
bound_model_params.arguments[param_name] = nncf_model_input(
bound_model_params.arguments[param_name])
else:
# Default was None - cannot wrap as-is. Will wrap a dummy tensor as specified in
# input infos - will conserve the call order of nncf_model_input nodes,
# and the post-hooks for the input node will execute. The result won't go anywhere, though.
nncf_logger.warning(
"Wrapping a dummy tensor for input {}".format(
'param_name'))
info_for_missing_input = self._fwd_params_to_input_infos_odict[
param_name]
device = 'cuda'
if self._module_ref_for_device is not None:
device = next(
self._module_ref_for_device.parameters()).device
dummy_tensor = create_mock_tensor(info_for_missing_input,
device)
_ = nncf_model_input(dummy_tensor)
return bound_model_params.args, bound_model_params.kwargs
def apply_init(self) -> SingleConfigQuantizerSetup:
from nncf.automl.environment.quantization_env import QuantizationEnv
from nncf.automl.agent.ddpg.ddpg import DDPG
from nncf.debug import DEBUG_LOG_DIR
if self._dump_autoq_data or is_debug():
dump_dir = self._init_args.config.get('log_dir', None)
if dump_dir is None:
dump_dir = DEBUG_LOG_DIR
self.dump_dir = Path(dump_dir) / Path("autoq_agent_dump")
self.dump_dir.mkdir(parents=True, exist_ok=True)
self.policy_dict = OrderedDict() #key: episode
self.best_policy_dict = OrderedDict() #key: episode
self._init_args.config['episodic_nncfcfg'] = self.dump_dir / "episodic_nncfcfg"
os.makedirs(self._init_args.config['episodic_nncfcfg'], exist_ok=True)
try:
from torch.utils.tensorboard import SummaryWriter
self.tb_writer = SummaryWriter(self.dump_dir)
# log compression config to tensorboard
self.tb_writer.add_text('AutoQ/run_config',
json.dumps(self._init_args.config['compression'],
indent=4, sort_keys=False).replace("\n", "\n\n"), 0)
except ModuleNotFoundError:
logger.warning("Tensorboard installation not found! Install tensorboard Python package "
"in order for AutoQ tensorboard statistics data to be dumped")
start_ts = datetime.now()
from nncf.automl.environment.quantization_env import QuantizationEnvParams
env_params = QuantizationEnvParams(compression_ratio=self._params.compression_ratio,
eval_subset_ratio=self._params.eval_subset_ratio,
skip_constraint=self._params.skip_constraint,
finetune=self._params.finetune,
bits=self._params.bits,
dump_init_precision_data=self._dump_autoq_data,
log_dir=Path(DEBUG_LOG_DIR) / Path("autoq"))
# Instantiate Quantization Environment
env = QuantizationEnv(
self._model,
self.quantization_controller,
self._hw_precision_constraints,
self._init_args.data_loader,
self._init_args.eval_fn,
hw_config_type=self._hw_cfg_type,
params=env_params)
nb_state = len(env.state_list)
nb_action = 1
# Instantiate Automation Agent
agent = DDPG(nb_state, nb_action, self._iter_number, hparam_override=self._ddpg_hparams_override)
if self._dump_autoq_data and self.tb_writer is not None:
self.tb_writer.add_text('AutoQ/state_embedding', env.master_df[env.state_list].to_markdown())
best_policy, best_reward = self._search(agent, env)
end_ts = datetime.now()
final_qid_vs_qconfig_map = env.select_config_for_actions(best_policy)
final_quantizer_setup = self.quantization_controller.get_quantizer_setup_for_current_state()
for qp_id, qconf in final_qid_vs_qconfig_map.items():
final_quantizer_setup.quantization_points[qp_id].qconfig = qconf
logger.info('[AutoQ] best_reward: {}'.format(best_reward))
logger.info('[AutoQ] best_policy: {}'.format(best_policy))
logger.info("[AutoQ] Search Complete")
logger.info("[AutoQ] Elapsed time of AutoQ Precision Initialization (): {}".format(end_ts-start_ts))
return final_quantizer_setup
