def do_astats(self, args: argparse.Namespace):
"""
Calculate activation statistics on one or more input files."""
self._check_graph()
input_args = self._get_input_args(args)
stats_collector = ActivationStatsCollector()
step_idx = args.step
if step_idx is not None:
if len(step_idx) == 1:
step_idx = step_idx[0]
else:
step_idx = tuple(step_idx)
if len(args.input_files) == 0:
self.perror("You must enter some files to process")
return
for file_per_input in glob_input_files(args.input_files,
self.G.num_inputs):
LOG.info("input file %s", file_per_input)
data = [
import_data(input_file, **input_args)
for input_file in file_per_input
]
stats_collector.collect_stats(self.G, data)
fmt = ('tab' if args.output is None else args.output['fmt'])
tab = ActivationReporter(do_totals=(fmt != "csv"),
threshold=args.qsnr,
yield_fusions=args.detail
or isinstance(step_idx, tuple)).report(
self.G, stats_collector.reduce_stats())
output_table(tab, args)
def do_qerror(self, args):
"""
Show quantization error introduced by processing one or more input files."""
self._check_graph()
self._check_quantized()
fmt = ('tab' if args.output is None else args.output['fmt'])
input_args = self._get_input_args(args)
if args.step:
stats_collector = StepErrorStatsCollector(
quant_compare=args.compare_quantized)
else:
stats_collector = ErrorStatsCollector(
quant_compare=args.compare_quantized)
cnt = 0
for file_per_input in glob_input_files(args.input_files,
self.G.num_inputs):
cnt += 1
data = [
import_data(input_file, **input_args)
for input_file in file_per_input
]
stat = stats_collector.collect_stats(self.G, data)
if args.report_lowest is not None:
lowest = min((elem['qsnr'] for elem in stat.values()))
if lowest < args.report_lowest:
self.pfeedback(
"{} had QSNR below threshold".format(file_per_input))
if not cnt:
self.perror("no files to process")
return
tab = ErrorReporter(do_totals=(fmt != "csv"), one_input=cnt <= 1, with_chan=args.step)\
.report(self.G, stats_collector.reduce_stats())
output_table(tab, args)
def do_bcorr(self, args):
"""
Correct biases with average quantization error."""
self._check_graph()
self._check_quantized()
stats_collector = StepErrorStatsCollector()
input_args = self._get_input_args(args)
cnt = 0
for file_per_input in glob_input_files(args.input_files, self.G.num_inputs):
cnt += 1
data = [import_data(filename, **input_args) for filename in file_per_input]
stats_collector.collect_stats(self.G, data)
adjust_biases(self.G, stats_collector.reduce_stats())
def do_aquant(self, args: argparse.Namespace):
"""
Attempt to calculate quantization for graph using one or more sample input files."""
self._check_graph()
stats_collector = ActivationRangesCollector()
# if replaying state file then load the activation stats if they are present
if args.scheme == 'SQ8':
bits = 8
else:
bits = args.force_width
if self.replaying_history and self.history_stats:
astats = self.history_stats
else:
input_args = self._get_input_args(args)
processed_input = False
for file_per_input in glob_input_files(args.input_files,
self.G.num_inputs):
LOG.info("input file %s", file_per_input)
processed_input = True
data = [
import_data(input_file, **input_args)
for input_file in file_per_input
]
stats_collector.collect_stats(self.G, data)
if not processed_input:
self.perror("No input files found")
return
astats = stats_collector.stats
self._record_stats(astats)
quantizer = UnifiedQuantizer(args.scheme,
astats,
quantized_dimension=args.quant_dimension,
narrow_weights=not args.no_narrow_weights,
bits=bits)
qrecs = quantizer.quantize(self.G)
self.G.quantization = qrecs
# These should now be unnecessary
# if args.scheme == 'SQ8':
# concats_matcher = EqualizeSymmetricMultiplicativeQuantivedConcats()
# concats_matcher.match(self.G, set_identity=False)
# rnns_matcher = PropagateUpRNNInputQ()
# rnns_matcher.match(self.G, set_identity=False)
# softmax_qrec_matcher = PropagateSoftmaxSymQrec()
# softmax_qrec_matcher.match(self.G, set_identity=False)
# sig_swish_qrec_matcher = PropagateUpSigSwishInputQ()
# sig_swish_qrec_matcher.match(self.G, set_identity=False)
LOG.info("Quantization set. Use qshow command to see it.")
def __init__(self,
graph,
input_files,
validation,
input_args,
start_qsnr=30,
prediction_step_idx=-1,
min_step=0.5,
base_inputs=None) -> None:
self._graph = graph
self._input_files = list(
glob_input_files(input_files, self._graph.num_inputs))
self._validation = validation
self._start_qsnr = start_qsnr
self._input_args = input_args
self._prediction_step_idx = prediction_step_idx
self._min_step = min_step
self._base_inputs = base_inputs
def do_aquant(self, args: argparse.Namespace):
"""
Attempt to calculate quantization for graph using one or more sample input files."""
self._check_graph()
input_args = self._get_input_args(args)
processed_input = False
stats_collector = ACTIVATION_STATS[args.scheme]()
for file_per_input in glob_input_files(args.input_files,
self.G.num_inputs):
LOG.info("input file %s", file_per_input)
processed_input = True
data = [
import_data(input_file, **input_args)
for input_file in file_per_input
]
stats_collector.collect_stats(self.G, data)
if not processed_input:
self.perror("No input files found")
return
if args.scheme == 'SQ8':
astats = stats_collector.stats
quantizer = MultQuantizer(
astats,
8,
quantized_dimension=args.quant_dimension,
narrow_weights=not args.no_narrow_weights)
else:
astats = stats_collector.reduce_stats()
stats_collector = FilterStatsCollector()
fstats = stats_collector.collect_stats(self.G)
quantizer = SymmetricQuantizer(astats,
fstats,
force_width=args.force_width,
min_qsnr=args.qsnr)
qrecs = quantizer.quantize(self.G)
self.G.quantization = qrecs
if args.scheme == 'SQ8':
concats_matcher = EqualizeSymmetricMultiplicativeQuantivedConcats()
concats_matcher.match(self.G, set_identity=False)
rnns_matcher = PropagateUpRNNInputQ()
rnns_matcher.match(self.G, set_identity=False)
softmax_qrec_matcher = PropagateSoftmaxSymQrec()
softmax_qrec_matcher.match(self.G, set_identity=False)
LOG.info("Quantization set. Use qshow command to see it.")
def do_aquant(self, args: argparse.Namespace):
"""
Attempt to calculate quantization for graph using one or more sample input files."""
self._check_graph()
stats_collector = ActivationRangesCollector()
# if replaying state file then load the activation stats if they are present
opts = get_options_from_args(args)
state = ConstantInputParameters.save_compression_state(self.G)
try:
if self.replaying_history and self.history_stats:
astats = self.history_stats
else:
input_args = self._get_input_args(args)
processed_input = False
for file_per_input in glob_input_files(args.input_files,
self.G.num_inputs):
LOG.info("input file %s", file_per_input)
processed_input = True
data = [
import_data(input_file, **input_args)
for input_file in file_per_input
]
stats_collector.collect_stats(self.G, data)
if not processed_input:
self.perror("No input files found")
return
astats = stats_collector.stats
self._record_stats(astats)
if args.force_width:
opts['bits'] = args.force_width
quantizer = NewQuantizer(self.G, reset_all=True)
quantizer.schemes.append(args.scheme)
quantizer.set_stats(astats, opts)
quantizer.quantize()
self.G.add_dimensions()
LOG.info("Quantization set. Use qshow command to see it.")
finally:
ConstantInputParameters.restore_compression_state(self.G, state)
def do_aquant(self, args: argparse.Namespace):
"""
Attempt to calculate quantization for graph using one or more sample input files."""
self._check_graph()
stats_collector = ActivationRangesCollector()
# if replaying state file then load the activation stats if they are present
opts = get_options_from_args(args)
if self.replaying_history and self.history_stats:
astats = self.history_stats
else:
input_args = self._get_input_args(args)
processed_input = False
for file_per_input in glob_input_files(args.input_files,
self.G.num_inputs):
LOG.info("input file %s", file_per_input)
processed_input = True
data = [
import_data(input_file, **input_args)
for input_file in file_per_input
]
stats_collector.collect_stats(self.G, data)
if not processed_input:
self.perror("No input files found")
return
astats = stats_collector.stats
self._record_stats(astats)
if args.force_width:
opts['bits'] = args.force_width
quantizer = UnifiedQuantizer(args.scheme, astats, **opts)
# clear the existing quantization
self.G.quantization = None
qrecs = quantizer.quantize(self.G)
self.G.quantization = qrecs
RemoveUnnecessaryQuantizeOperators().match(self.G)
self.G.add_dimensions()
LOG.info("Quantization set. Use qshow command to see it.")
def do_validate(self, args: argparse.Namespace):
"""
Validate the model (quantized [-q] or not) in terms of prediction accuracy rate on a given dataset (images
folder). Ground truth labels can be embedded in files names ("filename_03.[png, ppm, pgm]", the number of
digits must be coherent with the number of networks outputs: e.g. in a 1000 classes problem the last digits
must be 3, "file_45.png" will raise an error) or can be written in a .json object (example: {'file0':label0,
'file1':label1, ...}) and given to the function with --label_json
"""
self._check_graph()
if args.quantize:
self._check_quantized()
qmode = QuantizationMode.all_dequantize()
else:
qmode = QuantizationMode.none()
LOG.info("quantization mode - %s", qmode)
input_args = self._get_input_args(args)
good_predictions = []
good_margin = 0
bad_margin = 0
number_samples = sum(1 for _ in glob_input_files(args.input_files))
if args.vww_instances_file:
validation = ValidateFromVWWInstances(
args.vww_instances_file,
class_thr=args.class_thr,
binary_classification=args.binary_classification)
elif args.label_json:
validation = ValidateFromJSON(
args.label_json,
class_thr=args.class_thr,
binary_classification=args.binary_classification)
elif args.class_number is not None:
validation = ValidateFromClass(
args.class_number,
class_thr=args.class_thr,
binary_classification=args.binary_classification)
else:
validation = ValidateFromName(
class_thr=args.class_thr,
binary_classification=args.binary_classification)
try:
ExecutionProgress.start()
for i, file_per_input in enumerate(
glob_input_files(args.input_files, self.G.num_inputs)):
if not args.silent:
LOG.info("input file %s", file_per_input)
data = [
import_data(input_file, **input_args)
for input_file in file_per_input
]
executer = GraphExecuter(self.G, qrecs=self.G.quantization)
outputs = executer.execute(data,
qmode=qmode,
silent=args.silent)
predicted_values = np.asarray(
outputs[args.prediction_step_idx])
good_prediction, class_predicted, real_class, margin = validation.validate(
file_per_input[0], predicted_values)
good_predictions.append(good_prediction)
if good_prediction:
good_margin += margin
else:
bad_margin += margin
if not args.silent:
LOG.info(
'Prediction is %s predicted %s correct %s margin %s',
good_prediction, class_predicted, real_class, margin)
if not i % args.progress_every and i > 0:
LOG.info(
'ACCURACY: %.3f %%',
100 * sum(good_predictions) / len(good_predictions))
ExecutionProgress.progress(i, number_samples)
ExecutionProgress.end()
except (KeyboardInterrupt, SystemExit):
pass
self.py_locals['labels'] = validation.labels
self.py_locals['predictions'] = validation.predictions
cnt = len(good_predictions)
if cnt:
ngood = sum(good_predictions)
nbad = cnt - ngood
if nbad:
LOG.info(
"%s out of %s predicted falsly with %s average margin",
nbad, cnt, bad_margin / nbad)
if ngood:
LOG.info(
"%s out of %s predicted correctly with %s average margin",
ngood, cnt, good_margin / ngood)
accuracy_rate = 100 * sum(good_predictions) / len(good_predictions)
LOG.info('Total accuracy: %.3f %%', accuracy_rate)
def do_dump(self, args: argparse.Namespace):
"""
Dump the activations resulting from running an input file through the graph.
You can use the current quantization settings and can also just quantify one
specific step of the graph."""
self._check_graph()
dequantize = args.dequantize if args.dequantize is not None\
else not (args.pickle or args.save)
if args.quantize or args.quantize_step or args.quantize_all_steps:
self._check_quantized()
if args.quantize:
if dequantize:
qmode = QuantizationMode.all_dequantize()
else:
qmode = QuantizationMode.all()
elif args.quantize_all_steps:
qmode = QuantizationMode.step_all()
dequantize = True
else:
qmode = QuantizationMode.step(args.quantize_step)
elif args.quantize_and_dequantize:
qmode = QuantizationMode.all_float_quantize_dequantize()
else:
qmode = QuantizationMode.none()
if args.step is not None:
step = args.step
num_steps = len(self.G.graph_state.steps)
if step < 0:
step = num_steps + step
if step < 0 or step > num_steps:
self.perror("step must be from {} to {}".format(
-num_steps, num_steps))
return
else:
step = None
input_args = self._get_input_args(args)
pickles = []
for file_per_input in glob_input_files(args.input_files,
self.G.num_inputs):
LOG.info("input file %s", file_per_input)
data = [
import_data(input_file, **input_args)
for input_file in file_per_input
]
executer = GraphExecuter(self.G, qrecs=self.G.quantization)
outputs = executer.execute(data, step_idx_limit=step, qmode=qmode)
if args.pickle or self._in_py or args.save:
pickles.append(outputs)
else:
self.G.print_intermediates(outputs,
limit=step,
width=args.number_width,
precision=args.precision,
channel=args.channel,
order=['c', 'h', 'w'],
checksum=args.checksum)
if args.visualize_detection:
img_in = Image.open(file_per_input[0]).convert('RGBA')
height = img_in.size[1] if input_args[
'height'] == -1 else input_args['height']
width = img_in.size[0] if input_args[
'width'] == -1 else input_args['width']
img_in = img_in.resize((width, height))
if self.G.has_ssd_postprocess:
bboxes, classes, scores, _ = [
outputs[graph_out.step_idx][0]
for graph_out in self.G.outputs()
]
draw = ImageDraw.Draw(img_in, 'RGBA')
for box, score, class_id in zip(bboxes, scores, classes):
if args.quantize and not args.dequantize:
ssd_node = [
node for node in self.G.nodes()
if isinstance(node, SSDDetectorParameters)
][0]
ssd_qrec = self.G.quantization[NodeId(ssd_node)]
x0, x1 = int(box[1] * width *
ssd_qrec.out_qs[0].scale), int(
box[3] * width *
ssd_qrec.out_qs[0].scale)
y0, y1 = int(box[0] * height *
ssd_qrec.out_qs[0].scale), int(
box[2] * height *
ssd_qrec.out_qs[0].scale)
score = score * ssd_qrec.out_qs[2].scale
else:
x0, x1 = int(box[1] * width), int(box[3] * width)
y0, y1 = int(box[0] * height), int(box[2] * height)
rect_points = (x0, y0), (x1, y0), (x1, y1), (x0,
y1), (x0,
y0)
draw.line(rect_points, fill='red', width=2)
txt = '{}@{}%'.format(class_id, int(score * 100))
draw.text([x0, y0 - 10], txt, fill=(0, 255, 0))
img_in.show()
if args.pickle or args.save or self._in_py:
if not pickles:
self.perror("no input files found")
return
if len(args.input_files) == self.G.num_inputs:
pickles = pickles[0]
if args.pickle:
with open(args.pickle, 'wb') as pickle_fp:
pickle.dump(pickles, pickle_fp)
if args.save:
if len(args.input_files) != self.G.num_inputs:
self.perror(
"can only save dumps on one input to tensor store")
return
self.tensor_store[args.save] = pickles
if self._in_py:
self.last_result = pickles
def gen_project(G,
settings,
project_folder,
script_commands,
overwrite=False,
performance=False,
quantized=False,
test_results=False,
save_inputs=False,
input_file=None,
input_args=None,
gen_atproject=False,
dump_tensors=False,
input_tensors=None,
tolerance=0.0):
settings = deepcopy(settings)
settings['graph_monitor_cycles'] = True
settings['graph_produce_node_names'] = True
settings['graph_produce_operinfos'] = True
code_gen = CodeGenerator(G, DefaultNamingConvension(G), settings)
if not os.path.exists(project_folder):
os.mkdir(project_folder)
qoutputs = None
if test_results:
np.random.seed(12345)
finput_tensors = []
input_tensors = []
for i, node in enumerate(G.input_nodes()):
out_q = G.quantization[NodeId(node)].out_qs[0]
if input_file:
file_per_input = glob_input_files(input_file, G.num_inputs)[0]
finput = import_data(file_per_input[i], **input_args)
else:
min_val = out_q.min if not out_q.is_floating else -1.0
max_val = out_q.max if not out_q.is_floating else 1.0
finput = get_rand(node.out_dims[0].shape,
low_high=(min_val, max_val))
finput_tensors.append(finput)
executer = GraphExecuter(G, qrecs=G.quantization)
qoutput_tensors = executer.execute(finput_tensors.copy(),
qmode=QuantizationMode.all())
qoutputs = []
for params in G.outputs():
outp = qoutput_tensors[params.step_idx][0]
qoutputs.append(outp)
for i, params in enumerate(G.input_nodes()):
inp = qoutput_tensors[params.step_idx][0]
input_tensors.append(inp)
if save_inputs:
nodeq = G.quantization[NodeId(params, None)].out_qs[0]
np.save(os.path.join(project_folder, f"fake_input_{i}.npy"),
nodeq.dequantize(inp))
main = os.path.join(project_folder, f"{code_gen.project_name}")
main_c = main + '.c'
main_h = main + '.h'
common_mk = os.path.join(project_folder, "common.mk")
nntool_script = os.path.join(project_folder, "nntool_script")
if overwrite or not os.path.exists(main_c):
with open(os.path.join(project_folder, f"{code_gen.project_name}.c"),
"w") as output_fp:
output_fp.write(
generate_main_appl_template(G, code_gen, input_tensors,
qoutputs, tolerance))
if overwrite or not os.path.exists(main_h):
with open(os.path.join(project_folder, f"{code_gen.project_name}.h"),
"w") as output_fp:
output_fp.write(generate_main_appl_header(G, code_gen))
if overwrite or not os.path.exists(common_mk):
open_args = parse_last_open(script_commands)
open_args = build_last_open_args(open_args) if open_args else ""
with open(os.path.join(project_folder, "common.mk"), "w") as output_fp:
if gen_atproject:
output_fp.write(
generate_main_appl_make_atproject(G, code_gen, quantized,
'Model.c'))
else:
output_fp.write(
generate_main_appl_make(G,
code_gen,
quantized,
open_args=open_args))
if overwrite or not os.path.exists(nntool_script):
with open(nntool_script, 'w') as fp:
# NOTE - gen_template_project is excluded so that tests work. Normally it will not be in the
# history.
fp.writelines(process_script(script_commands))
# always add performance since the main template uses it
for setting in [
'set graph_produce_node_names true',
'set graph_produce_operinfos true',
'set graph_monitor_cycles true'
]:
fp.write(f'{setting}\n')
if dump_tensors:
fp.write('set graph_dump_tensor 7\n')
if script_commands[-1] != "save_state":
fp.write('save_state\n')
if gen_atproject:
code_gen = CodeGenerator(G, DefaultNamingConvension(G), settings)
with open(os.path.join(project_folder, 'Model.c'), "w") as output_fp:
output_fp.write(default_template(G, code_generator=code_gen))
if G.has_expressions:
with open(os.path.join(project_folder, "Expression_Kernels.c"),
"w") as output_fp:
output_fp.write(
basic_kernel_source_template(G, code_generator=code_gen))
with open(os.path.join(project_folder, "Expression_Kernels.h"),
"w") as output_fp:
output_fp.write(
basic_kernel_header_template(G, code_generator=code_gen))
code_gen.write_constants(tensor_directory=project_folder)
ignore_function = None if overwrite else skip_existing_files(
project_folder)
shutil.copytree(os.path.join(os.environ.get("NNTOOL_PATH"),
'generation/project_template'),
project_folder,
dirs_exist_ok=True,
ignore=ignore_function)
if not gen_atproject:
try:
shutil.copy(
G.graph_identity.filename,
os.path.join(project_folder,
os.path.split(G.graph_identity.filename)[1]))
except shutil.SameFileError:
pass
def do_dump(self, args: argparse.Namespace):
"""
Dump the activations resulting from running an input file through the graph.
You can use the current quantization settings and can also just quantify one
specific step of the graph."""
self._check_graph()
dequantize = args.dequantize if args.dequantize is not None\
else not (args.pickle or args.save)
if args.quantize or args.quantize_step or args.quantize_all_steps:
self._check_quantized()
if args.quantize:
if dequantize:
qmode = QuantizationMode.all_dequantize()
else:
qmode = QuantizationMode.all()
elif args.quantize_all_steps:
qmode = QuantizationMode.step_all()
dequantize = True
else:
qmode = QuantizationMode.step(args.quantize_step)
elif args.quantize_and_dequantize:
qmode = QuantizationMode.all_float_quantize_dequantize()
else:
qmode = QuantizationMode.none()
if args.step is not None:
step = args.step
num_steps = len(self.G.graph_state.steps)
if step < 0:
step = num_steps + step
if step < 0 or step > num_steps:
self.perror("step must be from {} to {}".format(-num_steps, num_steps))
return
else:
step = None
input_args = self._get_input_args(args)
pickles = []
for file_per_input in glob_input_files(args.input_files, self.G.num_inputs):
LOG.info("input file %s", file_per_input)
data = [import_data(input_file, **input_args) for input_file in file_per_input]
executer = GraphExecuter(self.G, qrecs=self.G.quantization)
outputs = executer.execute(data, step_idx_limit=step,
qmode=qmode)
if args.pickle or self._in_py or args.save:
pickles.append(format_dump_file(self.G, outputs, not qmode.is_none,
args.dequantize, args.quantize_step))
else:
self.G.print_intermediates(outputs, limit=step, width=args.number_width,
precision=args.precision, channel=args.channel,
order=['c', 'h', 'w'])
if args.pickle or args.save or self._in_py:
if not pickles:
self.perror("no input files found")
return
if len(args.input_files) == 1:
pickles = pickles[0]
if args.pickle:
with open(args.pickle, 'wb') as pickle_fp:
pickle.dump(pickles, pickle_fp)
if args.save:
self.tensor_store[args.save] = pickles
if self._in_py:
self.last_result = pickles