def two_ir_mode(args):
core = get_plugin(args.device, args.l, args.config)
ref_core = get_plugin(args.reference_device, args.l, args.reference_config)
net = get_net(model=args.model, core=core)
net_layers, net_inputs, net_outputs = get_model_info(net)
ref_net = get_net(model=args.reference_model, core=ref_core)
ref_net_layers, ref_net_inputs, ref_net_outputs = get_model_info(ref_net)
log.info(f'{args.device} vs {args.reference_device}')
log.info(f'IR for {args.device} : {args.model}')
log.info(f'IR for {args.reference_device} : {args.reference_model}')
out_layers = get_layers_list(net_layers, net_inputs, net_outputs, args.layers)
ref_out_layers = get_layers_list(ref_net_layers, ref_net_inputs, ref_net_outputs, args.layers)
print_input_layers(net_inputs)
print_output_layers(out_layers)
layers_map = manage_user_outputs_with_mapping(mapping=args.mapping, reference_mapping=args.reference_mapping,
user_layers=out_layers)
inputs = input_processing(model_path=args.model, net_inputs=net_inputs, input_file=args.input,
layers_map=layers_map)
ref_inputs = input_processing(model_path=args.reference_model, net_inputs=ref_net_inputs, input_file=args.input,
layers_map=layers_map)
global_accuracy = []
global_times, ref_global_times = overall_accuracy_check(model=args.model, ref_model=args.reference_model,
out_layers=out_layers, ref_out_layers=ref_out_layers,
inputs=inputs, ref_inputs=ref_inputs, core=core,
device=args.device, ref_core=ref_core,
ref_device=args.reference_device, layers=args.layers,
num_of_iterations=args.num_of_iterations)
for out_layer in layers_map:
ref_out_layer = layers_map[out_layer]
if out_layer == ref_out_layer:
log.info(f'Layer {out_layer} statistics')
else:
log.info(f'Statistics \'{out_layer}\' vs \'{ref_out_layer}\'')
net_copy = get_net_copy_with_output(model=args.model, output=out_layer, core=core)
ref_net_copy = get_net_copy_with_output(model=args.reference_model, output=ref_out_layer, core=ref_core)
results = infer(net=net_copy, core=core, device=args.device, inputs=inputs, output=[out_layer])
if out_layer not in results:
continue
out_blob, pc = results[out_layer]
ref_results = infer(net=ref_net_copy, core=ref_core, device=args.reference_device,
inputs=ref_inputs, output=[ref_out_layer])
ref_out_blob, ref_pc = ref_results[ref_out_layer]
if ref_out_layer not in ref_results:
continue
a_m = accuracy_metrics(out_blob=out_blob, ref_out_blob=ref_out_blob)
performance_metrics(pc=pc, ref_pc=ref_pc)
blob_counters(out_blob=out_blob, ref_out_blob=ref_out_blob)
global_accuracy = update_global_accuracy_matrics(global_accuracy=global_accuracy, current_accuracy=a_m)
print_all_over_the_net_metrics(global_times=global_times, ref_global_times=ref_global_times,
global_accuracy=global_accuracy)
def load_mode(args):
core = get_plugin(args.device, args.l, args.config)
log.info(f'IR for {args.device} : {args.model}')
log.info(f'Loading blob from {args.load}')
net = get_net(model=args.model, core=core)
net_layers, net_inputs, net_outputs = get_model_info(net)
out_layers = get_layers_list(net_layers, net_inputs, net_outputs, args.layers)
print_input_layers(net_inputs)
print_output_layers(out_layers)
layers_map = manage_user_outputs_with_mapping(mapping=args.mapping, reference_mapping=args.reference_mapping,
user_layers=out_layers)
inputs = input_processing(args.model, net_inputs, args.input, layers_map)
global_accuracy = []
loaded = load_dump(args.load)
for out_layer in layers_map:
ref_out_layer = layers_map[out_layer]
if out_layer == ref_out_layer:
log.info(f'Layer {out_layer} statistics')
else:
log.info(f'Statistics \'{out_layer}\' vs \'{ref_out_layer}\'')
net_copy = get_net_copy_with_output(model=args.model, output=out_layer, core=core)
results = infer(net=net_copy, core=core, device=args.device, inputs=inputs, output=[out_layer])
if out_layer not in results:
continue
out_blob, pc = results[out_layer]
if ref_out_layer not in loaded:
continue
ref_out_blob = loaded[ref_out_layer]['blob']
a_m = accuracy_metrics(out_blob=out_blob, ref_out_blob=ref_out_blob)
if 'pc' in loaded[ref_out_layer]:
ref_pc = loaded[ref_out_layer]['pc']
performance_metrics(pc=pc, ref_pc=ref_pc)
blob_counters(out_blob=out_blob, ref_out_blob=ref_out_blob)
global_accuracy = update_global_accuracy_matrics(global_accuracy=global_accuracy, current_accuracy=a_m)
print_all_over_the_net_metrics(global_accuracy=global_accuracy)
def dump_mode(args):
core = get_plugin(args.device, args.l, args.config)
model = get_model(model_path=args.model, core=core)
model_ops, model_inputs, model_outputs = get_model_info(model)
out_ops = get_ops_list(model_ops, model_outputs, args.layers)
inputs = input_processing(args.model, model_inputs, args.input)
dump_dict = defaultdict(list)
for op in out_ops:
for i in range(op.get_output_size()):
if op.get_output_size() > 1:
log.info(f'Layer {op.friendly_name}, port {i} processing')
else:
log.info(f'Layer {op.friendly_name} processing')
model_copy, new_output = get_model_copy_with_output(
model=args.model, output=(op.friendly_name, i), core=core)
out_tensor, pc = infer(model=model_copy,
core=core,
device=args.device,
inputs=inputs,
output=new_output)
dump_dict[op.friendly_name].append(
np.array({
'tensor': out_tensor,
'pc': perf_counts_to_dump(pc)
}))
dump_dict["device"] = args.device
dump_output_file(args.model + '_' + args.device + '_dump.npz', dump_dict)
def one_ir_mode(args):
core = get_plugin(args.device, args.l, args.config)
net = get_net(model=args.model, core=core)
net_layers, net_inputs, net_outputs = get_model_info(net)
log.info(f'{args.device} vs {args.reference_device}')
log.info(f'The same IR on both devices: {args.model}')
out_layers = get_layers_list(net_layers, net_inputs, net_outputs,
args.layers)
print_input_layers(net_inputs)
print_output_layers(out_layers)
ref_core = get_plugin(args.reference_device, args.l, args.reference_config)
global_accuracy = []
inputs = input_processing(model_path=args.model,
net_inputs=net_inputs,
input_file=args.input)
global_times, ref_global_times = overall_accuracy_check(
model=args.model,
ref_model=args.model,
out_layers=out_layers,
ref_out_layers=out_layers,
inputs=inputs,
ref_inputs=inputs,
core=core,
device=args.device,
ref_core=ref_core,
ref_device=args.reference_device,
layers=args.layers,
num_of_iterations=args.num_of_iterations)
for out_layer in out_layers:
log.info(f'Layer {out_layer} statistics')
net_copy = get_net_copy_with_output(model=args.model,
output=out_layer,
core=core)
results = infer(net=net_copy,
core=core,
device=args.device,
inputs=inputs,
output=[out_layer])
if out_layer not in results:
continue
out_blob, pc = results[out_layer]
ref_results = infer(net=net_copy,
core=ref_core,
device=args.reference_device,
inputs=inputs,
output=[out_layer])
if out_layer not in ref_results:
continue
ref_out_blob, ref_pc = ref_results[out_layer]
a_m = accuracy_metrics(out_blob=out_blob, ref_out_blob=ref_out_blob)
performance_metrics(pc=pc, ref_pc=ref_pc)
blob_counters(out_blob=out_blob, ref_out_blob=ref_out_blob)
global_accuracy = update_global_accuracy_matrics(
global_accuracy=global_accuracy, current_accuracy=a_m)
print_all_over_the_net_metrics(global_times=global_times,
ref_global_times=ref_global_times,
global_accuracy=global_accuracy)
def one_ir_mode(args):
core = get_plugin(args.device, args.l, args.config)
model = get_model(model_path=args.model, core=core)
model_ops, model_inputs, model_outputs = get_model_info(model)
log.info(f'{args.device} vs {args.reference_device}')
log.info(f'The same IR on both devices: {args.model}')
out_ops = get_ops_list(model_ops, model_outputs, args.layers)
print_inputs(model_inputs)
print_output_ops(out_ops)
ref_core = get_plugin(args.reference_device, args.l, args.reference_config)
global_accuracy = []
inputs = input_processing(model_path=args.model,
model_inputs=model_inputs,
input_file=args.input)
global_times, ref_global_times = overall_accuracy_check(
model=args.model,
ref_model=args.model,
out_ops=out_ops,
ref_out_ops=out_ops,
inputs=inputs,
ref_inputs=inputs,
core=core,
device=args.device,
ref_core=ref_core,
ref_device=args.reference_device,
layers=args.layers,
num_of_iterations=args.num_of_iterations)
for op in out_ops:
log.info(f'Layer {op.friendly_name} statistics')
for i in range(op.get_output_size()):
if op.get_output_size() > 1:
log.info(f'Port {i}: ')
model_copy, new_output = get_model_copy_with_output(
model=args.model, output=(op.friendly_name, i), core=core)
out_tensor, pc = infer(model=model_copy,
core=core,
device=args.device,
inputs=inputs,
output=new_output)
ref_out_tensor, ref_pc = infer(model=model_copy,
core=ref_core,
device=args.reference_device,
inputs=inputs,
output=new_output)
a_m = accuracy_metrics(out_tensor, ref_out_tensor)
performance_metrics(args.device, pc, args.reference_device, ref_pc)
tensor_counters(out_tensor, ref_out_tensor)
global_accuracy = update_global_accuracy_matrics(
global_accuracy=global_accuracy, current_accuracy=a_m)
print_all_over_the_net_metrics(global_times=global_times,
ref_global_times=ref_global_times,
global_accuracy=global_accuracy)
def dump_mode(args):
core = get_plugin(args.device, args.l, args.config)
net = get_net(model=args.model, core=core)
func = ng.function_from_cnn(net)
ops = func.get_ops()
out_layers = get_layers_list(ops, net.input_info, net.outputs, args.layers)
inputs = input_processing(args.model, net.input_info, args.input)
dump_dict = {}
for out_layer in out_layers:
log.info(f'Layer {out_layer} processing')
net_copy = get_net_copy_with_output(model=args.model, output=out_layer, core=core)
results = infer(net=net_copy, core=core, device=args.device, inputs=inputs, output=[out_layer])
if out_layer not in results:
continue
out_blob, pc = results[out_layer]
dump_dict[out_layer] = np.array({'blob': out_blob, 'pc': pc})
dump_output_file(args.model + '_' + args.device + '_dump.npz', dump_dict)
def load_mode(args):
core = get_plugin(args.device, args.l, args.config)
log.info(f'IR for {args.device} : {args.model}')
log.info(f'Loading tensors from {args.load}')
model = get_model(model_path=args.model, core=core)
model_ops, model_inputs, model_outputs = get_model_info(model)
out_ops = get_ops_list(model_ops, model_outputs, args.layers)
print_inputs(model_inputs)
print_output_ops(out_ops)
inputs = input_processing(args.model, model_inputs, args.input)
global_accuracy = []
loaded = load_dump(args.load)
for op in out_ops:
if op.friendly_name in loaded:
log.info(f'Layer {op.friendly_name} statistics')
else:
log.info(
f'Statistics for layer \'{op.friendly_name}\' was not dumped. Skipping this layer.'
)
continue
for i in range(op.get_output_size()):
if op.get_output_size() > 1:
log.info(f'Port {i}: ')
model_copy, new_output = get_model_copy_with_output(
model=args.model, output=(op.friendly_name, i), core=core)
out_tensor, pc = infer(model=model_copy,
core=core,
device=args.device,
inputs=inputs,
output=new_output)
ref_out_tensor, ref_pc = loaded[
op.friendly_name][i]['tensor'], load_profiling_info(
loaded[op.friendly_name][i]['pc'])
a_m = accuracy_metrics(out_tensor, ref_out_tensor)
performance_metrics(args.device, pc, loaded["device"], ref_pc)
tensor_counters(out_tensor, ref_out_tensor)
global_accuracy = update_global_accuracy_matrics(
global_accuracy=global_accuracy, current_accuracy=a_m)
print_all_over_the_net_metrics(global_accuracy=global_accuracy)
def two_ir_mode(args):
core = get_plugin(args.device, args.l, args.config)
ref_core = get_plugin(args.reference_device, args.l, args.reference_config)
model = get_model(model_path=args.model, core=core)
model_ops, model_inputs, model_outputs = get_model_info(model)
ref_model = get_model(model_path=args.reference_model, core=ref_core)
ref_model_ops, _, _ = get_model_info(ref_model)
check_inputs_and_default_outputs_are_equal(model, ref_model)
log.info(f'{args.device} vs {args.reference_device}')
log.info(f'IR for {args.device} : {args.model}')
log.info(f'IR for {args.reference_device} : {args.reference_model}')
if args.reference_layers:
out_ops = get_ops_list(model_ops, model_outputs, args.layers)
ref_out_ops = get_ops_list(ref_model_ops, model_outputs,
args.reference_layers)
if len(out_ops) != len(ref_out_ops):
raise Exception(
"Number of layers to compare against should be equal!")
else:
ref_out_ops = out_ops = get_ops_list(
get_ops_intersection(model_ops, ref_model_ops), model_outputs,
args.layers)
print_inputs(model_inputs)
print_output_ops(get_ops_union(out_ops, ref_out_ops))
inputs = input_processing(model_path=args.model,
model_inputs=model_inputs,
input_file=args.input)
global_accuracy = []
global_times, ref_global_times = overall_accuracy_check(
model=args.model,
ref_model=args.reference_model,
out_ops=out_ops,
ref_out_ops=out_ops,
inputs=inputs,
ref_inputs=inputs,
core=core,
device=args.device,
ref_core=ref_core,
ref_device=args.reference_device,
layers=args.layers,
num_of_iterations=args.num_of_iterations)
for op, ref_op in zip(out_ops, ref_out_ops):
if op.friendly_name == ref_op.friendly_name:
log.info(f'Layer {op.friendly_name} statistics')
else:
if op.get_output_size() != ref_op.get_output_size():
log.warning(
f"Skipping {op.friendly_name} vs {ref_op.frinedly_name} comparison due to different number of outputs!"
)
continue
log.info(
f'Layer {op.friendly_name} vs {ref_op.friendly_name} statistics'
)
for i in range(op.get_output_size()):
if op.get_output_size() > 1:
log.info(f'Port {i}: ')
model_copy, new_output = get_model_copy_with_output(
model=args.model, output=(op.friendly_name, i), core=core)
ref_model_copy, ref_new_output = get_model_copy_with_output(
model=args.reference_model,
output=(ref_op.friendly_name, i),
core=ref_core)
out_tensor, pc = infer(model=model_copy,
core=core,
device=args.device,
inputs=inputs,
output=new_output)
ref_out_tensor, ref_pc = infer(model=ref_model_copy,
core=ref_core,
device=args.reference_device,
inputs=inputs,
output=ref_new_output)
a_m = accuracy_metrics(out_tensor, ref_out_tensor)
performance_metrics(args.device, pc, args.reference_device, ref_pc)
tensor_counters(out_tensor, ref_out_tensor)
global_accuracy = update_global_accuracy_matrics(
global_accuracy=global_accuracy, current_accuracy=a_m)
print_all_over_the_net_metrics(global_times=global_times,
ref_global_times=ref_global_times,
global_accuracy=global_accuracy)