def validate(self, epoch):
f1 = AverageMeter("f1")
acc = AverageMeter("acc")
losses = AverageMeter("losses")
self.model.eval()
with torch.no_grad():
for step, (x, y) in enumerate(self.test_loader):
bs = x.size(0)
logits = self.model(x)
if isinstance(logits, tuple):
logits, _ = logits
metrics = accuracy_metrics(logits, y)
loss = self.criterion(logits, y)
losses.update(loss.item(), bs)
acc.update(metrics["acc_score"], bs)
f1.update(metrics["f1_score"], bs)
logger.info("Valid: [{:3d}/{}] Loss {losses.avg:.3f} "
"acc {acc.avg:.2%}, f1 {f1.avg:.2%}".format(epoch + 1,
100,
losses=losses,
acc=acc,
f1=f1))
return f1.avg, losses.avg
def load_mode(args):
core = get_plugin(args.device, args.l, args.config)
log.info('IR for {} : {}'.format(args.device, args.model))
log.info('Loading blob from {}'.format(args.load))
net = get_net(model=args.model, core=core)
net_layers, net_inputs, net_outputs = get_model_info(net)
out_layers = get_layers_list(net_layers, net_inputs, net_outputs, args.layers)
print_input_layers(net_inputs)
print_output_layers(out_layers)
layers_map = manage_user_outputs_with_mapping(mapping=args.mapping, reference_mapping=args.reference_mapping,
user_layers=out_layers)
inputs = input_processing(args.model, net_inputs, args.input, layers_map)
global_accuracy = []
loaded = load_dump(args.load)
for out_layer in layers_map:
ref_out_layer = layers_map[out_layer]
if out_layer == ref_out_layer:
log.info('Layer {} statistics'.format(out_layer))
else:
log.info('Statistics \'{}\' vs \'{}\''.format(out_layer, ref_out_layer))
net_copy = get_net_copy_with_output(model=args.model, output=out_layer, core=core)
results = infer(net=net_copy, core=core, device=args.device, inputs=inputs, output=[out_layer])
if out_layer not in results:
continue
out_blob, pc = results[out_layer]
if ref_out_layer not in loaded:
continue
ref_out_blob = loaded[ref_out_layer]['blob']
a_m = accuracy_metrics(out_blob=out_blob, ref_out_blob=ref_out_blob)
if 'pc' in loaded[ref_out_layer]:
ref_pc = loaded[ref_out_layer]['pc']
performance_metrics(pc=pc, ref_pc=ref_pc)
blob_counters(out_blob=out_blob, ref_out_blob=ref_out_blob)
global_accuracy = update_global_accuracy_matrics(global_accuracy=global_accuracy, current_accuracy=a_m)
print_all_over_the_net_metrics(global_accuracy=global_accuracy)
def one_ir_mode(args):
net = get_net(model=args.model)
net_layers, net_inputs, net_outputs, precision = get_model_info(net)
log.info('{}:{} vs {}:{}'.format(args.device, precision, args.reference_device, precision))
log.info('The same IR on both devices: {}'.format(args.model))
out_layers = get_layers_list(net_layers, net_inputs, net_outputs, args.layers)
print_input_layers(net_inputs)
print_output_layers(out_layers)
plugin = get_plugin(args.device, args.l, args.plugin_path, args.config)
ref_plugin = get_plugin(args.reference_device, args.l, args.plugin_path, args.reference_config)
global_accuracy = []
inputs = input_processing(model_path=args.model, net_inputs=net_inputs, input_file=args.input)
global_times, ref_global_times = overall_accuracy_check(model=args.model, ref_model=args.model,
out_layers=out_layers, ref_out_layers=out_layers,
inputs=inputs, ref_inputs=inputs, plugin=plugin,
ref_plugin=ref_plugin, layers=args.layers,
num_of_iterations=args.num_of_iterations)
for out_layer in out_layers:
log.info('Layer {} statistics'.format(out_layer))
net_copy = get_net_copy_with_output(model=args.model, output=out_layer)
results = infer(net=net_copy, plugin=plugin, inputs=inputs, output=[out_layer])
if out_layer not in results:
continue
out_blob, pc = results[out_layer]
ref_results = infer(net=net_copy, plugin=ref_plugin, inputs=inputs, output=[out_layer])
if out_layer not in ref_results:
continue
ref_out_blob, ref_pc = ref_results[out_layer]
a_m = accuracy_metrics(out_blob=out_blob, ref_out_blob=ref_out_blob)
performance_metrics(pc=pc, ref_pc=ref_pc)
blob_counters(out_blob=out_blob, ref_out_blob=ref_out_blob)
global_accuracy = update_global_accuracy_matrics(global_accuracy=global_accuracy, current_accuracy=a_m)
print_all_over_the_net_metrics(global_times=global_times, ref_global_times=ref_global_times,
global_accuracy=global_accuracy)
def one_ir_mode(args):
core = get_plugin(args.device, args.l, args.config)
net = get_net(model=args.model, core=core)
net_layers, net_inputs, net_outputs = get_model_info(net)
log.info(f'{args.device} vs {args.reference_device}')
log.info(f'The same IR on both devices: {args.model}')
out_layers = get_layers_list(net_layers, net_inputs, net_outputs,
args.layers)
print_input_layers(net_inputs)
print_output_layers(out_layers)
ref_core = get_plugin(args.reference_device, args.l, args.reference_config)
global_accuracy = []
inputs = input_processing(model_path=args.model,
net_inputs=net_inputs,
input_file=args.input)
global_times, ref_global_times = overall_accuracy_check(
model=args.model,
ref_model=args.model,
out_layers=out_layers,
ref_out_layers=out_layers,
inputs=inputs,
ref_inputs=inputs,
core=core,
device=args.device,
ref_core=ref_core,
ref_device=args.reference_device,
layers=args.layers,
num_of_iterations=args.num_of_iterations)
for out_layer in out_layers:
log.info(f'Layer {out_layer} statistics')
net_copy = get_net_copy_with_output(model=args.model,
output=out_layer,
core=core)
results = infer(net=net_copy,
core=core,
device=args.device,
inputs=inputs,
output=[out_layer])
if out_layer not in results:
continue
out_blob, pc = results[out_layer]
ref_results = infer(net=net_copy,
core=ref_core,
device=args.reference_device,
inputs=inputs,
output=[out_layer])
if out_layer not in ref_results:
continue
ref_out_blob, ref_pc = ref_results[out_layer]
a_m = accuracy_metrics(out_blob=out_blob, ref_out_blob=ref_out_blob)
performance_metrics(pc=pc, ref_pc=ref_pc)
blob_counters(out_blob=out_blob, ref_out_blob=ref_out_blob)
global_accuracy = update_global_accuracy_matrics(
global_accuracy=global_accuracy, current_accuracy=a_m)
print_all_over_the_net_metrics(global_times=global_times,
ref_global_times=ref_global_times,
global_accuracy=global_accuracy)
def two_ir_mode(args):
core = get_plugin(args.device, args.l, args.config)
ref_core = get_plugin(args.reference_device, args.l, args.reference_config)
net = get_net(model=args.model, core=core)
net_layers, net_inputs, net_outputs = get_model_info(net)
ref_net = get_net(model=args.reference_model, core=ref_core)
ref_net_layers, ref_net_inputs, ref_net_outputs = get_model_info(ref_net)
log.info('{} vs {}'.format(args.device, args.reference_device))
log.info('IR for {} : {}'.format(args.device, args.model))
log.info('IR for {} : {}'.format(args.reference_device, args.reference_model))
out_layers = get_layers_list(net_layers, net_inputs, net_outputs, args.layers)
ref_out_layers = get_layers_list(ref_net_layers, ref_net_inputs, ref_net_outputs, args.layers)
print_input_layers(net_inputs)
print_output_layers(out_layers)
layers_map = manage_user_outputs_with_mapping(mapping=args.mapping, reference_mapping=args.reference_mapping,
user_layers=out_layers)
inputs = input_processing(model_path=args.model, net_inputs=net_inputs, input_file=args.input,
layers_map=layers_map)
ref_inputs = input_processing(model_path=args.reference_model, net_inputs=ref_net_inputs, input_file=args.input,
layers_map=layers_map)
global_accuracy = []
global_times, ref_global_times = overall_accuracy_check(model=args.model, ref_model=args.reference_model,
out_layers=out_layers, ref_out_layers=ref_out_layers,
inputs=inputs, ref_inputs=ref_inputs, core=core,
device=args.device, ref_core=ref_core,
ref_device=args.reference_device, layers=args.layers,
num_of_iterations=args.num_of_iterations)
for out_layer in layers_map:
ref_out_layer = layers_map[out_layer]
if out_layer == ref_out_layer:
log.info('Layer {} statistics'.format(out_layer))
else:
log.info('Statistics \'{}\' vs \'{}\''.format(out_layer, ref_out_layer))
net_copy = get_net_copy_with_output(model=args.model, output=out_layer, core=core)
ref_net_copy = get_net_copy_with_output(model=args.reference_model, output=ref_out_layer, core=ref_core)
results = infer(net=net_copy, core=core, device=args.device, inputs=inputs, output=[out_layer])
if out_layer not in results:
continue
out_blob, pc = results[out_layer]
ref_results = infer(net=ref_net_copy, core=ref_core, device=args.reference_device,
inputs=ref_inputs, output=[ref_out_layer])
ref_out_blob, ref_pc = ref_results[ref_out_layer]
if ref_out_layer not in ref_results:
continue
a_m = accuracy_metrics(out_blob=out_blob, ref_out_blob=ref_out_blob)
performance_metrics(pc=pc, ref_pc=ref_pc)
blob_counters(out_blob=out_blob, ref_out_blob=ref_out_blob)
global_accuracy = update_global_accuracy_matrics(global_accuracy=global_accuracy, current_accuracy=a_m)
print_all_over_the_net_metrics(global_times=global_times, ref_global_times=ref_global_times,
global_accuracy=global_accuracy)
def train(self, epoch):
f1 = AverageMeter("f1")
acc = AverageMeter("acc")
losses = AverageMeter("losses")
self.model.train()
cur_step = epoch * len(self.train_loader)
cur_lr = self.optimizer.param_groups[0]["lr"]
logger.info("Epoch %d LR %.6f", epoch, cur_lr)
for step, (x, y) in enumerate(self.train_loader):
bs = x.size(0)
self.optimizer.zero_grad()
logits = self.model(x)
if isinstance(logits, tuple):
logits, aux_logits = logits
aux_loss = criterion(aux_logits, y)
else:
aux_loss = 0.
metrics = accuracy_metrics(logits, y)
loss = self.criterion(logits, y)
loss = loss + 0.4 * aux_loss
loss.backward()
nn.utils.clip_grad_norm_(self.model.parameters(), 5.)
self.optimizer.step()
losses.update(loss.item(), bs)
acc.update(metrics["acc_score"], bs)
f1.update(metrics["f1_score"], bs)
cur_step += 1
logger.info("Train: [{:3d}/{}] Loss {losses.avg:.3f} "
"acc {acc.avg:.2%}, f1 {f1.avg:.2%}".format(epoch + 1,
100,
losses=losses,
acc=acc,
f1=f1))
return losses.avg
