def load_mode(args):
core = get_plugin(args.device, args.l, args.config)
log.info('IR for {} : {}'.format(args.device, args.model))
log.info('Loading blob from {}'.format(args.load))
net = get_net(model=args.model, core=core)
net_layers, net_inputs, net_outputs = get_model_info(net)
out_layers = get_layers_list(net_layers, net_inputs, net_outputs, args.layers)
print_input_layers(net_inputs)
print_output_layers(out_layers)
layers_map = manage_user_outputs_with_mapping(mapping=args.mapping, reference_mapping=args.reference_mapping,
user_layers=out_layers)
inputs = input_processing(args.model, net_inputs, args.input, layers_map)
global_accuracy = []
loaded = load_dump(args.load)
for out_layer in layers_map:
ref_out_layer = layers_map[out_layer]
if out_layer == ref_out_layer:
log.info('Layer {} statistics'.format(out_layer))
else:
log.info('Statistics \'{}\' vs \'{}\''.format(out_layer, ref_out_layer))
net_copy = get_net_copy_with_output(model=args.model, output=out_layer, core=core)
results = infer(net=net_copy, core=core, device=args.device, inputs=inputs, output=[out_layer])
if out_layer not in results:
continue
out_blob, pc = results[out_layer]
if ref_out_layer not in loaded:
continue
ref_out_blob = loaded[ref_out_layer]['blob']
a_m = accuracy_metrics(out_blob=out_blob, ref_out_blob=ref_out_blob)
if 'pc' in loaded[ref_out_layer]:
ref_pc = loaded[ref_out_layer]['pc']
performance_metrics(pc=pc, ref_pc=ref_pc)
blob_counters(out_blob=out_blob, ref_out_blob=ref_out_blob)
global_accuracy = update_global_accuracy_matrics(global_accuracy=global_accuracy, current_accuracy=a_m)
print_all_over_the_net_metrics(global_accuracy=global_accuracy)
def one_ir_mode(args):
net = get_net(model=args.model)
net_layers, net_inputs, net_outputs, precision = get_model_info(net)
log.info('{}:{} vs {}:{}'.format(args.device, precision, args.reference_device, precision))
log.info('The same IR on both devices: {}'.format(args.model))
out_layers = get_layers_list(net_layers, net_inputs, net_outputs, args.layers)
print_input_layers(net_inputs)
print_output_layers(out_layers)
plugin = get_plugin(args.device, args.l, args.plugin_path, args.config)
ref_plugin = get_plugin(args.reference_device, args.l, args.plugin_path, args.reference_config)
global_accuracy = []
inputs = input_processing(model_path=args.model, net_inputs=net_inputs, input_file=args.input)
global_times, ref_global_times = overall_accuracy_check(model=args.model, ref_model=args.model,
out_layers=out_layers, ref_out_layers=out_layers,
inputs=inputs, ref_inputs=inputs, plugin=plugin,
ref_plugin=ref_plugin, layers=args.layers,
num_of_iterations=args.num_of_iterations)
for out_layer in out_layers:
log.info('Layer {} statistics'.format(out_layer))
net_copy = get_net_copy_with_output(model=args.model, output=out_layer)
results = infer(net=net_copy, plugin=plugin, inputs=inputs, output=[out_layer])
if out_layer not in results:
continue
out_blob, pc = results[out_layer]
ref_results = infer(net=net_copy, plugin=ref_plugin, inputs=inputs, output=[out_layer])
if out_layer not in ref_results:
continue
ref_out_blob, ref_pc = ref_results[out_layer]
a_m = accuracy_metrics(out_blob=out_blob, ref_out_blob=ref_out_blob)
performance_metrics(pc=pc, ref_pc=ref_pc)
blob_counters(out_blob=out_blob, ref_out_blob=ref_out_blob)
global_accuracy = update_global_accuracy_matrics(global_accuracy=global_accuracy, current_accuracy=a_m)
print_all_over_the_net_metrics(global_times=global_times, ref_global_times=ref_global_times,
global_accuracy=global_accuracy)
def one_ir_mode(args):
core = get_plugin(args.device, args.l, args.config)
net = get_net(model=args.model, core=core)
net_layers, net_inputs, net_outputs = get_model_info(net)
log.info(f'{args.device} vs {args.reference_device}')
log.info(f'The same IR on both devices: {args.model}')
out_layers = get_layers_list(net_layers, net_inputs, net_outputs,
args.layers)
print_input_layers(net_inputs)
print_output_layers(out_layers)
ref_core = get_plugin(args.reference_device, args.l, args.reference_config)
global_accuracy = []
inputs = input_processing(model_path=args.model,
net_inputs=net_inputs,
input_file=args.input)
global_times, ref_global_times = overall_accuracy_check(
model=args.model,
ref_model=args.model,
out_layers=out_layers,
ref_out_layers=out_layers,
inputs=inputs,
ref_inputs=inputs,
core=core,
device=args.device,
ref_core=ref_core,
ref_device=args.reference_device,
layers=args.layers,
num_of_iterations=args.num_of_iterations)
for out_layer in out_layers:
log.info(f'Layer {out_layer} statistics')
net_copy = get_net_copy_with_output(model=args.model,
output=out_layer,
core=core)
results = infer(net=net_copy,
core=core,
device=args.device,
inputs=inputs,
output=[out_layer])
if out_layer not in results:
continue
out_blob, pc = results[out_layer]
ref_results = infer(net=net_copy,
core=ref_core,
device=args.reference_device,
inputs=inputs,
output=[out_layer])
if out_layer not in ref_results:
continue
ref_out_blob, ref_pc = ref_results[out_layer]
a_m = accuracy_metrics(out_blob=out_blob, ref_out_blob=ref_out_blob)
performance_metrics(pc=pc, ref_pc=ref_pc)
blob_counters(out_blob=out_blob, ref_out_blob=ref_out_blob)
global_accuracy = update_global_accuracy_matrics(
global_accuracy=global_accuracy, current_accuracy=a_m)
print_all_over_the_net_metrics(global_times=global_times,
ref_global_times=ref_global_times,
global_accuracy=global_accuracy)
def two_ir_mode(args):
core = get_plugin(args.device, args.l, args.config)
ref_core = get_plugin(args.reference_device, args.l, args.reference_config)
net = get_net(model=args.model, core=core)
net_layers, net_inputs, net_outputs = get_model_info(net)
ref_net = get_net(model=args.reference_model, core=ref_core)
ref_net_layers, ref_net_inputs, ref_net_outputs = get_model_info(ref_net)
log.info('{} vs {}'.format(args.device, args.reference_device))
log.info('IR for {} : {}'.format(args.device, args.model))
log.info('IR for {} : {}'.format(args.reference_device, args.reference_model))
out_layers = get_layers_list(net_layers, net_inputs, net_outputs, args.layers)
ref_out_layers = get_layers_list(ref_net_layers, ref_net_inputs, ref_net_outputs, args.layers)
print_input_layers(net_inputs)
print_output_layers(out_layers)
layers_map = manage_user_outputs_with_mapping(mapping=args.mapping, reference_mapping=args.reference_mapping,
user_layers=out_layers)
inputs = input_processing(model_path=args.model, net_inputs=net_inputs, input_file=args.input,
layers_map=layers_map)
ref_inputs = input_processing(model_path=args.reference_model, net_inputs=ref_net_inputs, input_file=args.input,
layers_map=layers_map)
global_accuracy = []
global_times, ref_global_times = overall_accuracy_check(model=args.model, ref_model=args.reference_model,
out_layers=out_layers, ref_out_layers=ref_out_layers,
inputs=inputs, ref_inputs=ref_inputs, core=core,
device=args.device, ref_core=ref_core,
ref_device=args.reference_device, layers=args.layers,
num_of_iterations=args.num_of_iterations)
for out_layer in layers_map:
ref_out_layer = layers_map[out_layer]
if out_layer == ref_out_layer:
log.info('Layer {} statistics'.format(out_layer))
else:
log.info('Statistics \'{}\' vs \'{}\''.format(out_layer, ref_out_layer))
net_copy = get_net_copy_with_output(model=args.model, output=out_layer, core=core)
ref_net_copy = get_net_copy_with_output(model=args.reference_model, output=ref_out_layer, core=ref_core)
results = infer(net=net_copy, core=core, device=args.device, inputs=inputs, output=[out_layer])
if out_layer not in results:
continue
out_blob, pc = results[out_layer]
ref_results = infer(net=ref_net_copy, core=ref_core, device=args.reference_device,
inputs=ref_inputs, output=[ref_out_layer])
ref_out_blob, ref_pc = ref_results[ref_out_layer]
if ref_out_layer not in ref_results:
continue
a_m = accuracy_metrics(out_blob=out_blob, ref_out_blob=ref_out_blob)
performance_metrics(pc=pc, ref_pc=ref_pc)
blob_counters(out_blob=out_blob, ref_out_blob=ref_out_blob)
global_accuracy = update_global_accuracy_matrics(global_accuracy=global_accuracy, current_accuracy=a_m)
print_all_over_the_net_metrics(global_times=global_times, ref_global_times=ref_global_times,
global_accuracy=global_accuracy)
def majority_voting(scores_frame, true_labels_col, drop=False, t=0.5):
# Extract scores and true labels
true_labels = scores_frame[true_labels_col]
if drop:
scores = scores_frame.drop(labels=[true_labels_col] + drop, axis=1)
else:
scores = scores_frame.drop(labels=[true_labels_col], axis=1)
# Change scores to predicted class
preds = scores.copy()
preds[preds < t] = 0
preds[preds >= t] = 1
# Get majority class
majorities = []
for ID in preds.index:
row = preds.loc[ID]
mean = np.nanmean(row)
if mean < t:
majorities.append(0)
else:
majorities.append(1)
#majority = pd.DataFrame(majorities, columns=['preds'], index=preds.index)
# Get metrics
auc_score, sensitivity, specificity, mcc, con = u.performance_metrics(
y_true=true_labels, y_pred=majorities, threshold=t)
# Get ROC_AUC parameters
fpr, tpr, thresholds = roc_curve(y_true=true_labels, y_score=majorities)
return pd.Series(majorities, index=scores.index, name='majority'), [
auc_score, sensitivity, specificity, mcc
], (fpr, tpr)
def confident_majority_voting(scores_frame,
true_labels_col,
confidence=0.6,
drop=False,
verbose=False,
t=0.5):
# Extract scores and true labels
true_labels = scores_frame[true_labels_col]
if drop:
scores = scores_frame.drop(labels=[true_labels_col] + drop, axis=1)
else:
scores = scores_frame.drop(labels=[true_labels_col], axis=1)
# Find confident scores for each sample
confident_votes, labels, ids = [], [], []
no_con, no_cons = 0, {0: 0, 1: 0}
for ID in scores.index:
row = scores.loc[ID]
con = [v for v in row if v >= confidence or v <= 1 - confidence]
if len(con) < 1:
#print(row, '\n')
#con = [0.5]
no_con += 1
if true_labels.loc[ID] == 1:
no_cons[1] += 1
else:
no_cons[0] += 1
else:
labels.append(true_labels.loc[ID])
ids.append(ID)
# Change scores to classes
classes = [1 if c >= t else 0 for c in con]
mean = np.nanmean(classes)
if mean < t:
confident_votes.append(0)
else:
confident_votes.append(1)
# Look at samples with no confident scores
if no_con > 0 and verbose:
print('Fraction of samples with no confident votes: {0}/{1}'.format(
no_con, len(scores.index)))
print('Classbalance:', no_cons)
# Get metrics
auc_score, sensitivity, specificity, mcc, con = u.performance_metrics(
y_true=labels, y_pred=confident_votes, threshold=t)
# Get ROC_AUC parameters
fpr, tpr, thresholds = roc_curve(y_true=labels, y_score=confident_votes)
return pd.Series(confident_votes,
index=ids,
name='con_vot_' + str(round(confidence, 2))), [
auc_score, sensitivity, specificity, mcc
], (fpr, tpr)
def evaluate_performance(self):
self.performance = {}
for set in ["train", "test"]:
auc, unc, fpr, tpr, tpr_unc = utils.performance_metrics(
self.data[set]["label"], self.prediction[set],
self.data[set]["evt_weight_"], 10)
#auc, unc, fpr, tpr, thresh = utils.auc_and_unc(self.data[set]["label"], self.prediction[set], self.data[set]["evt_weight_"], 10)
self.performance[set] = {
"auc": auc,
"auc_unc": unc,
"fpr": fpr,
"tpr": tpr,
"tpr_unc": tpr_unc,
}
print(
"[MVA_HELPER] ASSESSING PERFORMANCE -- %s set: AUC = %.3f +/- %.3f"
% (set, self.performance[set]["auc"],
self.performance[set]["auc_unc"]))
self.make_plots()
self.save_performance()
def average_scoring(scores_frame, true_labels_col, drop=False, t=0.5):
# Extract scores and true labels
true_labels = scores_frame[true_labels_col]
if drop:
scores = scores_frame.drop(labels=[true_labels_col] + drop, axis=1)
else:
scores = scores_frame.drop(labels=[true_labels_col], axis=1)
# Get mean of each score
means = np.nanmean(scores, axis=1)
# Get metrics
auc_score, sensitivity, specificity, mcc, con = u.performance_metrics(
y_true=true_labels, y_pred=means, threshold=t)
# Get ROC_AUC parameters
fpr, tpr, thresholds = roc_curve(y_true=true_labels, y_score=means)
return pd.Series(means, index=scores.index,
name='mean'), [auc_score, sensitivity, specificity,
mcc], (fpr, tpr)
def train_and_evaluate_classifier(name, clf, validation_data,
validation_labels):
clf.fit(validation_data)
predictions = np.array(clf.predict(validation_data))
if name in PYOD_ALGOS:
predictions = np.where(predictions == 1, -1, predictions)
predictions = np.where(predictions == 0, 1, predictions)
tp, tn, fp, fn, tpr, tnr, ppv, npv, ts, pt, acc, f1, mcc = performance_metrics(
validation_labels, predictions)
result = [name, tp, tn, fp, fn, tpr, tnr, ppv, npv, ts, pt, acc, f1, mcc]
print("%s results:" % (name))
print(
tabulate([result], [
"ALGO", "TP", "TN", "FP", "FN", "TPR", "TNR", "PPV", "NPV", "TS",
"PT", "ACC", "F1", "MCC"
],
tablefmt="grid"))
return result
def sklearn_model(X_data, y_data, model, seed=seed, cv_splits=5, stratified=True, permute=False, verbose=False):
"""
Parameter optimization can be built on top of this function.
X_data: Model data, input as pandas dataframe.
y_data: True labels, input as numpy 1d array.
model: Sklearn class classifier object with function ".predict_proba()".
seed: Random seed, to make reproducable results.
cv_splits: Number of crossvalidation splits, max = N.
stratified: Whether or not to make k-fold splits stratified by target classes. Only used when cv_splits < N.
verbose: if True, prints results.
"""
# Check IDs are in same order for X and y
intersect = X_data.index.intersection(y_data.index)
X_data, y_data = X_data.loc[intersect], y_data.loc[intersect]
print(X_data.shape, y_data.shape)
# Data shapes
N, M = X_data.shape
# If permutation test is enabled
if permute:
y_shuffle = y_data['y_shuffle']
y_data = y_data['y']
# Set cross-validation (partition vector)
if cv_splits == N:
part_vec = np.array(list(range(N)))
else:
if stratified:
CV = StratifiedKFold(n_splits=cv_splits, shuffle=True, random_state=seed) # Split stratified by target value
else:
CV = KFold(n_splits=cv_splits, shuffle=True, random_state=seed) # Random split
part_vec, k = np.zeros(N), 0
for train_index, test_index in CV.split(X_data, y_data): # y must here be made a 1d array if error occurs
part_vec[test_index] = k
k += 1
# Set lists and parameters
train_scores, test_scores = [], []
train_pred, test_pred = [], []
y_target_train, y_target_test = [], []
ids_test, model_list = [], []
X, y = X_data.values, y_data.values.ravel()
if permute:
ys = y_shuffle.values.ravel()
# Run through cross-validation
for k in range(cv_splits):
if verbose:
print('\n# K-fold {0}/{1}'.format(k+1, cv_splits))
# Get training and test data
train_index, test_index = np.where(part_vec != k), np.where(part_vec == k)
X_train, X_test = X[train_index], X[test_index]
print(X_train.shape, X_test.shape)
if permute:
y_train, y_test = y[train_index], ys[test_index]
else:
y_train, y_test = y[train_index], y[test_index]
# Get the IDs of the test group
test_ids = X_data.index[test_index]
ids_test.append(test_ids)
# Classbalance
unique, counts = np.unique(y_train, return_counts=True)
print('\tClassbalance in training is:\t', dict(zip(unique, counts)))
unique, counts = np.unique(y_test, return_counts=True)
print('\tClassbalance in test is:\t', dict(zip(unique, counts)), '\n')
# Train model
model.fit(X_train, y_train)
# Calculate metrics
if cv_splits == N:
# Get prediction on training and test set
pred_train = model.predict_proba(X_train)[:,1]
pred_test = model.predict_proba(X_test)[:,1]
else:
# Get prediction on training and test set
pred_train = model.predict_proba(X_train)[:,1]
pred_test = model.predict_proba(X_test)[:,1]
#auc.append(roc_auc_score(y_test, y_test_pred))
auc_train, sens_train, spec_train, mcc_train, con_train = u.performance_metrics(y_true=y_train, y_pred=pred_train)
auc_test, sens_test, spec_test, mcc_test, con_test = u.performance_metrics(y_true=y_test, y_pred=pred_test)
# Append metrics to lists
if verbose:
print('AUC train = {0}, AUC test = {1}'.format(auc_train, auc_test))
train_scores.append([auc_train, sens_train, spec_train, mcc_train])
test_scores.append([auc_test, sens_test, spec_test, mcc_test])
# Append labels and predictions
#model_list.append(model)
model_list.append(model.feature_importances_.tolist()) # save feature importances instead
train_pred.append(pred_train)
test_pred.append(pred_test)
y_target_train.append(y_train)
y_target_test.append(y_test)
k += 1
# convert to long np array:
y_train_pred=np.array(train_pred)
y_test_pred=np.array(test_pred)
y_target_train=np.array(y_target_train)
y_target_test=np.array(y_target_test)
ids_test=np.array(ids_test)
# Flatten arrays
y_train_pred=np.concatenate(y_train_pred).flatten()
y_test_pred=np.concatenate(y_test_pred).flatten()
y_target_train=np.concatenate(y_target_train).flatten()
y_target_test=np.concatenate(y_target_test).flatten()
ids_test=np.concatenate(ids_test).flatten()
# Make dataframe for metrics
auc_train, sens_train, spec_train, mcc_train, con_train = u.performance_metrics(y_true=y_target_train, y_pred=y_train_pred)
auc_test, sens_test, spec_test, mcc_test, con_test = u.performance_metrics(y_true=y_target_test, y_pred=y_test_pred)
metrics_train = pd.DataFrame(data=np.array([[auc_train], [sens_train], [spec_train], [mcc_train]]).T, columns=['auc', 'sens', 'spec', 'mcc'], index=[seed])
metrics_test = pd.DataFrame(data=np.array([[auc_test], [sens_test], [spec_test], [mcc_test]]).T, columns=['auc', 'sens', 'spec', 'mcc'], index=[seed])
metrics = (metrics_train, metrics_test)
if verbose:
print("\n# Train AUC: " + str(round(auc_train,4)))
print("# Test AUC: " + str(round(auc_test,4)))
# calculate false positive and true positive rate at different thresholds (ROC curve):
fpr,tpr,thresholds = roc_curve(y_target_test, y_test_pred, drop_intermediate=False)
return metrics, (fpr, tpr), ids_test, (y_test_pred, y_target_test), thresholds, model_list
