def get_metric_report(info_dicts, args, keys=['-'], LR=None):
class_metric_strs, total_metrics = [], []
report_metrics = [
'jacc', 'acc', 'mcc', 'f1', 'recall', 'precision', 'var'
] if args.all_metrics else [args.report_metric]
for m in report_metrics:
for d in info_dicts:
d.update({'metric': m})
class_metrics = [get_metric(d) for d in info_dicts]
total_metrics.append(get_metric(info_dicts))
if LR is not None:
delim = '-'
else:
delim = {
'mcc': '#',
'f1': '+',
'jacc': '=',
'acc': '>',
'var': '%',
'recall': '<',
'precision': '~'
}[m]
class_metric_strs.append(", ".join(
'{} {} {:5.2f}'.format(k, delim, f * 100)
for k, f in zip(keys, class_metrics)))
return total_metrics, class_metric_strs
def _binary_threshold(preds,
labels,
metric,
micro,
global_tweaks=1000,
debug=False,
heads_per_class=1,
class_single_threshold=False):
avg_metric = 0
best_thresholds = []
info_dicts = []
category_metrics = []
# Compute threshold per class... *unless* multiple heads per class and one threshold required.
num_categories = labels.shape[1]
for category in range(num_categories):
category_best_threshold = category_best_metric = 0
for threshold in np.linspace(0.005, 1, 200):
if heads_per_class > 1 and class_single_threshold:
info_dict = update_info_dict(
defaultdict(int),
labels[:, (category * heads_per_class):(category + 1) *
heads_per_class],
preds[:, (category * heads_per_class):(category + 1) *
heads_per_class],
threshold=threshold)
else:
info_dict = update_info_dict(defaultdict(int),
labels[:, category],
preds[:, category],
threshold=threshold)
metric_score = get_metric(info_dict, metric, micro)
if metric_score > category_best_metric or category_best_metric == 0:
category_best_metric, category_best_threshold, category_best_info_dict = metric_score, threshold, info_dict
info_dicts.append(category_best_info_dict)
category_metrics.append(category_best_metric)
best_thresholds.append(category_best_threshold)
# HACK -- use micro average here, even if not elsewhere
micro = True
best_metric = get_metric(info_dicts, metric, micro)
# HACK: Attempt to tune thresholds simultaneously... for overall micro average
if num_categories < 2:
global_tweaks = 0
if debug and global_tweaks > 0:
print('best after invididual thresholds (micro %s)' % micro)
print(best_thresholds)
print(get_metric(info_dicts, metric, micro))
for i in range(global_tweaks):
# Choose random category
category = np.random.randint(num_categories)
curr_threshold = best_thresholds[category]
# tweak randomly
new_threshold = curr_threshold + (0.08 * (np.random.random() - 0.5))
if heads_per_class > 1 and class_single_threshold:
info_dict = update_info_dict(
defaultdict(int),
labels[:, (category * heads_per_class):(category + 1) *
heads_per_class],
preds[:, (category * heads_per_class):(category + 1) *
heads_per_class],
threshold=new_threshold)
else:
info_dict = update_info_dict(defaultdict(int),
labels[:, category],
preds[:, category],
threshold=new_threshold)
old_dict = info_dicts[category]
info_dicts[category] = info_dict
# compute *global* metrics
metric_score = get_metric(info_dicts, metric, micro)
# save new threshold if global metrics improve
if metric_score > best_metric:
# print('Better threshold %.3f for category %d' % (new_threshold, category))
best_thresholds[category] = round(new_threshold, 3)
best_metric = metric_score
else:
info_dicts[category] = old_dict
if debug and global_tweaks > 0:
print('final thresholds')
print(best_thresholds)
print(get_metric(info_dicts, metric, micro))
# OK, now *if* we used multiple heads per class (same threshold) copy these back out to final answers
if heads_per_class > 1 and class_single_threshold:
best_thresholds = np.concatenate([[best_thresholds[i]] *
heads_per_class
for i in range(num_categories)])
else:
best_thresholds = np.array(best_thresholds)
# print(best_thresholds)
return get_metric(info_dicts, metric,
micro), best_thresholds, category_metrics, info_dicts
def train_logreg(trX,
trY,
vaX=None,
vaY=None,
teX=None,
teY=None,
penalty='l1',
max_iter=100,
C=2**np.arange(-8, 1).astype(np.float),
seed=42,
model=None,
eval_test=True,
neurons=None,
drop_neurons=False,
report_metric='acc',
automatic_thresholding=False,
threshold_metric='acc',
micro=False):
# if only integer is provided for C make it iterable so we can loop over
if not isinstance(C, collections.Iterable):
C = list([C])
# extract features for given neuron indices
if neurons is not None:
if drop_neurons:
all_neurons = set(list(range(trX.shape[-1])))
neurons = set(list(neurons))
neurons = list(all_neurons - neurons)
trX = trX[:, neurons]
if vaX is not None:
vaX = vaX[:, neurons]
if teX is not None:
teX = teX[:, neurons]
# Cross validation over C
n_classes = 1
if len(trY.shape) > 1:
n_classes = trY.shape[-1]
scores = []
if model is None:
for i, c in enumerate(C):
if n_classes <= 1:
model = LogisticRegression(C=c,
penalty=penalty,
max_iter=max_iter,
random_state=seed)
model.fit(trX, trY)
blank_info_dict = {
'fp': 0,
'tp': 0,
'fn': 0,
'tn': 0,
'std': 0.,
'metric': threshold_metric,
'micro': micro
}
if vaX is not None:
info_dict = update_info_dict(
blank_info_dict.copy(), vaY,
model.predict_proba(vaX)[:, -1])
else:
info_dict = update_info_dict(
blank_info_dict.copy(), trY,
model.predict_proba(trX)[:, -1])
scores.append(get_metric(info_dict))
print(scores[-1])
del model
else:
info_dicts = []
model = []
for cls in range(n_classes):
_model = LogisticRegression(C=c,
penalty=penalty,
max_iter=max_iter,
random_state=seed)
_model.fit(trX, trY[:, cls])
blank_info_dict = {
'fp': 0,
'tp': 0,
'fn': 0,
'tn': 0,
'std': 0.,
'metric': threshold_metric,
'micro': micro
}
if vaX is not None:
info_dict = update_info_dict(
blank_info_dict.copy(), vaY[:, cls],
_model.predict_proba(vaX)[:, -1])
else:
info_dict = update_info_dict(
blank_info_dict.copy(), trY[:, cls],
_model.predict_proba(trX)[:, -1])
info_dicts.append(info_dict)
model.append(_model)
scores.append(get_metric(info_dicts))
print(scores[-1])
del model
c = C[np.argmax(scores)]
if n_classes <= 1:
model = LogisticRegression(C=c,
penalty=penalty,
max_iter=max_iter,
random_state=seed)
model.fit(trX, trY)
else:
model = []
for cls in range(n_classes):
_model = LogisticRegression(C=c,
penalty=penalty,
max_iter=max_iter,
random_state=seed)
_model.fit(trX, trY[:, cls])
model.append(_model)
else:
c = model.C
# predict probabilities and get accuracy of regression model on train, val, test as appropriate
# also get number of regression weights that are not zero. (number of features used for modeling)
scores = []
if n_classes == 1:
nnotzero = np.sum(model.coef_ != 0)
preds = model.predict_proba(trX)[:, -1]
train_score = get_metric(
update_info_dict(blank_info_dict.copy(), trY, preds),
report_metric)
else:
nnotzero = 0
preds = []
info_dicts = []
for cls in range(n_classes):
nnotzero += np.sum(model[cls].coef_ != 0)
_preds = model[cls].predict_proba(trX)[:, -1]
info_dicts.append(
update_info_dict(blank_info_dict.copy(), trY[:, cls], _preds))
preds.append(_preds)
nnotzero /= n_classes
train_score = get_metric(info_dicts, report_metric)
preds = np.concatenate([p.reshape((-1, 1)) for p in preds], axis=1)
scores.append(train_score * 100)
if vaX is None:
eval_data = trX
eval_labels = trY
val_score = train_score
else:
eval_data = vaX
eval_labels = vaY
if n_classes == 1:
preds = model.predict_proba(vaX)[:, -1]
val_score = get_metric(
update_info_dict(blank_info_dict.copy(), vaY, preds),
report_metric)
else:
preds = []
info_dicts = []
for cls in range(n_classes):
_preds = model[cls].predict_proba(vaX)[:, -1]
info_dicts.append(
update_info_dict(blank_info_dict.copy(), vaY[:, cls],
_preds))
preds.append(_preds)
val_score = get_metric(info_dicts, report_metric)
preds = np.concatenate([p.reshape((-1, 1)) for p in preds], axis=1)
val_preds = preds
val_labels = eval_labels
scores.append(val_score * 100)
eval_score = val_score
threshold = np.array([.5] * n_classes)
if automatic_thresholding:
_, threshold, _, _ = _binary_threshold(
preds.reshape(-1, n_classes), eval_labels.reshape(-1, n_classes),
threshold_metric, micro)
threshold = float(threshold.squeeze())
if teX is not None and teY is not None and eval_test:
eval_data = teX
eval_labels = teY
if n_classes == 1:
preds = model.predict_proba(eval_data)[:, -1]
else:
preds = []
for cls in range(n_classes):
_preds = model[cls].predict_proba(eval_data)[:, -1]
preds.append(_preds)
preds = np.concatenate([p.reshape((-1, 1)) for p in preds], axis=1)
if n_classes == 1:
threshold = float(threshold.squeeze())
eval_score = get_metric(
update_info_dict(blank_info_dict.copy(),
eval_labels,
preds,
threshold=threshold), report_metric)
else:
info_dicts = []
for cls in range(n_classes):
info_dicts.append(
update_info_dict(blank_info_dict.copy(),
eval_labels[:, cls],
preds[:, cls],
threshold=threshold[cls]))
eval_score = get_metric(info_dicts, report_metric)
scores.append(eval_score * 100)
return model, scores, preds, c, nnotzero