def run_5_combos(dataset_name, df, model_dir, params: Bunch):
runner = Runner(dataset_name, df, model_dir, params)
make_clear_dir(model_dir)
# exp_dir = make_sibling_dir(__file__, 'experiments')
# exp = Experiment(name=dataset_name, debug=False, save_dir=exp_dir)
# exp.tag(params)
# natural training
runner.train(params.mod(perturb_frac=0.0))
nat_nat = runner.eval(params.mod(perturb_frac=0.0))
nat_nat.update(train=0.0, test=0.0)
# exp.log(nat_nat)
printed_cols = list(
set(nat_nat.keys()).difference(
['f_a_dict', 'f_g_dict', 'wts_dict', 'corrs_dict']))
print(pd.DataFrame([nat_nat])[printed_cols])
nat_per = runner.eval(params)
nat_per.update(train=0.0, test=params.perturb_frac)
# exp.log(nat_per)
print(pd.DataFrame([nat_per])[printed_cols])
# adversarial training: Start with naturally trained classifier
# for epochs/2, then train on adversarial inputs for remaining
# epochs/2
make_clear_dir(model_dir)
clean_train_epochs = int(
params.get('clean_pre_train', 0.5) * params.epochs)
dirty_train_epochs = params.epochs - clean_train_epochs
if clean_train_epochs > 0:
runner.train(params.mod(perturb_frac=0.0, epochs=clean_train_epochs))
runner.train(params.mod(epochs=dirty_train_epochs))
per_nat = runner.eval(params.mod(perturb_frac=0.0))
per_nat.update(train=params.perturb_frac, test=0.0)
# exp.log(per_nat)
print(pd.DataFrame([per_nat])[printed_cols])
per_per = runner.eval(params)
per_per.update(train=params.perturb_frac, test=params.perturb_frac)
# exp.log(per_per)
print(pd.DataFrame([per_per])[printed_cols])
per_per_all = runner.eval(params.mod(perturb_frac=1.0))
per_per_all.update(train=params.perturb_frac, test=1.0)
# exp.log(per_per_all)
print(pd.DataFrame([per_per_all])[printed_cols])
all_results = dict(nat_nat=nat_nat,
nat_per=nat_per,
per_nat=per_nat,
per_per=per_per,
per_per_all=per_per_all)
return all_results
def run_one(df: pd.DataFrame, params: Bunch, name='one'):
'''
Only do a single train (nat or adv) and test (nat or adv) combo
and return some metrics/values
:param df:
:param params:
:return:
'''
tf.set_random_seed(123)
np.random.seed(123)
model_dir = os.path.join('/tmp/robulin/exp', name)
runner = Runner(name, df, model_dir, params)
make_clear_dir(model_dir)
# adversarial training: pre-train on nat examples for
# some fraction of epochs, then on adversarial for remaining epochs
clean_train_epochs = int(
params.get('clean_pre_train', 0.5) * params.epochs)
dirty_train_epochs = params.epochs - clean_train_epochs
if clean_train_epochs > 0:
runner.train(params.mod(perturb_frac=0.0, epochs=clean_train_epochs))
runner.train(params.mod(epochs=dirty_train_epochs))
result = runner.eval(params.mod(perturb_frac=params.test_perturb_frac))
result.update(train=params.perturb_frac, test=params.test_perturb_frac)
few_fields = params.get('fields', [
'train', 'test', 'loss', 'auc', 'acc', 'wts_ent', 'wts_l1',
'wts_l1_linf', 'wts_1pct', 'wts_pct1pct', 'av_ent', 'av_high', 'a_ent',
'g_ent', 'f_a_ent', 'f_g_ent'
])
result_few = sub_dict(result, few_fields)
simple_keys = [
k for k, v in params.items() if type(v) in [int, float, str, bool]
]
result_few.update(sub_dict(params, simple_keys))
print(result_few)
result.update(sub_dict(params, simple_keys))
return result
def run_grid(params: Bunch):
'''
Run a grid of experiments based on params.grid and return collated
values/metrics in a data-frame
:param params:
:return:
'''
if params.get('dataset'):
df = fetch_data(params.dataset)
else:
df = gen_synth_df(params)
grid_dict = params.grid
params_list = list(ParameterGrid(grid_dict))
results = []
for p in params_list:
result = run_one(df,
params.mod(p),
name=params.get('dataset', 'synth'))
results += [result]
results = pd.DataFrame(results)
return results
def main(argv):
args = parser.parse_args(argv[1:])
log_codes = dict(e=tf.logging.ERROR,
i=tf.logging.INFO,
w=tf.logging.WARN,
d=tf.logging.DEBUG)
params = Bunch(yaml.load(open(args.hparams)))
tf.logging.set_verbosity(
log_codes.get(params.log.lower()[0], tf.logging.ERROR))
if params.eager:
tf.enable_eager_execution()
print('******** TF EAGER MODE ENABLED ***************')
timestr = time.strftime("%Y%m%d-%H%M%S")
default_out_file_base = os.path.join(
os.path.basename(args.hparams).split('.')[0], timestr)
out_file_base = args.out or default_out_file_base
results = run_grid(params)
if not params.get('dataset'): # i.e if synthetic, not UCI/pmlb
results['Wts_x_L1'] = results['wts_dict']. \
map(lambda w: np.sum(np.abs(np.array(
list(sub_dict_prefix(w, 'x').values())))))
results['Wts_r_L1'] = results['wts_dict']. \
map(lambda w: np.sum(np.abs(np.array(
list(sub_dict_prefix(w, 'r').values())))))
results['Wts_w_L1'] = results['wts_dict']. \
map(lambda w: np.sum(np.abs(np.array(
list(sub_dict_prefix(w, 'w').values())))))
results_simple = results.drop(['wts_dict', 'f_g_dict', 'f_a_dict'], axis=1)
results_dir = make_sibling_dir(__file__, 'results')
results_simple_file = os.path.join(results_dir, out_file_base + '.csv')
os.makedirs(os.path.dirname(results_simple_file), exist_ok=True)
with open(results_simple_file, 'w+') as fd:
results_simple.to_csv(fd, float_format='%.3f', index=False)
pkl_file = os.path.join(results_dir, out_file_base + '.pkl')
results.to_pickle(pkl_file)
print(df_simple(results))
print(f'******** Summary csv written to {results_simple_file}')
print(f'******** Pickled results dataframe at {pkl_file}')
def __init__(self,
dataset_name,
df: pd.DataFrame,
model_dir: str,
params: Bunch = None):
self.col_spec, self.target_name = df_column_specs(df)
self.segments = np.array([], dtype=np.int32)
self.feature_value_names = []
for i, s in enumerate(self.col_spec):
self.segments = np.append(self.segments, [np.repeat(i, s['card'])])
col_name = s['name']
if s['card'] == 1:
self.feature_value_names += [col_name]
else:
self.feature_value_names += [
col_name + '=' + str(i) for i in range(s['card'])
]
# prepend numeric to enforce lexicographic order so
# we can recover the model weights from tensorflow's variables
# in the right order
df = df.copy(deep=True)
df.columns = [c if c == self.target_name else f'{i:05d}_' + c \
for i, c in enumerate(df.columns)]
self.feature_columns = tf_feature_columns(df)
self.df_train, self.df_test = \
split_and_standardize(df, params.get('std', True))
dir = make_sibling_dir(__file__, f'datasets/{dataset_name}')
self.train_file = f'{dir}/train.csv'
self.test_file = f'{dir}/test.csv'
with open(self.train_file, mode='w+') as fd:
self.df_train.to_csv(fd, header=True, index=False)
with open(self.test_file, mode='w+') as fd:
self.df_test.to_csv(fd, header=True, index=False)
self.model_dir = model_dir
def gen_synth_df(params: Bunch):
N = params.get('N', 1000)
nC = params.get('nc', 8) # number of features predictive of label
nF = params.get('nr', 8) # how many features are random, uncorr with
nW = params.get('nw', 0) # how many features weakly correlated with label
corr1 = params.get('corrp', True) # whether predictive feats are corr
corr = params.get('corr', False) # whether random feats are corr
cat = params.get('cat',
False) # whether the random features are categorical
pred = params.get('pred', 0.7) # how often do the "predictive" features
weak_pred = params.get('weak_pred', 1.0) # predictivity of weak feature
np.random.seed(123)
y = np.random.choice(2, N, p=[0.5, 0.5])
y1 = (2 * y - 1.0).reshape([-1, 1])
df_agree = pd.DataFrame()
p = pred # probability of agreement with label
if nC > 0:
if corr1: # identical, i.e. highly correlated
agree = np.repeat(
np.array(np.random.choice(2, N, p=[1-p, p]), dtype=np.float32). \
reshape([-1,1]), nC, axis=1)
else: # uncorrelated
agree = np.array(np.random.choice(2, N * nC, p=[1-p, p]),
dtype=np.float32). \
reshape([-1, nC])
agree = y1 * (2 * agree - 1)
agree_cols = ['x' + str(i + 1) for i in range(nC)]
df_agree = pd.DataFrame(agree, columns=agree_cols)
#
df_tar = pd.DataFrame(dict(target=y))
# rest are random
df_rest = pd.DataFrame()
if nF > 0:
if cat: # 1 categorical feature with nF possible values
rest = np.array(np.random.choice(nF, N), dtype=np.int64). \
reshape([-1,1])
else:
if corr: # identical, i.e. highly correlated
rest = np.repeat(
np.array(np.random.choice(2, N, p=[0.5, 0.5]), dtype=np.float32). \
reshape([-1,1]), nF, axis=1)
else: # uncorrelated, i.i.d -1/1
rest = np.array(np.random.choice(2, N * nF, p=[0.5, 0.5]),
dtype=np.float32).reshape([-1, nF])
rest = 2 * rest - 1.0
rest_cols = ['r'] if cat else ['r' + str(i + 1) for i in range(nF)]
df_rest = pd.DataFrame(rest, columns=rest_cols)
df_weak = pd.DataFrame()
if nW > 0: # normal, slightly correlated with label
means = y1 * weak_pred / math.sqrt(nW)
weak = np.repeat(np.random.normal(means, 1.0), nW, axis=1)
weak_cols = ['w'] if cat else ['w' + str(i + 1) for i in range(nW)]
df_weak = pd.DataFrame(weak, columns=weak_cols)
df = pd.concat([df_agree, df_rest, df_weak, df_tar], axis=1)
return df
def binary_classification_model(features, labels, mode, params: utils.Bunch):
"""Custom model; initially just linear (logistic or poisson)
"""
params = utils.Bunch(**params)
optimizer = params.get('optimizer', 'ftrl')
l1_reg = params.get('l1_reg', 0.0)
l2_reg = params.get('l2_reg', 0.0)
tf.set_random_seed(123)
labels = tf.cast(labels, tf.float32)
epsilon = params.perturb_norm_bound
norm_order = params.get('perturb_norm_order', 2)
train_perturb_frac = params.train_perturb_frac
test_perturb_frac = params.test_perturb_frac
# do the various feature transforms according to the
# 'feature_column' param, so now we have the feature-vector
# that we will do computations on.
x = tf.feature_column.input_layer(features, params.feature_columns)
# for units in params['hidden_units']:
# net = tf.layers.dense(net, units=units, activation=tf.nn.relu)
# Compute logits (1 per class).
#logits = tf.layers.dense(net, params['n_classes'], activation=None)
#logits = tf.layers.dense(net, 1, activation=None, name='dense')
dense = tf.layers.Dense(1, activation=None,
kernel_initializer=\
tf.keras.initializers.zeros(),
#tf.keras.initializers.RandomNormal(seed=123),
bias_initializer= \
tf.keras.initializers.zeros())
#tf.keras.initializers.RandomNormal(seed=123))
if len(dense.trainable_variables) == 0:
dense(x) # to force the kernel initialization
# this is the "kernel" i.e. weights, does not include bias
coefs = dense.trainable_variables[0]
bias = dense.trainable_variables[1][0]
perturb_frac = train_perturb_frac if mode == tf.estimator.ModeKeys.TRAIN \
else test_perturb_frac
x_perturbed, _ = RobustLogisticModel.perturb_continuous(
x,
labels,
coefs,
norm_bound=epsilon,
norm_order=norm_order,
perturb_frac=perturb_frac,
seed=123)
logits = dense(x_perturbed)
if params.activation == 'sigmoid':
predictions = tf.sigmoid(logits)
elif params.activation == 'sign':
predictions = tf.maximum(0.0, tf.sign(logits))
else: # assume relu
predictions = tf.nn.relu(logits)
labels = tf.reshape(labels, [-1, 1])
# Compute predictions.
predicted_classes = tf.maximum(tf.sign(predictions - 0.5), 0)
# if mode == tf.estimator.ModeKeys.PREDICT:
# predictions = {
# 'class_ids': predicted_classes[:, tf.newaxis],
# 'probabilities': tf.nn.softmax(logits), # not really used
# 'logits': logits,
# }
# return tf.estimator.EstimatorSpec(mode, predictions=predictions)
# Compute loss.
if params.activation == 'sigmoid':
loss = tf.reduce_mean(
tf.keras.backend.binary_crossentropy(target=labels,
output=logits,
from_logits=True))
elif params.activation == 'sign':
loss = tf.reduce_mean(-(2 * labels - 1) * logits)
else:
raise Exception(f'loss not known for activation {params.activation}')
if l1_reg > 0 and optimizer != 'ftrl':
loss = loss + l1_reg * tf.norm(coefs, ord=1)
if l2_reg > 0 and optimizer != 'ftrl':
loss = loss + l2_reg * tf.sqrt(tf.maximum(0.0, tf.nn.l2_loss(coefs)))
adv_reg_lambda = params.get('adv_reg_lambda', 0.0)
if adv_reg_lambda and perturb_frac > 0.0:
clean_logits = dense(x)
clean_loss = tf.reduce_mean(
tf.keras.backend.binary_crossentropy(target=labels,
output=clean_logits,
from_logits=True))
loss = clean_loss + adv_reg_lambda * loss
# Compute evaluation metrics.
accuracy = tf.metrics.accuracy(labels=labels,
predictions=predicted_classes,
name='acc_op')
auc = tf.metrics.auc(labels=labels, predictions=predictions, name='auc-op')
# add metrics etc for tensorboard
tf.summary.scalar('accuracy', accuracy[1])
tf.summary.scalar('auc', auc[1])
tf.summary.scalar('loss', loss)
if mode == tf.estimator.ModeKeys.EVAL:
# axiomatic attribution (Integrated Grads)
feat_val_attribs = attribution.logistic_attribution(x, coefs, bias)
feat_val_corr_stats = attribution.label_corr_stats(x, labels)
av_attribs = tf.reduce_mean(feat_val_attribs, axis=0)
attrib_entropy = tf_entropy(feat_val_attribs)
num_high_attribs = num_above_relative_threshold(feat_val_attribs,
thresh=params.get(
'thresh', 0.1))
av_attrib_entropy = tf.metrics.mean(attrib_entropy)
av_high_attribs = tf.metrics.mean(num_high_attribs)
mean_attribs = tf.metrics.mean_tensor(av_attribs, name='attrib')
xy_av = tf.metrics.mean_tensor(feat_val_corr_stats.xy, name='xy_av')
x_av = tf.metrics.mean_tensor(feat_val_corr_stats.x, name='x_av')
y_av = tf.metrics.mean_tensor(feat_val_corr_stats.y, name='y_av')
xsq_av = tf.metrics.mean_tensor(feat_val_corr_stats.xsq, name='xsq_av')
ysq_av = tf.metrics.mean_tensor(feat_val_corr_stats.ysq, name='ysq_av')
# ad-hoc attribution (AFVI)
afvi = attribution.logistic_afvi(x, coefs, bias)
mean_afvi = tf.metrics.mean_tensor(afvi, name='afvi')
metrics = dict(accuracy=accuracy,
auc=auc,
attrib_ent=av_attrib_entropy,
high_attribs=av_high_attribs,
attrib=mean_attribs,
afvi=mean_afvi,
xy_av=xy_av,
x_av=x_av,
y_av=y_av,
xsq_av=xsq_av,
ysq_av=ysq_av)
# the histograms don't work in eval mode??
tf.summary.histogram('attrib', mean_attribs[1])
tf.summary.histogram('afvi', mean_afvi[1])
return tf.estimator.EstimatorSpec(mode,
loss=loss,
eval_metric_ops=metrics)
# Create training op.
assert mode == tf.estimator.ModeKeys.TRAIN
#
if optimizer == 'adam':
loss_optimizer = tf.train.AdamOptimizer(learning_rate=params.lr)
elif optimizer == 'ftrl':
loss_optimizer = tf.train.FtrlOptimizer(
learning_rate=params.lr,
l1_regularization_strength=l1_reg,
l2_regularization_strength=l2_reg)
elif optimizer == 'adagrad':
loss_optimizer = tf.train.AdagradOptimizer(learning_rate=params.lr)
elif optimizer == 'sgd':
loss_optimizer = tf.train.GradientDescentOptimizer(
learning_rate=params.lr)
else:
raise Exception(f"Unknown optimizer: {optimizer}")
train_op = loss_optimizer.minimize(loss,
global_step=tf.train.get_global_step())
return tf.estimator.EstimatorSpec(mode, loss=loss, train_op=train_op)
def plot_multi(df: pd.DataFrame,
x_ids=None,
var='type',
value='value',
include=None,
ignore=None,
order_by=None,
ascending=False,
title=None,
rename=dict(),
legend=True,
threshold=0.001,
kind='bar',
show=True,
tight=True,
ax=None,
params: Bunch = Bunch()):
# rcParams.update({'figure.autolayout': True})
if rename:
df = df.rename(columns=rename)
if x_ids is None:
df['__x'] = df.index
x_ids = '__x'
if order_by is not None:
df = df.sort_values(by=order_by, ascending=ascending)
if ignore:
df = df.drop(ignore, axis=1)
if include:
df = df[include]
plt.interactive(False)
dfm = pd.melt(df, id_vars=x_ids, var_name=var, value_name=value)
biggest = np.max(np.abs(dfm[value]))
dfm = dfm[abs(dfm[value]) >= biggest * threshold]
x_labels = df[x_ids].astype(str).tolist()
x_labels = [x for x in x_labels if x in dfm[x_ids].astype(str).tolist()]
max_x_label = max([len(s) for s in x_labels])
if kind == 'bar':
g = sns.catplot(x=x_ids,
y=value,
hue=var,
data=dfm,
kind=kind,
order=x_labels,
row_order=x_labels,
legend=False,
legend_out=True,
ax=ax)
if params.get('xtick_font_size'):
g.set_xticklabels(fontdict=dict(fontsize=params.xtick_font_size))
if max_x_label > 4:
g.set_xticklabels(rotation=90)
else: # line
g = sns.lineplot(x=x_ids,
y=value,
hue=var,
data=dfm,
legend=False,
ax=ax)
#g.set_xticklabels(labels=x_labels)
if title:
plt.title(title)
if tight:
plt.tight_layout()
if legend:
plt.legend(loc='upper right', title=var)
if show:
plt.show()
return g