def main(argv):
args = parser.parse_args(argv[1:])
log_codes = dict(e=tf.logging.ERROR,
i=tf.logging.INFO,
w=tf.logging.WARN,
d=tf.logging.DEBUG)
tf.logging.set_verbosity(
log_codes.get(args.log.lower()[0], tf.logging.ERROR))
dataset_name = args.dataset
epochs = args.epochs
batch_size = args.batch_size
if args.eager:
tf.enable_eager_execution()
print('******** TF EAGER MODE ENABLED ***************')
params = Bunch(perturb_l2_bound=0.2,
epochs=epochs,
batch_size=batch_size,
multi_gpu=args.multi_gpu,
lr=0.01)
res_10 = get_run_results(dataset_name, params.mod(perturb_frac=0.1))
res_20 = get_run_results(dataset_name, params.mod(perturb_frac=0.2))
res_50 = get_run_results(dataset_name, params.mod(perturb_frac=0.5))
res_80 = get_run_results(dataset_name, params.mod(perturb_frac=0.8))
res_100 = get_run_results(dataset_name, params.mod(perturb_frac=1.0))
def get_field(dfs, conditions, column):
return [df.query(conditions).iloc[0][column] for df in dfs]
dfs = [res_10, res_20, res_50, res_80, res_100]
df = pd.DataFrame(
dict(TrainAdvPct=[10, 20, 50, 80, 100],
TestNat=get_field(dfs, 'train > 0 & test == 0.0', 'acc'),
TestAdv=get_field(dfs, 'train > 0 & test == 1.0', 'acc')))
if platform.system() != 'Linux':
dfm = df.melt('TrainAdvPct',
var_name='TestMode',
value_name='Accuracy')
g = sns.factorplot(x="TrainAdvPct",
y="Accuracy",
hue='TestMode',
data=dfm)
plt.show()
def main(argv):
args = parser.parse_args(argv[1:])
log_codes = dict(e=tf.logging.ERROR,
i=tf.logging.INFO,
w=tf.logging.WARN,
d=tf.logging.DEBUG)
tf.logging.set_verbosity(
log_codes.get(args.log.lower()[0], tf.logging.ERROR))
dataset_name = args.dataset
epochs = args.epochs
batch_size = args.batch_size
if args.eager:
tf.enable_eager_execution()
print('******** TF EAGER MODE ENABLED ***************')
params = Bunch(perturb_norm_bound=args.eps,
perturb_norm_order=args.norm,
epochs=epochs,
batch_size=batch_size,
lr=0.01,
adv_reg_lambda=0.0,
clean_pre_train=0.0,
multi_gpu=args.multi_gpu)
get_run_results(dataset_name, params.mod(perturb_frac=args.cont))
def train(self, params: Bunch):
tf.set_random_seed(123)
np.random.seed(123)
config = None
if platform.system() == 'Linux' and multi_gpu: # we're on the GPU
# Thanks to
# https://medium.com/tensorflow/multi-gpu-training-with-estimators-tf-keras-and-tf-data-ba584c3134db
NUM_GPUS = 10
strategy = tf.contrib.distribute.MirroredStrategy(
num_gpus=NUM_GPUS)
config = tf.estimator.RunConfig(train_distribute=strategy)
classifier = tf.estimator.Estimator(
model_fn=binary_classification_model,
model_dir=self.model_dir,
config=config,
params=params.mod(segments=self.segments,
feature_columns=self.feature_columns,
train_perturb_frac=params.perturb_frac,
test_perturb_frac=0.0))
def input_fn():
df_target = self.df_train[self.target_name]
ds = tf.data.Dataset.from_tensor_slices(
(dict(self.df_train), df_target))
ds = ds.shuffle(buffer_size=1000, seed=123).\
repeat(params.epochs).prefetch(params.batch_size*3).\
batch(params.batch_size)
if platform.system() == 'Linux':
ds = ds.apply(
tf.contrib.data.prefetch_to_device(device='/device:gpu:0'))
return ds
# define train input_fn
# input_fn = lambda: tf.contrib.data.make_csv_dataset(
# self.train_file,
# batch_size=batch_size,
# num_epochs=epochs,
# shuffle=True,
# shuffle_buffer_size=10000,
# shuffle_seed=123,
# prefetch_buffer_size=batch_size*3,
# num_parallel_reads=10,
# label_name=self.target_name
# )
# Tried to use this to show train/test loss at each epoch but
# doesn't work as there's an outstanding bug-fix in TF.
# evaluator = tf.contrib.estimator.InMemoryEvaluatorHook(
# classifier, eval_input_fn, steps=None, every_n_iter=100)
# Train the Model.
classifier.train(
input_fn=input_fn, # hooks=[evaluator],
steps=None)
return classifier
def run_one(df: pd.DataFrame, params: Bunch, name='one'):
'''
Only do a single train (nat or adv) and test (nat or adv) combo
and return some metrics/values
:param df:
:param params:
:return:
'''
tf.set_random_seed(123)
np.random.seed(123)
model_dir = os.path.join('/tmp/robulin/exp', name)
runner = Runner(name, df, model_dir, params)
make_clear_dir(model_dir)
# adversarial training: pre-train on nat examples for
# some fraction of epochs, then on adversarial for remaining epochs
clean_train_epochs = int(
params.get('clean_pre_train', 0.5) * params.epochs)
dirty_train_epochs = params.epochs - clean_train_epochs
if clean_train_epochs > 0:
runner.train(params.mod(perturb_frac=0.0, epochs=clean_train_epochs))
runner.train(params.mod(epochs=dirty_train_epochs))
result = runner.eval(params.mod(perturb_frac=params.test_perturb_frac))
result.update(train=params.perturb_frac, test=params.test_perturb_frac)
few_fields = params.get('fields', [
'train', 'test', 'loss', 'auc', 'acc', 'wts_ent', 'wts_l1',
'wts_l1_linf', 'wts_1pct', 'wts_pct1pct', 'av_ent', 'av_high', 'a_ent',
'g_ent', 'f_a_ent', 'f_g_ent'
])
result_few = sub_dict(result, few_fields)
simple_keys = [
k for k, v in params.items() if type(v) in [int, float, str, bool]
]
result_few.update(sub_dict(params, simple_keys))
print(result_few)
result.update(sub_dict(params, simple_keys))
return result
def run_5_combos(dataset_name, df, model_dir, params: Bunch):
runner = Runner(dataset_name, df, model_dir, params)
make_clear_dir(model_dir)
# exp_dir = make_sibling_dir(__file__, 'experiments')
# exp = Experiment(name=dataset_name, debug=False, save_dir=exp_dir)
# exp.tag(params)
# natural training
runner.train(params.mod(perturb_frac=0.0))
nat_nat = runner.eval(params.mod(perturb_frac=0.0))
nat_nat.update(train=0.0, test=0.0)
# exp.log(nat_nat)
printed_cols = list(
set(nat_nat.keys()).difference(
['f_a_dict', 'f_g_dict', 'wts_dict', 'corrs_dict']))
print(pd.DataFrame([nat_nat])[printed_cols])
nat_per = runner.eval(params)
nat_per.update(train=0.0, test=params.perturb_frac)
# exp.log(nat_per)
print(pd.DataFrame([nat_per])[printed_cols])
# adversarial training: Start with naturally trained classifier
# for epochs/2, then train on adversarial inputs for remaining
# epochs/2
make_clear_dir(model_dir)
clean_train_epochs = int(
params.get('clean_pre_train', 0.5) * params.epochs)
dirty_train_epochs = params.epochs - clean_train_epochs
if clean_train_epochs > 0:
runner.train(params.mod(perturb_frac=0.0, epochs=clean_train_epochs))
runner.train(params.mod(epochs=dirty_train_epochs))
per_nat = runner.eval(params.mod(perturb_frac=0.0))
per_nat.update(train=params.perturb_frac, test=0.0)
# exp.log(per_nat)
print(pd.DataFrame([per_nat])[printed_cols])
per_per = runner.eval(params)
per_per.update(train=params.perturb_frac, test=params.perturb_frac)
# exp.log(per_per)
print(pd.DataFrame([per_per])[printed_cols])
per_per_all = runner.eval(params.mod(perturb_frac=1.0))
per_per_all.update(train=params.perturb_frac, test=1.0)
# exp.log(per_per_all)
print(pd.DataFrame([per_per_all])[printed_cols])
all_results = dict(nat_nat=nat_nat,
nat_per=nat_per,
per_nat=per_nat,
per_per=per_per,
per_per_all=per_per_all)
return all_results
def main(argv):
args = parser.parse_args(argv[1:])
log_codes = dict(e=tf.logging.ERROR,
i=tf.logging.INFO,
w=tf.logging.WARN,
d=tf.logging.DEBUG)
tf.logging.set_verbosity(
log_codes.get(args.log.lower()[0], tf.logging.ERROR))
dataset_name = args.dataset
epochs = args.epochs
batch_size = args.batch_size
if args.eager:
tf.enable_eager_execution()
print('******** TF EAGER MODE ENABLED ***************')
params = Bunch(
perturb_norm_bound=args.eps,
perturb_norm_order=args.norm,
std=True, # DO NOT NORMALIZE DATA
epochs=epochs,
batch_size=batch_size,
lr=0.01,
adv_reg_lambda=0.0,
clean_pre_train=0.0,
multi_gpu=args.multi_gpu)
# synthetic dataset
N = 1000
nF = 2 # number of noise featuers
np.random.seed(123)
y = np.random.choice(2, N, p=[0.5, 0.5])
y1 = 2 * y - 1.0
agree = np.random.choice(2, N, p=[0.2, 0.8])
agree = 2 * agree - 1.0
x1 = np.array(agree * y1, dtype=np.float32)
df1 = pd.DataFrame(dict(x1=x1))
df_tar = pd.DataFrame(dict(target=y))
rest = np.array(np.random.choice(2, N * nF), dtype=np.float32). \
reshape([-1, nF])
rest_cols = ['x' + str(i + 2) for i in range(nF)]
df_rest = pd.DataFrame((2 * rest - 1.0), columns=rest_cols)
df = pd.concat([df1, df_rest, df_tar], axis=1)
get_run_results(dataset_name, params.mod(perturb_frac=args.cont), df=df)
def main(argv):
args = parser.parse_args(argv[1:])
log_codes = dict(e=tf.logging.ERROR,
i=tf.logging.INFO,
w=tf.logging.WARN,
d=tf.logging.DEBUG)
tf.logging.set_verbosity(
log_codes.get(args.log.lower()[0], tf.logging.ERROR))
dataset_name = args.dataset
epochs = args.epochs
batch_size = args.batch_size
if args.eager:
tf.enable_eager_execution()
print('******** TF EAGER MODE ENABLED ***************')
params = Bunch(
perturb_norm_bound=args.eps,
perturb_norm_order=args.norm,
std=False, # DO NOT NORMALIZE DATA
epochs=epochs,
activation=args.act,
batch_size=batch_size,
lr=0.01,
adv_reg_lambda=0.0,
clean_pre_train=0.0,
multi_gpu=args.multi_gpu)
# synthetic dataset
# three features: one strongly indicative of label,
# the other two highly correlated by random
data_params = Bunch(N=1000,
nc=args.nc,
nr=args.nf,
nw=args.nw,
corr=args.corr,
corrp=args.corr1,
cat=args.cat)
df = gen_synth_df(data_params)
get_run_results(dataset_name, params.mod(perturb_frac=args.cont), df=df)
def run_grid(params: Bunch):
'''
Run a grid of experiments based on params.grid and return collated
values/metrics in a data-frame
:param params:
:return:
'''
if params.get('dataset'):
df = fetch_data(params.dataset)
else:
df = gen_synth_df(params)
grid_dict = params.grid
params_list = list(ParameterGrid(grid_dict))
results = []
for p in params_list:
result = run_one(df,
params.mod(p),
name=params.get('dataset', 'synth'))
results += [result]
results = pd.DataFrame(results)
return results
def eval(self, params: Bunch):
tf.set_random_seed(123)
np.random.seed(123)
classifier = tf.estimator.Estimator(
model_fn=binary_classification_model,
model_dir=self.model_dir,
config=tf.estimator.RunConfig(tf_random_seed=123),
params=params.mod(segments=self.segments,
feature_columns=self.feature_columns,
train_perturb_frac=0.0,
test_perturb_frac=params.perturb_frac))
def input_fn(train=False):
df = self.df_train if train else self.df_test
df_target = df[self.target_name]
ds = tf.data.Dataset.from_tensor_slices((dict(df), df_target)).\
prefetch(params.batch_size*3).\
batch(params.batch_size)
if platform.system() == 'Linux':
ds = ds.apply(
tf.contrib.data.prefetch_to_device(device='/device:gpu:0'))
return ds
# input_fn = lambda: tf.contrib.data.make_csv_dataset(
# self.test_file,
# batch_size=batch_size,
# prefetch_buffer_size=batch_size*3,
# num_parallel_reads=10,
# num_epochs=1,
# shuffle=False,
# label_name=self.target_name
# )
eval_result = classifier.evaluate(
input_fn=lambda: input_fn(train=False))
train_result = classifier.evaluate(
input_fn=lambda: input_fn(train=True))
# Note we are computing ALL attribution metrics on
# UNPERTURBED TRAIN data !
feature_value_attribs = train_result['attrib']
feature_value_attribs_ent = tf_entropy(feature_value_attribs)
feature_attribs = tf.segment_sum(feature_value_attribs, self.segments)
feature_attribs_ent = tf_entropy(feature_attribs)
afvi = train_result['afvi']
afvi_ent = tf_entropy(afvi)
feature_afvi = tf.segment_sum(afvi, self.segments)
feature_afvi_ent = tf_entropy(feature_afvi)
col_names = [s['name'] for s in self.col_spec]
feature_attribs_dict = dict(
zip(col_names, np.round(tf_numpy(feature_attribs), 4)))
feature_afvi_dict = dict(
zip(col_names, np.round(tf_numpy(feature_afvi), 4)))
wts = classifier.get_variable_value('dense/kernel').squeeze()
wts_dict = dict(zip(self.feature_value_names, wts))
feat_label_corrs = \
(train_result['xy_av'] - train_result['x_av'] * train_result['y_av']) / \
np.sqrt( (train_result['xsq_av'] - train_result['x_av']**2) * \
(train_result['ysq_av'] - train_result['y_av']**2) )
corrs_dict = dict(zip(self.feature_value_names, feat_label_corrs))
wts_ent = tf_numpy(tf_entropy(wts))
wts_l1 = tf_numpy(tf.norm(wts, ord=1))
wts_max = np.max(np.abs(wts))
wts_l1_linf = wts_l1 / wts_max
wts_1pct = np.sum(np.abs(wts) > 0.01 * wts_max)
wts_pct1pct = 100 * np.sum(np.abs(wts) > 0.01 * wts_max) / len(wts)
results = dict(acc=np.round(eval_result['accuracy'], 3),
auc=np.round(eval_result['auc'], 3),
loss=np.round(eval_result['loss'], 3),
wts_ent=np.round(wts_ent, 3),
wts_1pct=wts_1pct,
wts_pct1pct=wts_pct1pct,
wts_l1=np.round(wts_l1, 3),
wts_l1_linf=np.round(wts_l1_linf, 3),
av_ent=np.round(train_result['attrib_ent'], 7),
av_high=np.round(train_result['high_attribs'], 1),
a_ent=np.round(tf_numpy(afvi_ent), 3),
f_a_ent=np.round(tf_numpy(feature_afvi_ent), 3),
g_ent=np.round(tf_numpy(feature_value_attribs_ent), 3),
f_g_ent=np.round(tf_numpy(feature_attribs_ent), 3),
f_a_dict=feature_afvi_dict,
f_g_dict=feature_attribs_dict,
wts_dict=wts_dict,
corrs_dict=corrs_dict)
return results