def train_student(dataset, nb_teachers, shift_dataset,inverse_w=None, weight = True):
"""
This function trains a student using predictions made by an ensemble of
teachers. The student and teacher models are trained using the same
neural network architecture.
:param dataset: string corresponding to mnist, cifar10, or svhn
:param nb_teachers: number of teachers (in the ensemble) to learn from
:param weight: whether this is an importance weight sampling
:return: True if student training went well
"""
assert input.create_dir_if_needed(FLAGS.train_dir)
# Call helper function to prepare student data using teacher predictions
stdnt_data = shift_dataset['data']
stdnt_labels = shift_dataset['pred']
print('number for deep is {}'.format(len(stdnt_labels)))
if FLAGS.deeper:
ckpt_path = FLAGS.train_dir + '/' + str(dataset) + '_' + str(nb_teachers) + '_student_deeper.ckpt' #NOLINT(long-line)
else:
ckpt_path = FLAGS.train_dir + '/' + str(dataset) + '_' + str(nb_teachers) + '_student.ckpt' # NOLINT(long-line)
if FLAGS.cov_shift == True:
"""
need to compute the weight for student
curve weight into some bound, in case the weight is too large
"""
weights = inverse_w
else:
print('len of shift data'.format(len(shift_dataset['data'])))
weights = np.zeros(len(stdnt_data))
print('len of weight={} len of labels= {} '.format(len(weights), len(stdnt_labels)))
for i, x in enumerate(weights):
weights[i] = np.float32(inverse_w[stdnt_labels[i]])
if weight == True:
assert deep_cnn.train(stdnt_data, stdnt_labels, ckpt_path, weights= weights)
else:
deep_cnn.train(stdnt_data, stdnt_labels, ckpt_path)
# Compute final checkpoint name for student (with max number of steps)
ckpt_path_final = ckpt_path + '-' + str(FLAGS.max_steps - 1)
if dataset == 'adult':
private_data, private_labels = input.ld_adult(test_only = False, train_only= True)
elif dataset =='mnist':
private_data, private_labels = input.ld_mnist(test_only = False, train_only = True)
elif dataset =="svhn":
private_data, private_labels = input.ld_svhn(test_only=False, train_only=True)
# Compute student label predictions on remaining chunk of test set
teacher_preds = deep_cnn.softmax_preds(private_data, ckpt_path_final)
student_preds = deep_cnn.softmax_preds(stdnt_data, ckpt_path_final)
# Compute teacher accuracy
precision_t = metrics.accuracy(teacher_preds, private_labels)
precision_s = metrics.accuracy(student_preds, stdnt_labels)
precision_true = metrics.accuracy(student_preds, shift_dataset['label'])
print('Precision of teacher after training:{} student={} true precision for student {}'.format(precision_t, precision_s,precision_true))
return precision_t, precision_s
def load_dataset(dataset, test_only=False, train_only=False):
if dataset == 'svhn':
test_data, test_labels = input.ld_svhn(test_only=test_only)
return test_data, test_labels
elif dataset == 'cifar10':
test_data, test_labels = input.ld_cifar10(test_only=test_only)
elif dataset == 'mnist':
test_data, test_labels = input.ld_mnist(test_only=test_only)
elif dataset == 'adult':
test_data, test_labels = input.ld_adult(test_only = test_only)
else:
print("Check value of dataset flag")
return test_data, test_labels
def predict_teacher(dataset, nb_teachers):
"""
This is for obtaining the weight from student / teache, don't involve any noise
:param dataset: string corresponding to mnist, cifar10, or svhn
:param nb_teachers: number of teachers (in the ensemble) to learn from
:param teacher: if teacher is true, then predict with training dataset, else students
:return: out prediction based on cnn
"""
assert input.create_dir_if_needed(FLAGS.train_dir)
train_only = True
test_only = False
# create path to save teacher predict teacher model
filepath = FLAGS.data_dir + "/" + str(dataset) + '_' + str(
nb_teachers) + '_teacher_clean_votes_label_shift' + str(
FLAGS.lap_scale) + '.npy'
# Load the dataset
if dataset == 'svhn':
test_data, test_labels = input.ld_svhn(test_only, train_only)
elif dataset == 'cifar10':
test_data, test_labels = input.ld_cifar10(test_only, train_only)
elif dataset == 'mnist':
test_data, test_labels = input.ld_mnist(test_only, train_only)
elif dataset == 'adult':
test_data, test_labels = input.ld_adult(test_only, train_only)
else:
print("Check value of dataset flag")
return False
if os.path.exists(filepath):
pred_labels = np.load(filepath)
return pred_labels, test_labels
teachers_preds = ensemble_preds(dataset, nb_teachers, test_data)
# Aggregate teacher predictions to get student training labels
pred_labels = aggregation.noisy_max(FLAGS.nb_teachers, teachers_preds, 0)
utils.save_file(filepath, pred_labels)
# Print accuracy of aggregated labels
ac_ag_labels = metrics.accuracy(pred_labels, test_labels)
print("obtain_weight Accuracy of the aggregated labels: " +
str(ac_ag_labels))
return pred_labels, test_labels
def prepare_student_data(dataset, nb_teachers,shift_idx,nb_q=None):
"""
Takes a dataset name and the size of the teacher ensemble and prepares
training data for the student model, according to parameters indicated
in flags above.
:param dataset: string corresponding to mnist, cifar10, or svhn
:param nb_teachers: number of teachers (in the ensemble) to learn from
:param save: if set to True, will dump student training labels predicted by
the ensemble of teachers (with Laplacian noise) as npy files.
It also dumps the clean votes for each class (without noise) and
the labels assigned by teachers
:return: pairs of (data, labels) to be used for student training and testing
"""
if dataset == 'svhn':
test_data, test_labels = input.ld_svhn(test_only=True)
elif dataset == 'cifar10':
test_data, test_labels = input.ld_cifar10(test_only=True)
elif dataset == 'mnist':
test_data, test_labels = input.ld_mnist(test_only=True)
elif dataset == 'adult':
test_data, test_labels = input.ld_adult(test_only = True)
else:
print("Check value of dataset flag")
return False
if nb_q !=None:
shift_idx = np.random.choice(shift_idx, nb_q)
# Prepare filepath for numpy dump of clean votessvhn_250_student_clean_test.npy
filepath = FLAGS.data_dir + "/" + str(dataset) + '_' + str(nb_teachers) + '_student_clean_test.npy' # NOLINT(long-line)
if os.path.exists(filepath):
with open(filepath,'rb')as f:
clean_votes = np.load(f)
keep_idx, result = gaussian(FLAGS.nb_labels, clean_votes,shift_idx)
precision_true = metrics.accuracy(result, test_labels[keep_idx])
print('number of idx={} precision_true from gaussian for shift data={}'.format(len(keep_idx[0]), precision_true))
return keep_idx, test_data[keep_idx], result
print('not find file for clean student vote')
def predict_data(dataset, nb_teachers, teacher=False):
"""
This is for obtaining the weight from student / teache, don't involve any noise
:param dataset: string corresponding to mnist, cifar10, or svhn
:param nb_teachers: number of teachers (in the ensemble) to learn from
:param teacher: if teacher is true, then predict with training dataset, else students
:return: out prediction based on cnn
"""
assert input.create_dir_if_needed(FLAGS.train_dir)
if teacher:
train_only = True
test_only = False
else:
train_only = False
test_only = True
# Load the dataset
if dataset == 'svhn':
test_data, test_labels = input.ld_svhn(test_only, train_only)
elif dataset == 'cifar10':
test_data, test_labels = input.ld_cifar10(test_only, train_only)
elif dataset == 'mnist':
test_data, test_labels = input.ld_mnist(test_only, train_only)
elif dataset == 'adult':
test_data, test_labels = input.ld_adult(test_only, train_only)
else:
print("Check value of dataset flag")
return False
teachers_preds = ensemble_preds(dataset, nb_teachers, test_data)
# Aggregate teacher predictions to get student training labels
pred_labels = aggregation.noisy_max(FLAGS.nb_teachers, teachers_preds, 0)
# Print accuracy of aggregated labels
ac_ag_labels = metrics.accuracy(pred_labels, test_labels)
print("obtain_weight Accuracy of the aggregated labels: " +
str(ac_ag_labels))
return test_data, pred_labels, test_labels
def pca_transform(dataset, FLAGS):
"""
Do PCA transform on both teacher and student dataset
:param dataset:
:return:
pca transformed teacher and student dataset
"""
teacher_file_name = FLAGS.data + '/PCA_teacher' + dataset + '.pkl'
student_file_name = FLAGS.data + '/PCA_student' + dataset + '.pkl'
#if os.path.exists(teacher_file_name):
#return
test_only = False
train_only = False
# Load the dataset
if dataset == 'svhn':
dim = 3072
train_data, train_labels, test_data, test_labels = input.ld_svhn(
test_only, train_only)
ori_train = train_data.shape
ori_test = test_data.shape
test_data = test_data.reshape((-1, dim))
train_data = train_data.reshape((-1, dim))
elif dataset == 'cifar10':
train_data, train_labels, test_data, test_labels = input.ld_cifar10(
test_only, train_only)
dim = 3072
elif dataset == 'mnist':
train_data, train_labels, test_data, test_labels = input.ld_mnist(
test_only, train_only)
ori_train = train_data.shape
ori_test = test_data.shape
dim = 784
test_data = test_data.reshape((-1, dim))
train_data = train_data.reshape((-1, dim))
elif dataset == 'adult':
train_data, train_labels, test_data, test_labels = input.ld_adult(
test_only, train_only)
dim = 108
else:
print("Check value of dataset flag")
return False
pca = PCA(n_components=1)
pca.fit(test_data)
max_component = pca.components_.T
projection = np.dot(test_data, max_component)
min_v = np.min(projection)
mean_v = np.mean(projection)
a = 1e2
b = 1
mu = min_v + (mean_v - min_v) / a
var = (mean_v - min_v) / b
prob = scipy.stats.norm(mu, var).pdf(projection)
true_prob = np.ones(len(test_data)) / len(test_data)
true_ratio = true_prob / prob * np.sum(prob)
prob = np.ravel(prob.T) # transform into 1d dim
index = np.where(prob > 0)[0]
sample = np.random.choice(index,
len(index),
replace=True,
p=prob / np.sum(prob))
test_data = test_data[sample]
test_label = test_labels[sample]
train_data = np.reshape(train_data, ori_train)
test_data = np.reshape(test_data, ori_test)
test = {}
test['data'] = test_data
test['label'] = test_label
test['index'] = sample
f = open(teacher_file_name, 'wb')
pickle.dump(train_data, f)
f = open(student_file_name, 'wb')
pickle.dump(test, f)
print('finish pca transform')
def train_student(dataset,
nb_teachers,
weight=True,
inverse_w=None,
shift_dataset=None):
"""
This function trains a student using predictions made by an ensemble of
teachers. The student and teacher models are trained using the same
neural network architecture.
:param dataset: string corresponding to mnist, cifar10, or svhn
:param nb_teachers: number of teachers (in the ensemble) to learn from
:param weight: whether this is an importance weight sampling
:return: True if student training went well
"""
assert input.create_dir_if_needed(FLAGS.train_dir)
# Call helper function to prepare student data using teacher predictions
if shift_dataset is not None:
stdnt_data, stdnt_labels = prepare_student_data(
dataset, nb_teachers, save=True, shift_data=shift_dataset)
else:
if FLAGS.PATE2 == True:
keep_idx, stdnt_data, stdnt_labels = prepare_student_data(
dataset, nb_teachers, save=True)
else:
stdnt_data, stdnt_labels = prepare_student_data(dataset,
nb_teachers,
save=True)
rng = np.random.RandomState(FLAGS.dataset_seed)
rand_ix = rng.permutation(len(stdnt_labels))
stdnt_data = stdnt_data[rand_ix]
stdnt_labels = stdnt_labels[rand_ix]
print('number for deep is {}'.format(len(stdnt_labels)))
# Unpack the student dataset, here stdnt_labels are already the ensemble noisy version
# Prepare checkpoint filename and path
if FLAGS.deeper:
ckpt_path = FLAGS.train_dir + '/' + str(dataset) + '_' + str(
nb_teachers) + '_student_deeper.ckpt' #NOLINT(long-line)
else:
ckpt_path = FLAGS.train_dir + '/' + str(dataset) + '_' + str(
nb_teachers) + '_student.ckpt' # NOLINT(long-line)
# Start student training
if FLAGS.cov_shift == True:
"""
need to compute the weight for student
curve weight into some bound, in case the weight is too large
"""
weights = inverse_w
#y_s = np.expand_dims(y_s, axis=1)
else:
print('len of shift data'.format(len(shift_dataset['data'])))
weights = np.zeros(len(stdnt_data))
print('len of weight={} len of labels= {} '.format(
len(weights), len(stdnt_labels)))
for i, x in enumerate(weights):
weights[i] = np.float32(inverse_w[stdnt_labels[i]])
if weight == True:
if FLAGS.PATE2 == True:
assert deep_cnn.train(stdnt_data,
stdnt_labels,
ckpt_path,
weights=weights[keep_idx])
else:
assert deep_cnn.train(stdnt_data,
stdnt_labels,
ckpt_path,
weights=weights)
else:
deep_cnn.train(stdnt_data, stdnt_labels, ckpt_path)
# Compute final checkpoint name for student (with max number of steps)
ckpt_path_final = ckpt_path + '-' + str(FLAGS.max_steps - 1)
if dataset == 'adult':
private_data, private_labels = input.ld_adult(test_only=False,
train_only=True)
elif dataset == 'mnist':
private_data, private_labels = input.ld_mnist(test_only=False,
train_only=True)
elif dataset == "svhn":
private_data, private_labels = input.ld_svhn(test_only=False,
train_only=True)
# Compute student label predictions on remaining chunk of test set
teacher_preds = deep_cnn.softmax_preds(private_data, ckpt_path_final)
student_preds = deep_cnn.softmax_preds(stdnt_data, ckpt_path_final)
# Compute teacher accuracy
precision_t = metrics.accuracy(teacher_preds, private_labels)
precision_s = metrics.accuracy(student_preds, stdnt_labels)
if FLAGS.cov_shift == True:
student_file_name = FLAGS.data + 'PCA_student' + FLAGS.dataset + '.pkl'
f = open(student_file_name, 'rb')
test = pickle.load(f)
if FLAGS.PATE2 == True:
test_labels = test['label'][keep_idx]
else:
test_labels = test['label']
precision_true = metrics.accuracy(student_preds, test_labels)
print(
'Precision of teacher after training:{} student={} true precision for student {}'
.format(precision_t, precision_s, precision_true))
return len(test_labels), precision_t, precision_s
def prepare_student_data(dataset, nb_teachers, save=False, shift_data=None):
"""
Takes a dataset name and the size of the teacher ensemble and prepares
training data for the student model, according to parameters indicated
in flags above.
:param dataset: string corresponding to mnist, cifar10, or svhn
:param nb_teachers: number of teachers (in the ensemble) to learn from
:param save: if set to True, will dump student training labels predicted by
the ensemble of teachers (with Laplacian noise) as npy files.
It also dumps the clean votes for each class (without noise) and
the labels assigned by teachers
:return: pairs of (data, labels) to be used for student training and testing
"""
if dataset == 'svhn':
test_data, test_labels = input.ld_svhn(test_only=True)
elif dataset == 'cifar10':
test_data, test_labels = input.ld_cifar10(test_only=True)
elif dataset == 'mnist':
test_data, test_labels = input.ld_mnist(test_only=True)
elif dataset == 'adult':
test_data, test_labels = input.ld_adult(test_only=True)
else:
print("Check value of dataset flag")
return False
if FLAGS.cov_shift == True:
student_file_name = FLAGS.data + 'PCA_student' + FLAGS.dataset + '.pkl'
f = open(student_file_name, 'rb')
test = pickle.load(f)
test_data = test['data']
test_labels = test['label']
# Prepare [unlabeled] student training data (subset of test set)
stdnt_data = test_data
assert input.create_dir_if_needed(FLAGS.train_dir)
gau_filepath = FLAGS.data_dir + "/" + str(dataset) + '_' + str(
nb_teachers) + '_student_votes_sigma1:' + str(
FLAGS.sigma1) + '_sigma2:' + str(
FLAGS.sigma2) + '.npy' # NOLINT(long-line)
# Prepare filepath for numpy dump of clean votes
filepath = FLAGS.data_dir + "/" + str(dataset) + '_' + str(
nb_teachers) + '_student_clean_votes' + str(
FLAGS.lap_scale) + '.npy' # NOLINT(long-line)
# Prepare filepath for numpy dump of clean labels
filepath_labels = FLAGS.data_dir + "/" + str(dataset) + '_' + str(
nb_teachers) + '_teachers_labels_lap_' + str(
FLAGS.lap_scale) + '.npy' # NOLINT(long-line)
"""
if os.path.exists(filepath):
if FLAGS.PATE2 == True:
with open(filepath,'rb')as f:
clean_votes = np.load(f)
keep_idx, result = gaussian(FLAGS.nb_labels, clean_votes)
precision_true = metrics.accuracy(result, test_labels[keep_idx])
print('number of idx={}'.format(len(keep_idx[0])))
return keep_idx, stdnt_data[keep_idx], result
"""
# Load the dataset
# Make sure there is data leftover to be used as a test set
assert FLAGS.stdnt_share < len(test_data)
if shift_data is not None:
#no noise
# replace original student data with shift data
stdnt_data = shift_data['data']
test_labels = shift_data['label']
print('*** length of shift_data {} lable length={}********'.format(
len(stdnt_data), len(test_labels)))
# Compute teacher predictions for student training data
teachers_preds = ensemble_preds(dataset, nb_teachers, stdnt_data)
# Aggregate teacher predictions to get student training labels
if not save:
stdnt_labels = aggregation.noisy_max(teachers_preds, FLAGS.lap_scale)
else:
# Request clean votes and clean labels as well
stdnt_labels, clean_votes, labels_for_dump = aggregation.noisy_max(
FLAGS.nb_labels,
teachers_preds,
FLAGS.lap_scale,
return_clean_votes=True) #NOLINT(long-line)
if FLAGS.PATE2 == True:
keep_idx, result = gaussian(FLAGS.nb_labels, clean_votes)
# Dump clean_votes array
with tf.gfile.Open(filepath, mode='w') as file_obj:
np.save(file_obj, clean_votes)
# Dump labels_for_dump array
with tf.gfile.Open(filepath_labels, mode='w') as file_obj:
np.save(file_obj, labels_for_dump)
# Print accuracy of aggregated labels
if FLAGS.PATE2 == True:
with tf.gfile.Open(gau_filepath, mode='w') as file_obj:
np.save(file_obj, result)
ac_ag_labels = metrics.accuracy(result, test_labels[keep_idx])
print(
"number of gaussian student {} Accuracy of the aggregated labels:{} "
.format(len(result), ac_ag_labels))
return keep_idx, stdnt_data[keep_idx], result
else:
ac_ag_labels = metrics.accuracy(stdnt_labels, test_labels)
print("Accuracy of the aggregated labels: " + str(ac_ag_labels))
if save:
# Prepare filepath for numpy dump of labels produced by noisy aggregation
filepath = FLAGS.data_dir + "/" + str(dataset) + '_' + str(
nb_teachers) + '_student_labels_lap_' + str(
FLAGS.lap_scale) + '.npy' #NOLINT(long-line)
# Dump student noisy labels array
with tf.gfile.Open(filepath, mode='w') as file_obj:
np.save(file_obj, stdnt_labels)
return stdnt_data, stdnt_labels