def test_attack_models_are_created(
data,
shadow_model_fn,
shadow_model_serializer,
attack_model_fn,
attack_model_serializer,
):
(X_train, y_train), _ = data
smb = ShadowModelBundle(
shadow_model_fn,
shadow_dataset_size=SHADOW_DATASET_SIZE,
num_models=3,
serializer=shadow_model_serializer,
)
X_shadow, y_shadow = smb.fit_transform(X_train,
y_train,
fit_kwargs=dict(epochs=5,
verbose=False))
amb = AttackModelBundle(attack_model_fn,
num_classes=NUM_CLASSES,
serializer=attack_model_serializer)
# Fit the attack models.
amb.fit(X_shadow, y_shadow, fit_kwargs=dict(epochs=5, verbose=False))
# Predict membership for some training data.
membership_guesses = amb.predict(X_shadow[:100])
assert membership_guesses.shape == (len(X_shadow[:100]), )
def test_shadow_transform_with_custom_shadow_indices(data, shadow_model_fn,
model_serializer,
num_models,
max_models_for_transform):
(X_train, y_train), _ = data
smb = ShadowModelBundle(
shadow_model_fn,
shadow_dataset_size=SHADOW_DATASET_SIZE,
num_models=num_models,
serializer=model_serializer,
)
smb._fit(X_train, y_train, fit_kwargs=dict(epochs=5, verbose=False))
shadow_indices = range(max_models_for_transform)
X_shadow, y_shadow = smb._transform(shadow_indices=shadow_indices)
assert X_shadow.shape[
0] == 2 * max_models_for_transform * SHADOW_DATASET_SIZE
assert y_shadow.shape[
0] == 2 * max_models_for_transform * SHADOW_DATASET_SIZE
def test_shadow_models_are_created_and_data_is_transformed(
data, shadow_model_fn, model_serializer, num_models):
(X_train, y_train), _ = data
smb = ShadowModelBundle(
shadow_model_fn,
shadow_dataset_size=SHADOW_DATASET_SIZE,
num_models=num_models,
serializer=model_serializer,
)
assert not hasattr(smb, "shadow_models_")
X_shadow, y_shadow = smb.fit_transform(X_train,
y_train,
fit_kwargs=dict(epochs=5,
verbose=False))
# X_shadow are prediction vectors for random examples in X_train
assert len(X_shadow) == len(y_shadow) == (2 * num_models *
SHADOW_DATASET_SIZE)
# X_shadow are concatenations of prediction vectors and true
# class.
assert X_shadow.shape[1] == 2 * NUM_CLASSES
def demo(argv):
del argv # Unused.
(X_train, y_train), (X_test, y_test) = get_data()
# Train the target model.
print("Training the target model...")
target_model = target_model_fn()
target_model.fit(
X_train, y_train, batch_size=128, epochs=FLAGS.target_epochs, validation_split=0.5, shuffle=True
)
#target_model.fit(
# X_train, y_train, batch_size=128, epochs=FLAGS.target_epochs, validation_data=(X_test, y_test), shuffle=True
#)
# Train the shadow models.
smb = ShadowModelBundle(
target_model_fn,
shadow_dataset_size=SHADOW_DATASET_SIZE,
num_models=FLAGS.num_shadows,
)
# We assume that attacker's data were not seen in target's training.
attacker_X_train, attacker_X_test, attacker_y_train, attacker_y_test = train_test_split(
X_test, y_test, test_size=0.1
)
print(attacker_X_train.shape, attacker_X_test.shape)
print("Training the shadow models...")
X_shadow, y_shadow = smb.fit_transform(
attacker_X_train,
attacker_y_train,
fit_kwargs=dict(
epochs=FLAGS.target_epochs,
verbose=True,
validation_data=(attacker_X_test, attacker_y_test),
),
)
# ShadowModelBundle returns data in the format suitable for the AttackModelBundle.
amb = AttackModelBundle(attack_model_fn, num_classes=NUM_CLASSES)
# Fit the attack models.
print("Training the attack models...")
amb.fit(
X_shadow, y_shadow, fit_kwargs=dict(epochs=FLAGS.attack_epochs, verbose=True)
)
# Test the success of the attack.
# Prepare examples that were in the training, and out of the training.
data_in = X_train[:ATTACK_TEST_DATASET_SIZE], y_train[:ATTACK_TEST_DATASET_SIZE]
data_out = X_test[:ATTACK_TEST_DATASET_SIZE], y_test[:ATTACK_TEST_DATASET_SIZE]
# Compile them into the expected format for the AttackModelBundle.
attack_test_data, real_membership_labels = prepare_attack_data(
target_model, data_in, data_out
)
# Compute the attack accuracy.
attack_guesses = amb.predict(attack_test_data)
attack_accuracy = np.mean(attack_guesses == real_membership_labels)
print(attack_accuracy)
def demo(argv):
x = pd.read_csv(
r"C:\document\education\python\python\predict mortality\data\drug_table_mortality_with_no.csv"
)
y = pd.read_csv(
r"C:\document\education\python\python\predict mortality\data\mortality_table.csv"
)
# (X_train, y_train), (X_test, y_test) = get_data()
X, Y = ga.shuffleData(x, y)
X_test = X[25000:] #####################################################
y_test = Y[
25000:] #########################################################
X_train = X[:8 * scala]
y_train = Y[:8 * scala]
# Train the target model.
print("Training the target model...")
target_model = train_target(X, Y)
# Train the shadow models.
smb = ShadowModelBundle(
target_model_fn,
shadow_dataset_size=SHADOW_DATASET_SIZE,
num_models=FLAGS.num_shadows,
)
# We assume that attacker's data were not seen in target's training.
attacker_X_train, attacker_X_test, attacker_y_train, attacker_y_test = train_test_split(
X_test, y_test, test_size=0.1)
print(attacker_X_train.shape, attacker_X_test.shape)
print("Training the shadow models...")
X_shadow, y_shadow = smb.fit_transform(
attacker_X_train,
attacker_y_train,
fit_kwargs=dict(
epochs=FLAGS.target_epochs,
verbose=True,
validation_data=(attacker_X_test, attacker_y_test),
),
)
print("shadow_shape")
print(X_shadow.shape)
print(y_shadow)
# ShadowModelBundle returns data in the format suitable for the AttackModelBundle.
amb = AttackModelBundle(attack_model_fn, num_classes=NUM_CLASSES)
# Fit the attack models.
print("Training the attack models...")
amb.fit(X_shadow,
y_shadow,
fit_kwargs=dict(epochs=FLAGS.attack_epochs, verbose=True))
# Test the success of the attack.
# Prepare examples that were in the training, and out of the training.
data_in = X_train[:
ATTACK_TEST_DATASET_SIZE], y_train[:
ATTACK_TEST_DATASET_SIZE]
data_out = X_test[:
ATTACK_TEST_DATASET_SIZE], y_test[:
ATTACK_TEST_DATASET_SIZE]
# Compile them into the expected format for the AttackModelBundle.
attack_test_data, real_membership_labels = prepare_attack_data(
target_model, data_in, data_out)
# Compute the attack accuracy.
attack_guesses = amb.predict(attack_test_data)
attack_accuracy = np.mean(attack_guesses == real_membership_labels)
print(attack_accuracy)
def main():
print("Training the target model...")
# split target dataset to train and valid, and make them evenly divisible by batch size
target_X_train, target_y_train, target_X_valid, target_y_valid = split_to_be_divisible(
target_X, target_y, 0.2, batch_size)
target_X_train = target_X_train.values.reshape(
(target_X_train.shape[0], target_X_train.shape[1], 1))
target_X_valid = target_X_valid.values.reshape(
(target_X_valid.shape[0], target_X_valid.shape[1], 1))
input_shape = (target_X_train.shape[1], target_X_train.shape[2])
tm = target_model(input_shape)
tm.fit(target_X_train,
target_y_train,
batch_size=batch_size,
epochs=epochs,
validation_data=[target_X_valid, target_y_valid],
verbose=1)
print("Training the shadow models.")
# train only one shadow model
SHADOW_DATASET_SIZE = int(shadow_X.shape[0] / 2)
smb = ShadowModelBundle(
shadow_model,
shadow_dataset_size=SHADOW_DATASET_SIZE,
num_models=1,
)
# Training the shadow models with same parameter of target model, and generate attack data...
attacker_X, attacker_y = smb.fit_transform(
shadow_X,
shadow_y.values,
fit_kwargs=dict(epochs=epochs, batch_size=batch_size, verbose=1),
)
print("Training attack model...")
clf = RandomForestClassifier(max_depth=2)
clf.fit(attacker_X, attacker_y)
# Test the success of the attack.
ATTACK_TEST_DATASET_SIZE = unused_X.shape[0]
# Prepare examples that were in the training, and out of the training.
data_in = target_X_train[:
ATTACK_TEST_DATASET_SIZE], target_y_train[:
ATTACK_TEST_DATASET_SIZE]
unused_X1 = unused_X.values.reshape(
(unused_X.shape[0], unused_X.shape[1], 1))
data_out = unused_X1[:
ATTACK_TEST_DATASET_SIZE], unused_y[:
ATTACK_TEST_DATASET_SIZE]
# Compile them into the expected format for the AttackModelBundle.
attack_test_data, real_membership_labels = prepare_attack_data(
tm, data_in, data_out)
# Compute the attack accuracy.
attack_guesses = clf.predict(attack_test_data)
attack_accuracy = np.mean(attack_guesses == real_membership_labels)
print('attack accuracy: {}'.format(attack_accuracy))
acc = accuracy_score(real_membership_labels, attack_guesses)
print('attack acc: {}'.format(acc))
prec = precision_score(real_membership_labels, attack_guesses)
print('Precision: {}'.format(prec))
recall = recall_score(real_membership_labels, attack_guesses)
print('Recall: {}'.format(recall))
fscore = f1_score(real_membership_labels, attack_guesses)
print('F1-Score: {}'.format(fscore))
return model
X, Y = ga.shuffleData(x, y)
X_test = X[22000:28000] #####################################################
y_test = Y[22000:
28000] #########################################################
X_train = X[:8 * scala]
y_train = Y[:8 * scala]
target_model = train_target(X, Y)
smb = ShadowModelBundle(
target_model_fn,
shadow_dataset_size=shadow_dataset_size,
num_models=num_shadows,
)
attacker_X_train, attacker_X_test, attacker_y_train, attacker_y_test = train_test_split(
X_test, y_test, test_size=0.1)
"""
the xshadow and yshadow are not strictly follow the shape of attacher_train
because retranformation is done in below
"""
X_shadow, y_shadow = smb.fit_transform(
attacker_X_train,
attacker_y_train,
fit_kwargs=dict(
epochs=10,
verbose=False,
def demo(argv):
del argv # Unused.
(X_train, y_train), (X_test, y_test) = load_mnist()
# Train the target model.
print("Training the target model...")
target_model = mnist_dpsgd_tutorial_keras.main()
target_model.fit(X_train,
y_train,
epochs=FLAGS.target_epochs,
validation_split=0.1,
verbose=True,
validation_data=(X_test, y_test),
batch_size=250)
# Train the shadow models.
smb = ShadowModelBundle(
mnist_dpsgd_tutorial_keras.main,
shadow_dataset_size=SHADOW_DATASET_SIZE,
num_models=FLAGS.num_shadows,
)
# We assume that attacker's data were not seen in target's training.
attacker_X_train, attacker_X_test, attacker_y_train, attacker_y_test = train_test_split(
X_test, y_test, test_size=0.1)
print(attacker_X_train.shape, attacker_X_test.shape)
print("Training the shadow models...")
X_shadow, y_shadow = smb.fit_transform(
attacker_X_train,
attacker_y_train,
fit_kwargs=dict(epochs=FLAGS.target_epochs,
verbose=True,
validation_data=(attacker_X_test, attacker_y_test),
batch_size=250),
)
# ShadowModelBundle returns data in the format suitable for the AttackModelBundle.
amb = AttackModelBundle(attack_model_fn, num_classes=NUM_CLASSES)
# Fit the attack models.
print("Training the attack models...")
amb.fit(X_shadow,
y_shadow,
fit_kwargs=dict(epochs=FLAGS.attack_epochs, verbose=True))
# Test the success of the attack.
# Prepare examples that were in the training, and out of the training.
data_in = X_train[:
ATTACK_TEST_DATASET_SIZE], y_train[:
ATTACK_TEST_DATASET_SIZE]
data_out = X_test[:
ATTACK_TEST_DATASET_SIZE], y_test[:
ATTACK_TEST_DATASET_SIZE]
# Compile them into the expected format for the AttackModelBundle.
attack_test_data, real_membership_labels = prepare_attack_data(
target_model, data_in, data_out)
# Compute the attack accuracy.
attack_guesses = amb.predict(attack_test_data)
attack_accuracy = np.mean(attack_guesses == real_membership_labels)
print(attack_accuracy)
fscores = precision_recall_fscore_support(real_membership_labels,
attack_guesses)
[print(fscore) for fscore in fscores]
from sklearn.metrics import precision_score, recall_score, f1_score
print(f1_score(real_membership_labels, attack_guesses))
idxfilter=lambda x: x % 55 == 2,
batch_size=batch_size)
data = []
for i in range(len(val_gen)):
data.append(val_gen[i])
X_test = np.concatenate(tuple([d[0] for d in data]))
y_test = np.concatenate(tuple([d[1] for d in data]))
smb_serial = MySerializer(model_fn=create_dpsgd_model_f,
prefix="./smb_DPSGD__model_weights_timeS_" + "ZENA")
# Train the shadow models.
smb = ShadowModelBundle(
create_dpsgd_model_f,
shadow_dataset_size=SHADOW_DATASET_SIZE, #dataset size for shadow
num_models=num_shadows, #how many shadows
serializer=smb_serial)
# We assume that attacker's data were not seen in target's training.
attacker_X_train, attacker_X_test, attacker_y_train, attacker_y_test = train_test_split(
X_test, y_test, test_size=0.2)
print(attacker_X_train.shape, attacker_X_test.shape)
print("Training the shadow models...")
saved = 0 # MANUALLY
if saved == 0:
#if model is not saved
X_shadow, y_shadow = smb.fit_transform(
def demo(argv):
del argv .
(x_train, y_train), (x_test, y_test) = get_data()
# 训练target model
print("Training the target model...")
target_model = target_model_fn()
target_model.fit(
x_train, y_train, epochs=FLAGS.target_epochs, validation_split=0.9, verbose=True
)
# 训练shadow model
smb = ShadowModelBundle(
target_model_fn,
shadow_dataset_size=SHADOW_DATASET_SIZE,
num_models=FLAGS.num_shadows,
)
# Shadow model所用数据
attacker_x_train, attacker_x_test, attacker_y_train, attacker_y_test = train_test_split(
x_test, y_test, test_size=0.1
)
print(attacker_x_train.shape, attacker_x_test.shape)
print("Training the shadow models...")
X_shadow, y_shadow = smb.fit_transform(
attacker_x_train,
attacker_y_train,
fit_kwargs=dict(
epochs=FLAGS.target_epochs,
verbose=True,
validation_data=(attacker_x_test, attacker_y_test),
),
)
# ShadowModelBundle 返回 AttackModelBundle 所需的数据形式
amb = AttackModelBundle(attack_model_fn, num_classes=NUM_CLASSES)
# 训练attack models.
print("Training the attack models...")
amb.fit(
X_shadow, y_shadow, fit_kwargs=dict(epochs=FLAGS.attack_epochs, verbose=True)
)
# 测试attack model效果
# 生成测试集
data_in = x_train[:ATTACK_TEST_DATASET_SIZE], y_train[:ATTACK_TEST_DATASET_SIZE]
data_out = x_test[:ATTACK_TEST_DATASET_SIZE], y_test[:ATTACK_TEST_DATASET_SIZE]
attack_test_data, real_membership_labels = prepare_attack_data(
target_model, data_in, data_out
)
# 计算成员检测攻击的准确度
attack_guesses = amb.predict(attack_test_data)
attack_accuracy = np.mean(attack_guesses == real_membership_labels)
print(attack_accuracy)