def train(self, synA, labels):
""" Train a membership inference attack on a labelled training set
:param synA: list of ndarrays: A list of synthetic datasets
:param labels: list: A list of labels that indicate whether target was in the training data (LABEL_IN=1) or not (LABEL_OUT=0)
"""
if self.FeatureSet is not None:
synA = stack([self.FeatureSet.extract(s) for s in synA])
else:
if isinstance(synA[0], DataFrame):
synA = [self._impute_missing_values(s) for s in synA]
synA = stack([
convert_df_to_array(s, self.metadata).flatten()
for s in synA
])
else:
synA = stack([s.flatten() for s in synA])
if not isinstance(labels, ndarray):
labels = array(labels)
self.AttackClassifier.fit(synA, labels)
logger.debug('Finished training MIA distinguisher')
self.trained = True
del synA, labels
def get_confidence(self, synT, secret):
"""Calculate probability that attacker correctly predicts whether target was present in model's training data"""
assert self.trained, 'Attack must first be trained.'
if self.FeatureSet is not None:
synT = stack([self.FeatureSet.extract(s) for s in synT])
else:
if isinstance(synT[0], DataFrame):
synT = stack([
convert_df_to_array(s, self.metadata).flatten()
for s in synT
])
else:
synT = stack([s.flatten() for s in synT])
probs = self.Distinguisher.predict_proba(synT)
return [p[s] for p, s in zip(probs, secret)]
def train(self, synT):
"""
Train a MLE attack to reconstruct an unknown sensitive value from a vector of known attributes
:param synT: DataFrame: A synthetic dataset of shape (n, k + 1)
"""
# Split data into known and sensitive
if isinstance(synT, DataFrame):
assert self.sensitiveAttribute in list(
synT), f'DataFrame only contains columns {list(synT)}'
synKnown = synT.drop(self.sensitiveAttribute, axis=1)
synSensitive = synT[self.sensitiveAttribute]
synKnown = convert_df_to_array(synKnown, self.metadata)
synSensitive = convert_series_to_array(synSensitive, self.metadata)
else:
assert isinstance(synT, ndarray), f"Unknown data type {type(synT)}"
# If input data is array assume that self.metadata is the schema of the array
attrList = [c['name'] for c in self.metadata['columns']]
sensitiveIdx = attrList.index(self.sensitiveAttribute)
synKnown = synT[:, [
i for i in range(len(attrList)) if i != sensitiveIdx
]]
synSensitive = synT[:, sensitiveIdx]
n, k = synKnown.shape
# Centre independent variables for better regression performance
self.scaleFactor = mean(synKnown, axis=0)
synKnownScaled = synKnown - self.scaleFactor
synKnownScaled = concatenate(
[ones((len(synKnownScaled), 1)), synKnownScaled],
axis=1) # append all ones for inclu intercept in beta vector
# Get MLE for linear coefficients
self.RegressionModel.fit(synKnownScaled, synSensitive)
self.coefficients = self.RegressionModel.coef_
self.sigma = sum((synSensitive - synKnownScaled.dot(self.coefficients))
**2) / (n - k)
logger.debug('Finished training regression model')
self.trained = True
def get_likelihood(self, targetKnown, targetSensitive):
"""
Calculate the adversary's likelihood over the target's sensitive value
:param targetKnown: ndarray or DataFrame: Partial target record with known attributes
:param targetSensitive: float: Target's sensitive attribute value
:return:
"""
assert self.trained, 'Attack must first be trained on some data before can predict sensitive target value'
targetKnown = convert_df_to_array(
targetKnown,
self.metadata) # extract attribute values for known attributes
targetKnownScaled = targetKnown - self.scaleFactor
targetKnownScaled = concatenate(
[ones((len(targetKnownScaled), 1)), targetKnownScaled], axis=1)
pdfLikelihood = norm(loc=targetKnownScaled.dot(self.coefficients),
scale=sqrt(self.sigma))
return pdfLikelihood.pdf(targetSensitive)
def attack(self, synT):
""" Makes a guess about target's presence in the training set of the model that produced the synthetic input data
:param synT: ndarray or DataFrame: A synthetic dataset
"""
assert self.trained, 'Attack must first be trained before can predict membership'
if self.FeatureSet is not None:
synT = stack([self.FeatureSet.extract(s) for s in synT])
else:
if isinstance(synT[0], DataFrame):
synT = stack([
convert_df_to_array(s, self.metadata).flatten()
for s in synT
])
else:
synT = stack([s.flatten() for s in synT])
return round(self.AttackClassifier.predict(synT),
0).astype(int).tolist()
def attack(self, targetKnown):
"""
Makes a guess about the target's secret attribute from the synthetic data
:param targetKnown: ndarray or DataFrame: Partial target record with known attributes
:return guess: float: Guess about the target's sensitive attribute value
"""
assert self.trained, 'Attack must first be trained on some data before can predict sensitive target value'
# Centre target record attributes
if isinstance(targetKnown, DataFrame):
targetKnown = convert_df_to_array(targetKnown, self.metadata)
else:
assert isinstance(
targetKnown, ndarray), f'Unknown data type {type(targetKnown)}'
targetKnownScaled = targetKnown - self.scaleFactor
targetKnownScaled = concatenate(
[ones((len(targetKnownScaled), 1)), targetKnownScaled], axis=1)
return targetKnownScaled.dot(self.coefficients)
def get_probability_of_success(self, synT, secret):
"""Calculate probability that attacker correctly predicts whether target was present in model's training data
:param synT: ndarray or DataFrame: A synthetic dataset
:param secret: int: Target's true secret. Either LABEL_IN=1 or LABEL_OUT=0
"""
assert self.trained, 'Attack must first be trained on some random data before can predict membership of target data'
if self.FeatureSet is not None:
synT = stack([self.FeatureSet.extract(s) for s in synT])
else:
if isinstance(synT[0], DataFrame):
synT = stack([
convert_df_to_array(s, self.metadata).flatten()
for s in synT
])
else:
synT = stack([s.flatten() for s in synT])
probs = self.AttackClassifier.predict_proba(synT)
return [p[s] for p, s in zip(probs, secret)]
def evaluate_ai(GenModel, rawWithoutTargets, targetRecords, targetIDs, rawA,
rawTindices, sensitiveAttribute, sizeSynT, nSynT, metadata):
logger.info(
f'Start evaluation of generative target model {GenModel.__name__} on {len(targetIDs)} targets under MLE-AI.'
)
results = {
'LinearRegression': {
'Target': [],
'TrueValue': [],
'ProbCorrectPrior': [],
'MLERawT': [],
'SigmaRawT': [],
'ProbCorrectRawT': [],
'MLESynT': [],
'SigmaSynT': [],
'ProbCorrectSynT': [],
'TestRun': []
}
}
for nr, rt in enumerate(rawTindices):
logger.info(f'Raw target test set {nr+1}/{len(rawTindices)}')
# Get raw target test set
rawT = rawWithoutTargets.loc[rt, :]
# Get baseline from raw data
AttackBaseline = AttributeInferenceAttackLinearRegression(
sensitiveAttribute, metadata, rawA)
logger.info(f'Train Attack {AttackBaseline.__name__} on RawT')
AttackBaseline.train(rawT)
# Train generative model on raw data and sample synthetic copies
logger.info(f'Start fitting {GenModel.__class__.__name__} to RawT')
if GenModel.datatype is ndarray:
rawT = convert_df_to_array(rawT, metadata)
GenModel.fit(rawT)
logger.info(
f'Sample {nSynT} copies of synthetic data from {GenModel.__class__.__name__}'
)
synT = [GenModel.generate_samples(sizeSynT) for _ in range(nSynT)]
logger.info(
f'Start Attack evaluation on SynT for {len(targetIDs)} targets')
with Pool(processes=PROCESSES) as pool:
tasks = [(s, targetRecords, targetIDs, sensitiveAttribute,
AttackBaseline, metadata, rawA) for s in synT]
resList = pool.map(worker_run_mleai, tasks)
# Gather results
for res in resList:
for k, v in res[AttackBaseline.__name__].items():
results[AttackBaseline.__name__][k].extend(v)
results[AttackBaseline.__name__]['TestRun'].extend(
[f'Run {nr + 1}' for _ in range(len(targetIDs))])
return results
def worker_run_mia(params):
GenModel, attacksList, target, targetID, rawWithoutTargets, metadata, rawAidx, rawTindices, sizeRawT, sizeSynT, nSynT, nSynA, nShadows = params
# Generate shadow model data for training attacks on this target
if GenModel.datatype is DataFrame:
rawA = rawWithoutTargets.loc[rawAidx, :]
else:
rawA = convert_df_to_array(rawWithoutTargets.loc[rawAidx, :], metadata)
target = convert_df_to_array(target, metadata)
synA, labelsSynA = generate_mia_shadow_data_shufflesplit(
GenModel, target, rawA, sizeRawT, sizeSynT, nShadows, nSynA)
for Attack in attacksList:
Attack.train(synA, labelsSynA)
# Clean up
del synA, labelsSynA
results = {
AM.__name__: {
'TargetID': [],
'TestRun': [],
'ProbSuccess': [],
'RecordPrivacyLossSyn': [],
'RecordPrivacyLossRaw': [],
'RecordPrivacyGain': []
}
for AM in attacksList
}
for nr, rt in enumerate(rawTindices):
# Generate synthetic datasets from generative model trained on RawT WITHOUT Target
if GenModel.datatype is DataFrame:
rawTout = rawWithoutTargets.loc[rt, :]
else:
rawTout = convert_df_to_array(rawWithoutTargets.loc[rt, :],
metadata)
GenModel.fit(rawTout) # Fit model
synTwithoutTarget = [
GenModel.generate_samples(sizeSynT) for _ in range(nSynT)
]
# Generate synthetic datasets from generative model trained on RawT PLUS Target
if GenModel.datatype is DataFrame:
rawTin = rawTout.append(target)
else:
if len(target.shape) == 1:
target = target.reshape(1, len(target))
rawTin = concatenate([rawTout, target])
GenModel.fit(rawTin)
synTwithTarget = [
GenModel.generate_samples(sizeSynT) for _ in range(nSynT)
]
# Create balanced test dataset
synT = synTwithTarget + synTwithoutTarget
labelsSynT = [LABEL_IN for _ in range(len(synTwithTarget))
] + [LABEL_OUT for _ in range(len(synTwithoutTarget))]
# Run attacks on synthetic datasets from target generative model
for AM in attacksList:
res = run_mia(AM, synT, labelsSynT, targetID, nr)
for k, v in res.items():
results[AM.__name__][k].extend(v)
del synT, labelsSynT
return results