def train(self, synA, labels): """ Train a membership inference attack on a labelled training set :param synA: list of ndarrays: A list of synthetic datasets :param labels: list: A list of labels that indicate whether target was in the training data (LABEL_IN=1) or not (LABEL_OUT=0) """ if self.FeatureSet is not None: synA = stack([self.FeatureSet.extract(s) for s in synA]) else: if isinstance(synA[0], DataFrame): synA = [self._impute_missing_values(s) for s in synA] synA = stack([ convert_df_to_array(s, self.metadata).flatten() for s in synA ]) else: synA = stack([s.flatten() for s in synA]) if not isinstance(labels, ndarray): labels = array(labels) self.AttackClassifier.fit(synA, labels) logger.debug('Finished training MIA distinguisher') self.trained = True del synA, labels
def get_confidence(self, synT, secret): """Calculate probability that attacker correctly predicts whether target was present in model's training data""" assert self.trained, 'Attack must first be trained.' if self.FeatureSet is not None: synT = stack([self.FeatureSet.extract(s) for s in synT]) else: if isinstance(synT[0], DataFrame): synT = stack([ convert_df_to_array(s, self.metadata).flatten() for s in synT ]) else: synT = stack([s.flatten() for s in synT]) probs = self.Distinguisher.predict_proba(synT) return [p[s] for p, s in zip(probs, secret)]
def train(self, synT): """ Train a MLE attack to reconstruct an unknown sensitive value from a vector of known attributes :param synT: DataFrame: A synthetic dataset of shape (n, k + 1) """ # Split data into known and sensitive if isinstance(synT, DataFrame): assert self.sensitiveAttribute in list( synT), f'DataFrame only contains columns {list(synT)}' synKnown = synT.drop(self.sensitiveAttribute, axis=1) synSensitive = synT[self.sensitiveAttribute] synKnown = convert_df_to_array(synKnown, self.metadata) synSensitive = convert_series_to_array(synSensitive, self.metadata) else: assert isinstance(synT, ndarray), f"Unknown data type {type(synT)}" # If input data is array assume that self.metadata is the schema of the array attrList = [c['name'] for c in self.metadata['columns']] sensitiveIdx = attrList.index(self.sensitiveAttribute) synKnown = synT[:, [ i for i in range(len(attrList)) if i != sensitiveIdx ]] synSensitive = synT[:, sensitiveIdx] n, k = synKnown.shape # Centre independent variables for better regression performance self.scaleFactor = mean(synKnown, axis=0) synKnownScaled = synKnown - self.scaleFactor synKnownScaled = concatenate( [ones((len(synKnownScaled), 1)), synKnownScaled], axis=1) # append all ones for inclu intercept in beta vector # Get MLE for linear coefficients self.RegressionModel.fit(synKnownScaled, synSensitive) self.coefficients = self.RegressionModel.coef_ self.sigma = sum((synSensitive - synKnownScaled.dot(self.coefficients)) **2) / (n - k) logger.debug('Finished training regression model') self.trained = True
def get_likelihood(self, targetKnown, targetSensitive): """ Calculate the adversary's likelihood over the target's sensitive value :param targetKnown: ndarray or DataFrame: Partial target record with known attributes :param targetSensitive: float: Target's sensitive attribute value :return: """ assert self.trained, 'Attack must first be trained on some data before can predict sensitive target value' targetKnown = convert_df_to_array( targetKnown, self.metadata) # extract attribute values for known attributes targetKnownScaled = targetKnown - self.scaleFactor targetKnownScaled = concatenate( [ones((len(targetKnownScaled), 1)), targetKnownScaled], axis=1) pdfLikelihood = norm(loc=targetKnownScaled.dot(self.coefficients), scale=sqrt(self.sigma)) return pdfLikelihood.pdf(targetSensitive)
def attack(self, synT): """ Makes a guess about target's presence in the training set of the model that produced the synthetic input data :param synT: ndarray or DataFrame: A synthetic dataset """ assert self.trained, 'Attack must first be trained before can predict membership' if self.FeatureSet is not None: synT = stack([self.FeatureSet.extract(s) for s in synT]) else: if isinstance(synT[0], DataFrame): synT = stack([ convert_df_to_array(s, self.metadata).flatten() for s in synT ]) else: synT = stack([s.flatten() for s in synT]) return round(self.AttackClassifier.predict(synT), 0).astype(int).tolist()
def attack(self, targetKnown): """ Makes a guess about the target's secret attribute from the synthetic data :param targetKnown: ndarray or DataFrame: Partial target record with known attributes :return guess: float: Guess about the target's sensitive attribute value """ assert self.trained, 'Attack must first be trained on some data before can predict sensitive target value' # Centre target record attributes if isinstance(targetKnown, DataFrame): targetKnown = convert_df_to_array(targetKnown, self.metadata) else: assert isinstance( targetKnown, ndarray), f'Unknown data type {type(targetKnown)}' targetKnownScaled = targetKnown - self.scaleFactor targetKnownScaled = concatenate( [ones((len(targetKnownScaled), 1)), targetKnownScaled], axis=1) return targetKnownScaled.dot(self.coefficients)
def get_probability_of_success(self, synT, secret): """Calculate probability that attacker correctly predicts whether target was present in model's training data :param synT: ndarray or DataFrame: A synthetic dataset :param secret: int: Target's true secret. Either LABEL_IN=1 or LABEL_OUT=0 """ assert self.trained, 'Attack must first be trained on some random data before can predict membership of target data' if self.FeatureSet is not None: synT = stack([self.FeatureSet.extract(s) for s in synT]) else: if isinstance(synT[0], DataFrame): synT = stack([ convert_df_to_array(s, self.metadata).flatten() for s in synT ]) else: synT = stack([s.flatten() for s in synT]) probs = self.AttackClassifier.predict_proba(synT) return [p[s] for p, s in zip(probs, secret)]
def evaluate_ai(GenModel, rawWithoutTargets, targetRecords, targetIDs, rawA, rawTindices, sensitiveAttribute, sizeSynT, nSynT, metadata): logger.info( f'Start evaluation of generative target model {GenModel.__name__} on {len(targetIDs)} targets under MLE-AI.' ) results = { 'LinearRegression': { 'Target': [], 'TrueValue': [], 'ProbCorrectPrior': [], 'MLERawT': [], 'SigmaRawT': [], 'ProbCorrectRawT': [], 'MLESynT': [], 'SigmaSynT': [], 'ProbCorrectSynT': [], 'TestRun': [] } } for nr, rt in enumerate(rawTindices): logger.info(f'Raw target test set {nr+1}/{len(rawTindices)}') # Get raw target test set rawT = rawWithoutTargets.loc[rt, :] # Get baseline from raw data AttackBaseline = AttributeInferenceAttackLinearRegression( sensitiveAttribute, metadata, rawA) logger.info(f'Train Attack {AttackBaseline.__name__} on RawT') AttackBaseline.train(rawT) # Train generative model on raw data and sample synthetic copies logger.info(f'Start fitting {GenModel.__class__.__name__} to RawT') if GenModel.datatype is ndarray: rawT = convert_df_to_array(rawT, metadata) GenModel.fit(rawT) logger.info( f'Sample {nSynT} copies of synthetic data from {GenModel.__class__.__name__}' ) synT = [GenModel.generate_samples(sizeSynT) for _ in range(nSynT)] logger.info( f'Start Attack evaluation on SynT for {len(targetIDs)} targets') with Pool(processes=PROCESSES) as pool: tasks = [(s, targetRecords, targetIDs, sensitiveAttribute, AttackBaseline, metadata, rawA) for s in synT] resList = pool.map(worker_run_mleai, tasks) # Gather results for res in resList: for k, v in res[AttackBaseline.__name__].items(): results[AttackBaseline.__name__][k].extend(v) results[AttackBaseline.__name__]['TestRun'].extend( [f'Run {nr + 1}' for _ in range(len(targetIDs))]) return results
def worker_run_mia(params): GenModel, attacksList, target, targetID, rawWithoutTargets, metadata, rawAidx, rawTindices, sizeRawT, sizeSynT, nSynT, nSynA, nShadows = params # Generate shadow model data for training attacks on this target if GenModel.datatype is DataFrame: rawA = rawWithoutTargets.loc[rawAidx, :] else: rawA = convert_df_to_array(rawWithoutTargets.loc[rawAidx, :], metadata) target = convert_df_to_array(target, metadata) synA, labelsSynA = generate_mia_shadow_data_shufflesplit( GenModel, target, rawA, sizeRawT, sizeSynT, nShadows, nSynA) for Attack in attacksList: Attack.train(synA, labelsSynA) # Clean up del synA, labelsSynA results = { AM.__name__: { 'TargetID': [], 'TestRun': [], 'ProbSuccess': [], 'RecordPrivacyLossSyn': [], 'RecordPrivacyLossRaw': [], 'RecordPrivacyGain': [] } for AM in attacksList } for nr, rt in enumerate(rawTindices): # Generate synthetic datasets from generative model trained on RawT WITHOUT Target if GenModel.datatype is DataFrame: rawTout = rawWithoutTargets.loc[rt, :] else: rawTout = convert_df_to_array(rawWithoutTargets.loc[rt, :], metadata) GenModel.fit(rawTout) # Fit model synTwithoutTarget = [ GenModel.generate_samples(sizeSynT) for _ in range(nSynT) ] # Generate synthetic datasets from generative model trained on RawT PLUS Target if GenModel.datatype is DataFrame: rawTin = rawTout.append(target) else: if len(target.shape) == 1: target = target.reshape(1, len(target)) rawTin = concatenate([rawTout, target]) GenModel.fit(rawTin) synTwithTarget = [ GenModel.generate_samples(sizeSynT) for _ in range(nSynT) ] # Create balanced test dataset synT = synTwithTarget + synTwithoutTarget labelsSynT = [LABEL_IN for _ in range(len(synTwithTarget)) ] + [LABEL_OUT for _ in range(len(synTwithoutTarget))] # Run attacks on synthetic datasets from target generative model for AM in attacksList: res = run_mia(AM, synT, labelsSynT, targetID, nr) for k, v in res.items(): results[AM.__name__][k].extend(v) del synT, labelsSynT return results