def predictD(self, dataLoader, numClasses):
#Basic variable setup
numSamples = len(dataLoader.dataset)
xTest, yTest = DMP.DataLoaderToTensor(dataLoader)
#Randomly select which samples are predicted by which model
sampleAssignments = numpy.random.randint(0, self.numModels,
(numSamples, ))
#memory for the solution
yPred = torch.zeros(numSamples, numClasses)
#Go through the models, get the sample subset they need to predict on, then do the prediction
for modelIndex in range(0, self.numModels):
currentModelPlus = self.modelPlusList[modelIndex]
#Filter the data to get the samples only the currrent model should predict on
dataLoaderSubset = self.filterSamplesIntoDataLoader(
modelIndex, sampleAssignments, xTest, yTest,
currentModelPlus.batchSize)
#Get the array of correct predictions
yPredCurrent = currentModelPlus.predictD(dataLoaderSubset,
numClasses)
currentIndex = 0
#Go through every sample
for i in range(0, numSamples):
#Check if the current sample was predicted using the current model
if sampleAssignments[i] == modelIndex:
yPred[i] = yPredCurrent[currentIndex]
currentIndex = currentIndex + 1
return yPred
def SelfAttentionGradientAttack(device, epsMax, numSteps, modelListPlus,
coefficientArray, dataLoader, clipMin,
clipMax):
#Basic graident variable setup
xClean, yClean = DMP.DataLoaderToTensor(dataLoader)
xAdv = xClean #Set the initial adversarial samples
numSamples = len(
dataLoader.dataset) #Get the total number of samples to attack
xShape = DMP.GetOutputShape(
dataLoader
) #Get the shape of the input (there may be easier way to do this)
#Compute eps step
epsStep = epsMax / numSteps
dataLoaderCurrent = dataLoader
for i in range(0, numSteps):
print("Running Step=", i)
xGradientCumulative = torch.zeros(numSamples, xShape[0], xShape[1],
xShape[2])
for m in range(0, len(modelListPlus)):
currentModelPlus = modelListPlus[m]
#First get the gradient from the model
xGradientCurrent = FGSMNativeGradient(device, dataLoaderCurrent,
currentModelPlus)
#Resize the graident to be the correct size
xGradientCurrent = torch.nn.functional.interpolate(
xGradientCurrent, size=(xShape[1], xShape[2]))
#Add the current computed gradient to the result
if currentModelPlus.modelName.find("ViT") >= 0:
attmap = GetAttention(dataLoaderCurrent, currentModelPlus)
xGradientCumulative = xGradientCumulative + coefficientArray[
m] * xGradientCurrent * attmap
else:
xGradientCumulative = xGradientCumulative + coefficientArray[
m] * xGradientCurrent
#Compute the sign of the graident and create the adversarial example
xAdv = xAdv + epsStep * xGradientCumulative.sign()
#Do the clipping
xAdv = torch.clamp(xAdv, clipMin, clipMax)
#Convert the result to dataloader
dataLoaderCurrent = DMP.TensorToDataLoader(
xAdv,
yClean,
transforms=None,
batchSize=dataLoader.batch_size,
randomizer=None)
return dataLoaderCurrent
def AdaptiveAttackShuffleDefense():
#Corresponding tag for saving files
#First part indicates the type of defense, second part indidcates the synthetic model and last part indicates the strenght of the attack (100%)
saveTag = "ViT-L-16, ViT-32(ImageNet21K), p100"
device = torch.device("cuda")
#Attack parameters
numAttackSamples = 1000
epsForAttacks = 0.031
clipMin = 0.0
clipMax = 1.0
#Parameters of training the synthetic model
imgSize = 224
batchSize = 32
numClasses = 10
numIterations = 4
epochsPerIteration = 10
epsForAug = 0.1 #when generating synthetic data, this value is eps for FGSM used to generate synthetic data
learningRate = (3e-2) / 2 #Learning rate of the synthetic model
#Load the training dataset, validation dataset and the defense
valLoader, defense = LoadShuffleDefenseAndCIFAR10()
trainLoader = DMP.GetCIFAR10Training(imgSize, batchSize)
#Get the clean data
xTest, yTest = DMP.DataLoaderToTensor(valLoader)
cleanLoader = DMP.GetCorrectlyIdentifiedSamplesBalancedDefense(
defense, numAttackSamples, valLoader, numClasses)
#Create the synthetic model
syntheticDir = "Models//imagenet21k_ViT-B_32.npz"
config = CONFIGS["ViT-B_32"]
syntheticModel = VisionTransformer(config,
imgSize,
zero_head=True,
num_classes=numClasses)
syntheticModel.load_from(numpy.load(syntheticDir))
syntheticModel.to(device)
#Do the attack
oracle = defense
dataLoaderForTraining = trainLoader
optimizerName = "sgd"
#Last line does the attack
AttackWrappersAdaptiveBlackBox.AdaptiveAttack(
saveTag, device, oracle, syntheticModel, numIterations,
epochsPerIteration, epsForAug, learningRate, optimizerName,
dataLoaderForTraining, cleanLoader, numClasses, epsForAttacks, clipMin,
clipMax)
def AddLoader(self, dataLoaderName, dataLoader):
#Torch limits the amount of data we can save to disk so we must use numpy to save
#torch.save(dataLoader, self.homeDir+dataLoaderName)
#First convert the tensor to a dataloader
xDataPytorch, yDataPytorch = DMP.DataLoaderToTensor(dataLoader)
#Second conver the pytorch arrays to numpy arrays for saving
xDataNumpy = xDataPytorch.cpu().detach().numpy()
yDataNumpy = yDataPytorch.cpu().detach().numpy()
#Save the data using numpy
numpy.save(self.homeDir + dataLoaderName + "XData", xDataNumpy)
numpy.save(self.homeDir + dataLoaderName + "YData", yDataNumpy)
#Save the file location string so we can re-load later
self.dataLoaderDirList.append(dataLoaderName)
#Delete the dataloader and associated variables from memory
del dataLoader
del xDataPytorch
del yDataPytorch
del xDataNumpy
del yDataNumpy
def validateD(self, dataLoader):
#Basic variable setup
numSamples = len(dataLoader.dataset)
xTest, yTest = DMP.DataLoaderToTensor(dataLoader)
#Randomly select which samples are predicted by which model
sampleAssignments = numpy.random.randint(0, self.numModels,
(numSamples, ))
acc = 0
#Go through the models, get the sample subset they need to predict on, then do the prediction
for modelIndex in range(0, self.numModels):
currentModelPlus = self.modelPlusList[modelIndex]
#Filter the data to get the samples only the currrent model should predict on
dataLoaderSubset = self.filterSamplesIntoDataLoader(
modelIndex, sampleAssignments, xTest, yTest,
currentModelPlus.batchSize)
#Get the array of correct predictions
currentAccArray = currentModelPlus.validateDA(dataLoaderSubset)
acc = acc + currentAccArray.sum()
#We have all the correcly predicted samples, now compute the final accuracy
acc = acc / float(numSamples)
return acc
def RaySAttack(model, epsMax, queryLimit, cleanLoader):
xClean, yClean = DMP.DataLoaderToTensor(cleanLoader)
rayS = RayS(model, epsilon=epsMax, order = np.inf)
xAdv = torch.zeros((xClean.shape[0], xClean.shape[1], xClean.shape[2], xClean.shape[3]))
#Go through and attack the samples
for i in range(0, xClean.shape[0]):
print(i)
start = time.time()
#yCurrent = torch.zeros((1,))
#yCurrent[0] =
xAdvCurrent, stop_queries, dist, isLessDist = rayS.attack_hard_label(xClean[i].cuda(), yClean[i].cuda(), target=None, query_limit=queryLimit, seed=None)
xAdvCurrent = xAdvCurrent.cpu().detach()
dist = torch.dist(xAdvCurrent, xClean[i], np.inf)
if dist>epsMax:
print("Attack failed, returning clean sample instead.")
xAdv[i] = xClean[i]
else:
xAdv[i] = xAdvCurrent
end = time.time()
print("Time Elapsed:", end-start)
#Put solution in dataloader and return
advLoader = DMP.TensorToDataLoader(xAdv, yClean, transforms = None, batchSize = cleanLoader.batch_size, randomizer = None)
return advLoader
def GetFirstCorrectlyOverlappingSamplesBalanced(device, sampleNum, numClasses,
dataLoader, modelPlusList):
numModels = len(modelPlusList)
totalSampleNum = len(dataLoader.dataset)
#First check if modelA needs resize
xTestOrig, yTestOrig = DMP.DataLoaderToTensor(dataLoader)
#We need to resize first
if modelPlusList[0].imgSizeH != xTestOrig.shape[2] or modelPlusList[
0].imgSizeW != xTestOrig.shape[3]:
xTestOrigResize = torch.zeros(xTestOrig.shape[0], 3,
modelPlusList[0].imgSizeH,
modelPlusList[0].imgSizeW)
rs = torchvision.transforms.Resize(
(modelPlusList[0].imgSizeH,
modelPlusList[0].imgSizeW)) #resize the samples for model A
#Go through every sample
for i in range(0, xTestOrig.shape[0]):
xTestOrigResize[i] = rs(
xTestOrig[i]) #resize to match dimensions required by modelA
#Make a new dataloader
dataLoader = DMP.TensorToDataLoader(xTestOrigResize,
yTestOrig,
transforms=None,
batchSize=dataLoader.batch_size,
randomizer=None)
#Get accuracy array for each model
accArrayCumulative = torch.zeros(
totalSampleNum
) #Create an array with one entry for ever sample in the dataset
for i in range(0, numModels):
accArray = modelPlusList[i].validateDA(dataLoader)
accArrayCumulative = accArrayCumulative + accArray
#Do some basic error checking
if sampleNum % numClasses != 0:
raise ValueError(
"Number of samples not divisable by the number of classes")
#Basic variable setup
samplePerClassCount = torch.zeros(
numClasses) #keep track of samples per class
maxRequireSamplesPerClass = int(
sampleNum / numClasses) #Max number of samples we need per class
xTest, yTest = DMP.DataLoaderToTensor(
dataLoader) #Get all the data as tensors
#Memory for the solution
xClean = torch.zeros(sampleNum, 3, modelPlusList[0].imgSizeH,
modelPlusList[0].imgSizeW)
yClean = torch.zeros(sampleNum)
sampleIndexer = 0
#Go through all the samples
for i in range(0, totalSampleNum):
currentClass = int(yTest[i])
#Check to make sure all classifiers identify the sample correctly AND we don't have enough of this class yet
if accArrayCumulative[i] == numModels and samplePerClassCount[
currentClass] < maxRequireSamplesPerClass:
#xClean[sampleIndexer] = rs(xTest[i]) #resize to match dimensions required by modelA
xClean[sampleIndexer] = xTest[i]
yClean[sampleIndexer] = yTest[i]
sampleIndexer = sampleIndexer + 1 #update the indexer
samplePerClassCount[currentClass] = samplePerClassCount[
currentClass] + 1 #Update the number of samples for this class
#Check the over all number of samples as well
if sampleIndexer != sampleNum:
print("Not enough clean samples found.")
#Do some error checking on the classes
for i in range(0, numClasses):
if samplePerClassCount[i] != maxRequireSamplesPerClass:
print(samplePerClassCount[i])
raise ValueError("We didn't find enough of class: " + str(i))
#Conver the solution into a dataloader
cleanDataLoader = DMP.TensorToDataLoader(
xClean,
yClean,
transforms=None,
batchSize=modelPlusList[0].batchSize,
randomizer=None)
#Do one last check to make sure all samples identify the clean loader correctly
for i in range(0, numModels):
cleanAcc = modelPlusList[i].validateD(cleanDataLoader)
if cleanAcc != 1.0:
print("Clean Acc " + modelPlusList[i].modelName + ":", cleanAcc)
raise ValueError("The clean accuracy is not 1.0")
#All error checking done, return the clean balanced loader
return cleanDataLoader