def filterSamplesIntoDataLoader(self, modelIndex, sampleAssignments, xTest,
yTest, batchSize):
numSamples = xTest.shape[0]
count = 0
#First determine how many samples the model needs to evaluate
for i in range(0, numSamples):
if sampleAssignments[i] == modelIndex:
count = count + 1
#We now know how many samples we need to evalute on, time to save them
xSubset = torch.zeros(
(count, xTest.shape[1], xTest.shape[2], xTest.shape[3]))
ySubset = torch.zeros((count))
#Go through and get the samples
currentIndex = 0
for i in range(0, numSamples):
if sampleAssignments[i] == modelIndex:
xSubset[currentIndex] = xTest[i]
ySubset[currentIndex] = yTest[i]
currentIndex = currentIndex + 1
#Do some basic error checkings
if currentIndex != count:
raise ValueError(
"Something went wrong in the indexing, expected count did not match actual count."
)
#Put the data into a dataloader and return
dataLoaderSubset = DMP.TensorToDataLoader(xSubset,
ySubset,
transforms=None,
batchSize=batchSize,
randomizer=None)
return dataLoaderSubset
def MIMNativePytorch(device, dataLoader, model, decayFactor, epsilonMax,
epsilonStep, numSteps, clipMin, clipMax, targeted):
model.eval() #Change model to evaluation mode for the attack
#Generate variables for storing the adversarial examples
numSamples = len(
dataLoader.dataset) #Get the total number of samples to attack
xShape = DMP.GetOutputShape(
dataLoader
) #Get the shape of the input (there may be easier way to do this)
xAdv = torch.zeros(numSamples, xShape[0], xShape[1], xShape[2])
yClean = torch.zeros(numSamples)
advSampleIndex = 0
batchSize = 0 #just do dummy initalization, will be filled in later
loss = torch.nn.CrossEntropyLoss()
tracker = 0
#Go through each sample
for xData, yData in dataLoader:
batchSize = xData.shape[
0] #Get the batch size so we know indexing for saving later
tracker = tracker + batchSize
print("Processing up to sample=", tracker)
#Put the data from the batch onto the device
xAdvCurrent = xData.to(device)
yCurrent = yData.type(torch.LongTensor).to(device)
# Set requires_grad attribute of tensor. Important for attack. (Pytorch comment, not mine)
#Initalize memory for the gradient momentum
gMomentum = torch.zeros(batchSize, xShape[0], xShape[1], xShape[2])
#Do the attack for a number of steps
for attackStep in range(0, numSteps):
xAdvCurrent.requires_grad = True
outputs = model(xAdvCurrent)
model.zero_grad()
cost = loss(outputs, yCurrent).to(device)
cost.backward()
#Update momentum term
gMomentum = decayFactor * gMomentum + GradientNormalizedByL1(
xAdvCurrent.grad)
#Update the adversarial sample
if targeted == True:
advTemp = xAdvCurrent - (epsilonStep *
torch.sign(gMomentum)).to(device)
else:
advTemp = xAdvCurrent + (epsilonStep *
torch.sign(gMomentum)).to(device)
#Adding clipping to maintain the range
xAdvCurrent = torch.clamp(advTemp, min=clipMin,
max=clipMax).detach_()
#Save the adversarial images from the batch
for j in range(0, batchSize):
xAdv[advSampleIndex] = xAdvCurrent[j]
yClean[advSampleIndex] = yData[j]
advSampleIndex = advSampleIndex + 1 #increment the sample index
#All samples processed, now time to save in a dataloader and return
advLoader = DMP.TensorToDataLoader(
xAdv,
yClean,
transforms=None,
batchSize=dataLoader.batch_size,
randomizer=None) #use the same batch size as the original loader
return advLoader
def GetLoaderAtIndex(self, index):
currentDataLoaderDir = self.homeDir + self.dataLoaderDirList[index]
#First load the numpy arrays
xData = numpy.load(currentDataLoaderDir + "XData.npy")
yData = numpy.load(currentDataLoaderDir + "YData.npy")
#Create a dataloader
currentDataLoader = DMP.TensorToDataLoader(torch.from_numpy(xData),
torch.from_numpy(yData),
transforms=None,
batchSize=self.batchSize,
randomizer=None)
#currentDataLoader = torch.load(currentDataLoaderDir)
#Do some memory clean up
del xData
del yData
return currentDataLoader
def SelfAttentionGradientAttack(device, epsMax, numSteps, modelListPlus,
coefficientArray, dataLoader, clipMin,
clipMax):
#Basic graident variable setup
xClean, yClean = DMP.DataLoaderToTensor(dataLoader)
xAdv = xClean #Set the initial adversarial samples
numSamples = len(
dataLoader.dataset) #Get the total number of samples to attack
xShape = DMP.GetOutputShape(
dataLoader
) #Get the shape of the input (there may be easier way to do this)
#Compute eps step
epsStep = epsMax / numSteps
dataLoaderCurrent = dataLoader
for i in range(0, numSteps):
print("Running Step=", i)
xGradientCumulative = torch.zeros(numSamples, xShape[0], xShape[1],
xShape[2])
for m in range(0, len(modelListPlus)):
currentModelPlus = modelListPlus[m]
#First get the gradient from the model
xGradientCurrent = FGSMNativeGradient(device, dataLoaderCurrent,
currentModelPlus)
#Resize the graident to be the correct size
xGradientCurrent = torch.nn.functional.interpolate(
xGradientCurrent, size=(xShape[1], xShape[2]))
#Add the current computed gradient to the result
if currentModelPlus.modelName.find("ViT") >= 0:
attmap = GetAttention(dataLoaderCurrent, currentModelPlus)
xGradientCumulative = xGradientCumulative + coefficientArray[
m] * xGradientCurrent * attmap
else:
xGradientCumulative = xGradientCumulative + coefficientArray[
m] * xGradientCurrent
#Compute the sign of the graident and create the adversarial example
xAdv = xAdv + epsStep * xGradientCumulative.sign()
#Do the clipping
xAdv = torch.clamp(xAdv, clipMin, clipMax)
#Convert the result to dataloader
dataLoaderCurrent = DMP.TensorToDataLoader(
xAdv,
yClean,
transforms=None,
batchSize=dataLoader.batch_size,
randomizer=None)
return dataLoaderCurrent
def formatDataLoader(self, dataLoader):
sampleShape = DMP.GetOutputShape(dataLoader)
#Check if we need to do resizing, if not just return the original loader
if sampleShape[1] == self.imgSizeH and sampleShape[2] == self.imgSizeW:
return dataLoader
else: #We need to do resizing
print("Resize required. Processing now.")
p = torchvision.transforms.ToPILImage()
t = torchvision.transforms.ToTensor()
numSamples = len(dataLoader.dataset)
sampleShape = DMP.GetOutputShape(
dataLoader) #Get the output shape from the dataloader
sampleIndex = 0
batchTracker = 0
xData = torch.zeros(numSamples, sampleShape[0], self.imgSizeH,
self.imgSizeW)
yData = torch.zeros(numSamples)
#Go through and process the data in batches...kind of
for i, (input, target) in enumerate(dataLoader):
batchSize = input.shape[
0] #Get the number of samples used in each batch
#print("Resize processing up to=", batchTracker)
batchTracker = batchTracker + batchSize
#Save the samples from the batch in a separate tensor
for batchIndex in range(0, batchSize):
#Convert to pil image, resize, convert back to tensor
xData[sampleIndex] = t(
self.resizeTransform(p(input[batchIndex])))
yData[sampleIndex] = target[batchIndex]
sampleIndex = sampleIndex + 1 #increment the sample index
#All the data has been resized, time to put in the dataloader
newDataLoader = DMP.TensorToDataLoader(xData,
yData,
transforms=None,
batchSize=self.batchSize,
randomizer=None)
#Note we don't use the original batch size because the image may have become larger
#i.e. to large to fit in GPU memory so we use the batch specified in the ModelPlus constructor
return newDataLoader
def RaySAttack(model, epsMax, queryLimit, cleanLoader):
xClean, yClean = DMP.DataLoaderToTensor(cleanLoader)
rayS = RayS(model, epsilon=epsMax, order = np.inf)
xAdv = torch.zeros((xClean.shape[0], xClean.shape[1], xClean.shape[2], xClean.shape[3]))
#Go through and attack the samples
for i in range(0, xClean.shape[0]):
print(i)
start = time.time()
#yCurrent = torch.zeros((1,))
#yCurrent[0] =
xAdvCurrent, stop_queries, dist, isLessDist = rayS.attack_hard_label(xClean[i].cuda(), yClean[i].cuda(), target=None, query_limit=queryLimit, seed=None)
xAdvCurrent = xAdvCurrent.cpu().detach()
dist = torch.dist(xAdvCurrent, xClean[i], np.inf)
if dist>epsMax:
print("Attack failed, returning clean sample instead.")
xAdv[i] = xClean[i]
else:
xAdv[i] = xAdvCurrent
end = time.time()
print("Time Elapsed:", end-start)
#Put solution in dataloader and return
advLoader = DMP.TensorToDataLoader(xAdv, yClean, transforms = None, batchSize = cleanLoader.batch_size, randomizer = None)
return advLoader
def FGSMNativePytorch(device, dataLoader, model, epsilonMax, clipMin, clipMax,
targeted):
model.eval() #Change model to evaluation mode for the attack
#Generate variables for storing the adversarial examples
numSamples = len(
dataLoader.dataset) #Get the total number of samples to attack
xShape = DMP.GetOutputShape(
dataLoader
) #Get the shape of the input (there may be easier way to do this)
xAdv = torch.zeros(numSamples, xShape[0], xShape[1], xShape[2])
yClean = torch.zeros(numSamples)
advSampleIndex = 0
batchSize = 0 #just do dummy initalization, will be filled in later
#Go through each sample
tracker = 0
for xData, yData in dataLoader:
batchSize = xData.shape[
0] #Get the batch size so we know indexing for saving later
tracker = tracker + batchSize
#print("Processing up to sample=", tracker)
#Put the data from the batch onto the device
xDataTemp = torch.from_numpy(xData.cpu().detach().numpy()).to(device)
yData = yData.type(torch.LongTensor).to(device)
# Set requires_grad attribute of tensor. Important for attack. (Pytorch comment, not mine)
xDataTemp.requires_grad = True
# Forward pass the data through the model
output = model(xDataTemp)
# Calculate the loss
loss = torch.nn.CrossEntropyLoss(ignore_index=10,
size_average=True,
reduce=True)
# Zero all existing gradients
model.zero_grad()
# Calculate gradients of model in backward pass
cost = loss(output, yData).to(device)
cost.backward()
# Collect datagrad
#xDataGrad = xDataTemp.grad.data
###Here we actual compute the adversarial sample
# Collect the element-wise sign of the data gradient
signDataGrad = xDataTemp.grad.data.sign()
# Create the perturbed image by adjusting each pixel of the input image
#print("xData:", xData.is_cuda)
#print("SignGrad:", signDataGrad.is_cuda)
if targeted == True:
perturbedImage = xData - epsilonMax * signDataGrad.cpu().detach(
) #Go negative of gradient
else:
perturbedImage = xData + epsilonMax * signDataGrad.cpu().detach()
# Adding clipping to maintain the range
perturbedImage = torch.clamp(perturbedImage, clipMin, clipMax)
#Save the adversarial images from the batch
for j in range(0, batchSize):
xAdv[advSampleIndex] = perturbedImage[j]
yClean[advSampleIndex] = yData[j]
advSampleIndex = advSampleIndex + 1 #increment the sample index
#Not sure if we need this but do some memory clean up
del xDataTemp
del signDataGrad
torch.cuda.empty_cache()
#All samples processed, now time to save in a dataloader and return
advLoader = DMP.TensorToDataLoader(
xAdv,
yClean,
transforms=None,
batchSize=dataLoader.batch_size,
randomizer=None) #use the same batch size as the original loader
return advLoader
def GetFirstCorrectlyOverlappingSamplesBalanced(device, sampleNum, numClasses,
dataLoader, modelPlusList):
numModels = len(modelPlusList)
totalSampleNum = len(dataLoader.dataset)
#First check if modelA needs resize
xTestOrig, yTestOrig = DMP.DataLoaderToTensor(dataLoader)
#We need to resize first
if modelPlusList[0].imgSizeH != xTestOrig.shape[2] or modelPlusList[
0].imgSizeW != xTestOrig.shape[3]:
xTestOrigResize = torch.zeros(xTestOrig.shape[0], 3,
modelPlusList[0].imgSizeH,
modelPlusList[0].imgSizeW)
rs = torchvision.transforms.Resize(
(modelPlusList[0].imgSizeH,
modelPlusList[0].imgSizeW)) #resize the samples for model A
#Go through every sample
for i in range(0, xTestOrig.shape[0]):
xTestOrigResize[i] = rs(
xTestOrig[i]) #resize to match dimensions required by modelA
#Make a new dataloader
dataLoader = DMP.TensorToDataLoader(xTestOrigResize,
yTestOrig,
transforms=None,
batchSize=dataLoader.batch_size,
randomizer=None)
#Get accuracy array for each model
accArrayCumulative = torch.zeros(
totalSampleNum
) #Create an array with one entry for ever sample in the dataset
for i in range(0, numModels):
accArray = modelPlusList[i].validateDA(dataLoader)
accArrayCumulative = accArrayCumulative + accArray
#Do some basic error checking
if sampleNum % numClasses != 0:
raise ValueError(
"Number of samples not divisable by the number of classes")
#Basic variable setup
samplePerClassCount = torch.zeros(
numClasses) #keep track of samples per class
maxRequireSamplesPerClass = int(
sampleNum / numClasses) #Max number of samples we need per class
xTest, yTest = DMP.DataLoaderToTensor(
dataLoader) #Get all the data as tensors
#Memory for the solution
xClean = torch.zeros(sampleNum, 3, modelPlusList[0].imgSizeH,
modelPlusList[0].imgSizeW)
yClean = torch.zeros(sampleNum)
sampleIndexer = 0
#Go through all the samples
for i in range(0, totalSampleNum):
currentClass = int(yTest[i])
#Check to make sure all classifiers identify the sample correctly AND we don't have enough of this class yet
if accArrayCumulative[i] == numModels and samplePerClassCount[
currentClass] < maxRequireSamplesPerClass:
#xClean[sampleIndexer] = rs(xTest[i]) #resize to match dimensions required by modelA
xClean[sampleIndexer] = xTest[i]
yClean[sampleIndexer] = yTest[i]
sampleIndexer = sampleIndexer + 1 #update the indexer
samplePerClassCount[currentClass] = samplePerClassCount[
currentClass] + 1 #Update the number of samples for this class
#Check the over all number of samples as well
if sampleIndexer != sampleNum:
print("Not enough clean samples found.")
#Do some error checking on the classes
for i in range(0, numClasses):
if samplePerClassCount[i] != maxRequireSamplesPerClass:
print(samplePerClassCount[i])
raise ValueError("We didn't find enough of class: " + str(i))
#Conver the solution into a dataloader
cleanDataLoader = DMP.TensorToDataLoader(
xClean,
yClean,
transforms=None,
batchSize=modelPlusList[0].batchSize,
randomizer=None)
#Do one last check to make sure all samples identify the clean loader correctly
for i in range(0, numModels):
cleanAcc = modelPlusList[i].validateD(cleanDataLoader)
if cleanAcc != 1.0:
print("Clean Acc " + modelPlusList[i].modelName + ":", cleanAcc)
raise ValueError("The clean accuracy is not 1.0")
#All error checking done, return the clean balanced loader
return cleanDataLoader