def MIMNativePytorch(device, dataLoader, model, decayFactor, epsilonMax,
epsilonStep, numSteps, clipMin, clipMax, targeted):
model.eval() #Change model to evaluation mode for the attack
#Generate variables for storing the adversarial examples
numSamples = len(
dataLoader.dataset) #Get the total number of samples to attack
xShape = DMP.GetOutputShape(
dataLoader
) #Get the shape of the input (there may be easier way to do this)
xAdv = torch.zeros(numSamples, xShape[0], xShape[1], xShape[2])
yClean = torch.zeros(numSamples)
advSampleIndex = 0
batchSize = 0 #just do dummy initalization, will be filled in later
loss = torch.nn.CrossEntropyLoss()
tracker = 0
#Go through each sample
for xData, yData in dataLoader:
batchSize = xData.shape[
0] #Get the batch size so we know indexing for saving later
tracker = tracker + batchSize
print("Processing up to sample=", tracker)
#Put the data from the batch onto the device
xAdvCurrent = xData.to(device)
yCurrent = yData.type(torch.LongTensor).to(device)
# Set requires_grad attribute of tensor. Important for attack. (Pytorch comment, not mine)
#Initalize memory for the gradient momentum
gMomentum = torch.zeros(batchSize, xShape[0], xShape[1], xShape[2])
#Do the attack for a number of steps
for attackStep in range(0, numSteps):
xAdvCurrent.requires_grad = True
outputs = model(xAdvCurrent)
model.zero_grad()
cost = loss(outputs, yCurrent).to(device)
cost.backward()
#Update momentum term
gMomentum = decayFactor * gMomentum + GradientNormalizedByL1(
xAdvCurrent.grad)
#Update the adversarial sample
if targeted == True:
advTemp = xAdvCurrent - (epsilonStep *
torch.sign(gMomentum)).to(device)
else:
advTemp = xAdvCurrent + (epsilonStep *
torch.sign(gMomentum)).to(device)
#Adding clipping to maintain the range
xAdvCurrent = torch.clamp(advTemp, min=clipMin,
max=clipMax).detach_()
#Save the adversarial images from the batch
for j in range(0, batchSize):
xAdv[advSampleIndex] = xAdvCurrent[j]
yClean[advSampleIndex] = yData[j]
advSampleIndex = advSampleIndex + 1 #increment the sample index
#All samples processed, now time to save in a dataloader and return
advLoader = DMP.TensorToDataLoader(
xAdv,
yClean,
transforms=None,
batchSize=dataLoader.batch_size,
randomizer=None) #use the same batch size as the original loader
return advLoader
def formatDataLoader(self, dataLoader):
sampleShape = DMP.GetOutputShape(dataLoader)
#Check if we need to do resizing, if not just return the original loader
if sampleShape[1] == self.imgSizeH and sampleShape[2] == self.imgSizeW:
return dataLoader
else: #We need to do resizing
print("Resize required. Processing now.")
p = torchvision.transforms.ToPILImage()
t = torchvision.transforms.ToTensor()
numSamples = len(dataLoader.dataset)
sampleShape = DMP.GetOutputShape(
dataLoader) #Get the output shape from the dataloader
sampleIndex = 0
batchTracker = 0
xData = torch.zeros(numSamples, sampleShape[0], self.imgSizeH,
self.imgSizeW)
yData = torch.zeros(numSamples)
#Go through and process the data in batches...kind of
for i, (input, target) in enumerate(dataLoader):
batchSize = input.shape[
0] #Get the number of samples used in each batch
#print("Resize processing up to=", batchTracker)
batchTracker = batchTracker + batchSize
#Save the samples from the batch in a separate tensor
for batchIndex in range(0, batchSize):
#Convert to pil image, resize, convert back to tensor
xData[sampleIndex] = t(
self.resizeTransform(p(input[batchIndex])))
yData[sampleIndex] = target[batchIndex]
sampleIndex = sampleIndex + 1 #increment the sample index
#All the data has been resized, time to put in the dataloader
newDataLoader = DMP.TensorToDataLoader(xData,
yData,
transforms=None,
batchSize=self.batchSize,
randomizer=None)
#Note we don't use the original batch size because the image may have become larger
#i.e. to large to fit in GPU memory so we use the batch specified in the ModelPlus constructor
return newDataLoader
def FGSMNativeGradient(device, dataLoader, modelPlus):
#Basic variable setup
model = modelPlus.model
model.eval() #Change model to evaluation mode for the attack
model.to(device)
sizeCorrectedLoader = modelPlus.formatDataLoader(dataLoader)
#Generate variables for storing the adversarial examples
numSamples = len(sizeCorrectedLoader.dataset
) #Get the total number of samples to attack
xShape = DMP.GetOutputShape(
sizeCorrectedLoader
) #Get the shape of the input (there may be easier way to do this)
xGradient = torch.zeros(numSamples, xShape[0], xShape[1], xShape[2])
yClean = torch.zeros(numSamples)
advSampleIndex = 0
batchSize = 0 #just do dummy initalization, will be filled in later
#Go through each sample
tracker = 0
for xData, yData in sizeCorrectedLoader:
batchSize = xData.shape[
0] #Get the batch size so we know indexing for saving later
tracker = tracker + batchSize
#print("Processing up to sample=", tracker)
#Put the data from the batch onto the device
xDataTemp = torch.from_numpy(xData.cpu().detach().numpy()).to(device)
yData = yData.type(torch.LongTensor).to(device)
# Set requires_grad attribute of tensor. Important for attack. (Pytorch comment, not mine)
xDataTemp.requires_grad = True
# Forward pass the data through the model
output = model(xDataTemp)
# Calculate the loss
loss = torch.nn.CrossEntropyLoss()
# Zero all existing gradients
model.zero_grad()
# Calculate gradients of model in backward pass
cost = loss(output, yData).to(device)
cost.backward()
xDataTempGrad = xDataTemp.grad.data
#Save the adversarial images from the batch
for j in range(0, batchSize):
xGradient[advSampleIndex] = xDataTempGrad[j]
yClean[advSampleIndex] = yData[j]
advSampleIndex = advSampleIndex + 1 #increment the sample index
#Not sure if we need this but do some memory clean up
del xDataTemp
torch.cuda.empty_cache()
#Memory management
del model
torch.cuda.empty_cache()
return xGradient
def SelfAttentionGradientAttack(device, epsMax, numSteps, modelListPlus,
coefficientArray, dataLoader, clipMin,
clipMax):
#Basic graident variable setup
xClean, yClean = DMP.DataLoaderToTensor(dataLoader)
xAdv = xClean #Set the initial adversarial samples
numSamples = len(
dataLoader.dataset) #Get the total number of samples to attack
xShape = DMP.GetOutputShape(
dataLoader
) #Get the shape of the input (there may be easier way to do this)
#Compute eps step
epsStep = epsMax / numSteps
dataLoaderCurrent = dataLoader
for i in range(0, numSteps):
print("Running Step=", i)
xGradientCumulative = torch.zeros(numSamples, xShape[0], xShape[1],
xShape[2])
for m in range(0, len(modelListPlus)):
currentModelPlus = modelListPlus[m]
#First get the gradient from the model
xGradientCurrent = FGSMNativeGradient(device, dataLoaderCurrent,
currentModelPlus)
#Resize the graident to be the correct size
xGradientCurrent = torch.nn.functional.interpolate(
xGradientCurrent, size=(xShape[1], xShape[2]))
#Add the current computed gradient to the result
if currentModelPlus.modelName.find("ViT") >= 0:
attmap = GetAttention(dataLoaderCurrent, currentModelPlus)
xGradientCumulative = xGradientCumulative + coefficientArray[
m] * xGradientCurrent * attmap
else:
xGradientCumulative = xGradientCumulative + coefficientArray[
m] * xGradientCurrent
#Compute the sign of the graident and create the adversarial example
xAdv = xAdv + epsStep * xGradientCumulative.sign()
#Do the clipping
xAdv = torch.clamp(xAdv, clipMin, clipMax)
#Convert the result to dataloader
dataLoaderCurrent = DMP.TensorToDataLoader(
xAdv,
yClean,
transforms=None,
batchSize=dataLoader.batch_size,
randomizer=None)
return dataLoaderCurrent
def FGSMNativePytorch(device, dataLoader, model, epsilonMax, clipMin, clipMax,
targeted):
model.eval() #Change model to evaluation mode for the attack
#Generate variables for storing the adversarial examples
numSamples = len(
dataLoader.dataset) #Get the total number of samples to attack
xShape = DMP.GetOutputShape(
dataLoader
) #Get the shape of the input (there may be easier way to do this)
xAdv = torch.zeros(numSamples, xShape[0], xShape[1], xShape[2])
yClean = torch.zeros(numSamples)
advSampleIndex = 0
batchSize = 0 #just do dummy initalization, will be filled in later
#Go through each sample
tracker = 0
for xData, yData in dataLoader:
batchSize = xData.shape[
0] #Get the batch size so we know indexing for saving later
tracker = tracker + batchSize
#print("Processing up to sample=", tracker)
#Put the data from the batch onto the device
xDataTemp = torch.from_numpy(xData.cpu().detach().numpy()).to(device)
yData = yData.type(torch.LongTensor).to(device)
# Set requires_grad attribute of tensor. Important for attack. (Pytorch comment, not mine)
xDataTemp.requires_grad = True
# Forward pass the data through the model
output = model(xDataTemp)
# Calculate the loss
loss = torch.nn.CrossEntropyLoss(ignore_index=10,
size_average=True,
reduce=True)
# Zero all existing gradients
model.zero_grad()
# Calculate gradients of model in backward pass
cost = loss(output, yData).to(device)
cost.backward()
# Collect datagrad
#xDataGrad = xDataTemp.grad.data
###Here we actual compute the adversarial sample
# Collect the element-wise sign of the data gradient
signDataGrad = xDataTemp.grad.data.sign()
# Create the perturbed image by adjusting each pixel of the input image
#print("xData:", xData.is_cuda)
#print("SignGrad:", signDataGrad.is_cuda)
if targeted == True:
perturbedImage = xData - epsilonMax * signDataGrad.cpu().detach(
) #Go negative of gradient
else:
perturbedImage = xData + epsilonMax * signDataGrad.cpu().detach()
# Adding clipping to maintain the range
perturbedImage = torch.clamp(perturbedImage, clipMin, clipMax)
#Save the adversarial images from the batch
for j in range(0, batchSize):
xAdv[advSampleIndex] = perturbedImage[j]
yClean[advSampleIndex] = yData[j]
advSampleIndex = advSampleIndex + 1 #increment the sample index
#Not sure if we need this but do some memory clean up
del xDataTemp
del signDataGrad
torch.cuda.empty_cache()
#All samples processed, now time to save in a dataloader and return
advLoader = DMP.TensorToDataLoader(
xAdv,
yClean,
transforms=None,
batchSize=dataLoader.batch_size,
randomizer=None) #use the same batch size as the original loader
return advLoader