def measureAccuracy(model,dataset, device): accuracy = 0 numBatches = 0 for (batch, (inp, targ, lens)) in enumerate(dataset): predictions,_ = model(inp.permute(1, 0).to(device), lens, device) batch_accuracy = emotionModel.accuracy(targ.to(device), predictions) accuracy += batch_accuracy numBatches += 1 accuracy = (accuracy / numBatches*1.0) return accuracy
def roni(currentModel, sgdUpdate, previousValError, iterationNum, val_dataset):
updatedModel = currentModel + sgdUpdate
tempModel = emotionModel.EmoGRU(vocab_inp_size, embedding_dim, units,
BATCH_SIZE, target_size)
tempModel.to(device)
layer = 0
for name, param in tempModel.named_parameters():
if param.requires_grad:
param.data = torch.tensor(updatedModel[layer])
layer += 1
val_accuracy = 0
batchNum = 0
for (batch, (inp, targ, lens)) in enumerate(val_dataset):
predictions, _ = tempModel(inp.permute(1, 0).to(device), lens, device)
batch_accuracy = emotionModel.accuracy(targ.to(device), predictions)
val_accuracy += batch_accuracy
batchNum += 1
val_accuracy = (val_accuracy / batchNum)
# print('{:.4f},{:.4f}'.format(val_accuracy, previousValError))
if iterationNum < 0:
if (previousValError - val_accuracy) <= RONI_THRESH:
return True
else:
return False
else:
if (val_accuracy - previousValError) >= RONIThresh[iterationNum]:
return True
else:
return False
targ = Variable(targ)
loss = 0
print(inp.shape)
print(inp.permute(1, 0).shape)
sys.exit(0)
predictions = model(Variable(inp.permute(1, 0))) # TODO:don't need _
loss += emoModel.loss_function(targ, predictions)
batch_loss = (loss / int(targ.shape[1]))
total_loss += batch_loss
optimizer.zero_grad()
loss.backward()
optimizer.step()
batch_accuracy = emoModel.accuracy(targ, predictions)
train_accuracy += batch_accuracy
if batch % 100 == 0:
print(batch_loss.data.cpu().numpy())
print('Epoch {} Batch {} Train. Loss {:.4f}'.format(
epoch + 1, batch,
batch_loss.data.cpu().numpy()[0]))
# break
### Validating
test_accuracy = 0
for (batch, (inp, targ, lens)) in enumerate(test_dataset):
predictions = model(Variable(inp.permute(1, 0)), lens)
targ = Variable(targ)
batch_accuracy = emoModel.accuracy(targ, predictions)
def main():
globalVocab = constructGlobalVocab()
input_tensor_clients = []
target_tensor_clients = []
for x in range(0, numClients):
trainFileName = ''
if x < (numClients - maliciousClients):
trainFileName = 'train_' + str(x) + '.tsv'
else:
trainFileName = 'train_' + str(x) + '_corrupted.tsv'
input_tensor_train, target_tensor_train = preprocessData(trainFileName,
globalVocab,
iid=iid)
input_tensor_clients.append(input_tensor_train)
target_tensor_clients.append(target_tensor_train)
validationFileName = 'val.tsv'
input_tensor_val, target_tensor_val = preprocessData(validationFileName,
globalVocab,
iid=iid)
# validationFileName = 'val.tsv'
input_tensor_attackRate, target_tensor_attackRate = preprocessData(
validationFileName, globalVocab, True, iid=iid)
testFileName = 'test.tsv'
input_tensor_test, target_tensor_test = preprocessData(testFileName,
globalVocab,
iid=iid)
TRAIN_BUFFER_SIZE = len(input_tensor_train)
VAL_BUFFER_SIZE = len(input_tensor_val)
ATTACKRATE_BUFFER_SIZE = len(input_tensor_attackRate)
TEST_BUFFER_SIZE = len(input_tensor_test)
BATCH_SIZE = 60
TRAIN_N_BATCH = TRAIN_BUFFER_SIZE // BATCH_SIZE
VAL_N_BATCH = VAL_BUFFER_SIZE // BATCH_SIZE
ATTACKRATE_N_BATCH = ATTACKRATE_BUFFER_SIZE // BATCH_SIZE
TEST_N_BATCH = TEST_BUFFER_SIZE // BATCH_SIZE
embedding_dim = 256
units = 1024
vocab_inp_size = len(globalVocab.word2idx)
target_size = num_emotions
train_dataset_clients = []
for client in range(0, numClients):
train_dataset = dataLoadUtils.MyData(input_tensor_clients[client],
target_tensor_clients[client])
train_dataset = DataLoader(train_dataset,
batch_size=BATCH_SIZE,
drop_last=True,
shuffle=True)
train_dataset_clients.append(train_dataset)
val_dataset = dataLoadUtils.MyData(input_tensor_val, target_tensor_val)
val_dataset = DataLoader(val_dataset,
batch_size=BATCH_SIZE,
drop_last=True,
shuffle=True)
attackRate_dataset = dataLoadUtils.MyData(input_tensor_attackRate,
target_tensor_attackRate)
attackRate_dataset = DataLoader(attackRate_dataset,
batch_size=BATCH_SIZE,
drop_last=True,
shuffle=True)
test_dataset = dataLoadUtils.MyData(input_tensor_test, target_tensor_test)
test_dataset = DataLoader(test_dataset,
batch_size=BATCH_SIZE,
drop_last=True,
shuffle=True)
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
model = emotionModel.EmoGRU(vocab_inp_size, embedding_dim, units,
BATCH_SIZE, target_size)
model.to(device)
train_dataset = train_dataset_clients
# print(len(train_dataset))
# # obtain one sample from the data iterator
# it = iter(train_dataset)
# x, y, x_len = next(it)
# # sort the batch first to be able to use with pac_pack sequence
# xs, ys, lens = emotionModel.sort_batch(x, y, x_len)
# print("Input size: ", xs.size())
# output, _ = model(xs.to(device), lens, device)
# print(output.size())
### Enabling cuda
use_cuda = True if torch.cuda.is_available() else False
device = torch.device("cuda" if use_cuda else "cpu")
model = emotionModel.EmoGRU(vocab_inp_size, embedding_dim, units,
BATCH_SIZE, target_size)
model.to(device)
### loss criterion and optimizer for training
criterion = nn.CrossEntropyLoss() # the same as log_softmax + NLLLoss
# optimizer = torch.optim.SGD(model.parameters(), lr=1e-2, momentum=0.75, weight_decay=0.001)
optimizer = torch.optim.Adam(model.parameters())
total_loss = 0
train_accuracy, val_accuracy = 0, 0
# print(train_dataset)
numIterations = 200
datasetIter = [(iter(dataset)) for dataset in train_dataset]
for iteration in range(0, numIterations):
nextBatches = [next(iterator) for iterator in datasetIter]
# print(iteration)
aggregatedGradients = np.zeros(0)
total_loss = 0
prev_validation_accuracy = 0 + val_accuracy
train_accuracy, val_accuracy, attackRate_accuracy = 0, 0, 0
randomPeers = 0
peerList = []
for verifier in range(numVerifiers):
peerList.append(random.randint(0, numClients))
for nextBatch in nextBatches:
inp = nextBatch[0]
# print(inp)
targ = nextBatch[1]
# print(targ)
lens = nextBatch[2]
loss = 0
predictions, _ = model(inp.permute(1, 0).to(device), lens,
device) # TODO:don't need
loss += emotionModel.loss_function(targ.to(device), predictions)
batch_loss = (loss / int(targ.shape[1]))
total_loss += batch_loss
optimizer.zero_grad()
loss.backward()
modelLayers = getCurrentModel(model)
layers = []
for name, param in model.named_parameters():
if param.requires_grad:
layers.append(param.grad)
goodUpdate = roni(modelLayers, layers, prev_validation_accuracy,
iteration, val_dataset)
print(goodUpdate)
# sys.exit()
# goodUpdate = roni(modelLayers, layers, prev_validation_accuracy)
# print(goodUpdate)
# return layers
if goodUpdate and roni:
if len(aggregatedGradients) == 0:
aggregatedGradients = layers
else:
for layerIdx in range(0, len(aggregatedGradients)):
aggregatedGradients[layerIdx] = aggregatedGradients[
layerIdx] + layers[layerIdx]
# break
if (len(aggregatedGradients) == 0):
continue
layer = 0
for name, param in model.named_parameters():
if param.requires_grad:
param.grad = aggregatedGradients[layer]
layer += 1
optimizer.step()
for (batch, (inp, targ, lens)) in enumerate(val_dataset):
predictions, _ = model(inp.permute(1, 0).to(device), lens, device)
batch_accuracy = emotionModel.accuracy(targ.to(device),
predictions)
val_accuracy += batch_accuracy
if ((val_accuracy / VAL_N_BATCH) > 90):
print(iteration)
break
val_accuracy = (val_accuracy / VAL_N_BATCH)
# print(' Val Acc. {:.4f}'.format(val_accuracy / VAL_N_BATCH))
# break
for (batch, (inp, targ, lens)) in enumerate(attackRate_dataset):
predictions, _ = model(inp.permute(1, 0).to(device), lens, device)
batch_accuracy = emotionModel.accuracy(targ.to(device),
predictions)
attackRate_accuracy += batch_accuracy
# print(' Attack Rate. {:.4f}'.format(attackRate_accuracy / ATTACKRATE_N_BATCH))
attackRate_accuracy = attackRate_accuracy / ATTACKRATE_N_BATCH
print('{},{:.4f},{:.4f}'.format(iteration, val_accuracy,
attackRate_accuracy))
def roniPeerToPeer(currentModel, sgdUpdate, peerIndices):
tempModel = emotionModel.EmoGRU(vocab_inp_size, embedding_dim, units,
BATCH_SIZE, target_size)
tempModel.to(device)
# print(peerIndices)
layer = 0
for name, param in tempModel.named_parameters():
if param.requires_grad:
param.data = currentModel[layer]
layer += 1
prev_accuracies = []
for peer in peerIndices:
peerAccuracy = 0
batchCount = 0
for (batch, (inp, targ,
lens)) in enumerate(train_dataset_clients[peer]):
predictions, _ = tempModel(
inp.permute(1, 0).to(device), lens, device)
batch_accuracy = emotionModel.accuracy(targ.to(device),
predictions)
# print(batch_accuracy)
peerAccuracy += batch_accuracy
batchCount += 1
break
peerAccuracy = (peerAccuracy / batchCount)
# print(peerAccuracy)
# print(batchCount)
prev_accuracies.append(peerAccuracy)
print(prev_accuracies)
# sys.exit()
updatedModel = currentModel + sgdUpdate
layer = 0
for name, param in tempModel.named_parameters():
if param.requires_grad:
param.data = updatedModel[layer]
layer += 1
new_accuracies = []
for peer in peerIndices:
peerAccuracy = 0
batchCount = 0
for (batch, (inp, targ,
lens)) in enumerate(train_dataset_clients[peer]):
predictions, _ = tempModel(
inp.permute(1, 0).to(device), lens, device)
batch_accuracy = emotionModel.accuracy(targ.to(device),
predictions)
peerAccuracy += batch_accuracy
batchCount += 1
peerAccuracy = (peerAccuracy / batchCount)
new_accuracies.append(peerAccuracy)
print(new_accuracies)
# sys.exit
diffs = [prev - new for prev, new in zip(prev_accuracies, new_accuracies)]
passes = sum(diff < RONI_THRESH for diff in diffs)
if passes >= 2:
return True
else:
return False