def train_ref(model,
criterion,
optimizer,
input,
target,
nb_epochs=200,
verbose=False):
mini_batch_size = 100
#empty recipient
loss_evolution = []
precision_evolution = []
#actual training
for e in range(nb_epochs):
loss_e = 0.
for b in range(0, input.size(0), mini_batch_size):
output = model(input.narrow(0, b, mini_batch_size))
loss = criterion(output, target.narrow(0, b, mini_batch_size))
loss_e += loss
model.zero_grad()
loss.backward()
optimizer.step()
#record the data
precision_evolution.append(
helpers.compute_accuracy(model, input, target) / target.shape[0] *
100)
loss_evolution.append(loss_e)
if verbose:
message = "epoch {:3}, loss {:10.4}".format(e, loss_e)
helpers.update_progress((e + 1.) / nb_epochs, message=message)
return loss_evolution, precision_evolution
def train_rep(model,
lr,
X,
P,
n_iter,
c_iter,
batch_size,
alpha=10,
C_reg=1,
compute_emd=False,
adv=True,
verbose=False):
"""
Train the fair representation using autoencoder provided by the user.
Parameters:
model: the Pytorch model of the autoencoder. The model should have two members, model.encoder, and the model.decoder.
Parameters:
lr: learning rate.
X: the input features.
P: the protected attribute.
n_iter: number of iterations.
c_iter: the number of iteration to trian the critic inside each training iteration.
batch_size: batch size.
alpha: the weight of the fairness contraint. Larger means more penalize on the violation of fairness.
C_reg: the penalization coefficient of the regularization of the encoder.
compute_emd: whether the EMD distance is calculated for each iteration. It may slow the training process significantly.
adv: if the model is trained adversarially, i.e. fairly. Setting it false will result in training a normal autoencoder.
verbose: if the training process is verbosely printed.
"""
time_s = time.time()
X_0 = X[P == 0]
X_1 = X[P == 1]
optim_encoder = optim.Adam(model.encoder.parameters(), lr=lr)
optim_decoder = optim.Adam(model.decoder.parameters(), lr=lr)
optim_crit = optim.Adam(model.critic.parameters(), lr=0.1)
l1_crit = nn.L1Loss(size_average=False)
n_of_batch = int(len(X) / (batch_size * 2)) * n_iter
for i in range(n_of_batch):
X_n = X_0[np.random.choice(len(X_0), batch_size)]
X_u = X_1[np.random.choice(len(X_1), batch_size)]
if adv:
w_dist_last = 0
eps = 1
#while w_dist <= 0:
while eps >= 1e-3:
#while True:
for t in range(c_iter):
optim_crit.zero_grad()
w_dist = model.wdist(X_n, X_u)
loss = -w_dist
loss.backward(retain_graph=True)
optim_crit.step()
eps = np.abs(w_dist.data.item() - w_dist_last)
# keep training crit until distance no longer decrease
w_dist_last = w_dist.data.item()
for p in model.critic.parameters():
p.data.clamp_(-0.1, 0.1)
# for t in range(c_iter):
optim_encoder.zero_grad()
optim_decoder.zero_grad()
# only use the encoder g
mse, wdist = model.forward(X_n, X_u)
if adv:
loss = mse + wdist * alpha
else:
loss = mse
# L1 regularization
reg_loss = 0
#for param in model.encoder.parameters():
# reg_loss += torch.abs(param).sum()
for layer in model.encoder:
if type(layer) is nn.Linear:
#norm = torch.sum(torch.pow(torch.sum(torch.abs(layer.weight), dim=0), 2))
norm = 0.0
for row in layer.weight.transpose(0, 1):
norm += torch.sum(torch.pow(row, 2))
reg_loss += norm
loss += C_reg * reg_loss
loss.backward(retain_graph=True)
# use mse and wdist to update g and f
optim_encoder.step()
optim_decoder.step()
text = 'mse: %.4f, critic: %.4f' % (mse.item(), wdist.item())
if compute_emd:
g_0 = model.encoder(X_u).detach().cpu().numpy()
g_1 = model.encoder(X_n).detach().cpu().numpy()
real_emd = emd_samples(g_0, g_1)
text += ", emd: %.4f" % real_emd
if verbose:
update_progress(i, n_of_batch, time_s, text=text + ' ')
def input():
# session['userid'] = 1
if request.method == "GET":
if logged() is False:
return render_template('welcome.html')
userid = session.get('userid')
# load info from db and save it in session
load_savings(userid, savings_db)
load_expenses(userid, expenses_db)
load_goals(userid, goals_db)
if load_user_info(userid, user_db) is False:
return redirect('/error')
return render_template('input.html',
savings=session.get('savings'),
expenses=session.get('expenses'),
goals=session.get('goals'),
user_info=session.get('user_info'))
if request.method == "POST":
# DEF: saving all info from input page to session
# save salary
session['salary'] = float(request.form.get('salary'))
# preload accounts from session
savings = session.get('savings')
expenses = session.get('expenses')
goals = session.get('goals')
#for all saving types load current value from input page
for key in savings:
savings[key]['value'] = float(request.form.get(key))
#change progress info
savings = update_progress(savings, key)
# for all expenses load value from input page
for key in expenses:
expenses[key]['value'] = float(request.form.get(key))
# for all goal load current value
for key in goals:
goals[key]['value'] = float(request.form.get(key))
# change progress info
goals = update_progress(goals, key)
#save updated info in session
session['expenses'] = expenses
session['savings'] = savings
session['goals'] = goals
print(
f'\n\n-----USER INPUT processed and saved in session\n\nexpenses: {expenses} \n\nsavings:{savings}\n\ngoals: {goals} \n\nsalary:{session.get("salary")}'
)
return redirect('/output')
def run(self):
"""
Codice eseguito nel thread.
Riceve dal peer l'md5 del file che desidera scaricare e lo invia diviso in parti
"""
try:
cmd = self.conn.recv(4) # Ricezione del comando di download dal peer, deve contenere RETR
except socket.error as e:
print 'Socket Error: ' + e.message
except Exception as e:
print 'Error: ' + e.message
else:
if cmd == "RETR":
try:
self.md5 = self.conn.recv(32) # Ricezione dell'md5 del file da inviare
print 'Received md5: ' + self.md5
except socket.error as e:
print 'Socket Error: ' + e.message
except Exception as e:
print 'Error: ' + e.message
else:
found_name = None
for idx, file in enumerate(self.file_list): # Ricerca del file da inviare tra quelli disponibili
if file.md5 == self.md5:
found_name = file.name
if found_name is None:
print 'Found no file with md5: ' + self.md5
else:
chunk_size = 1024 # Dimensione di una parte di file
try:
file = open("shareable/" + found_name, "rb")
except Exception as e:
print 'Error: ' + e.message + "\n"
else:
tot_dim = self.filesize("shareable/" + found_name) # Calcolo delle dimesioni del file
n_chunks = int(tot_dim // chunk_size) # Calcolo del numero di parti
resto = tot_dim % chunk_size # Eventuale resto
if resto != 0.0:
n_chunks += 1
file.seek(0, 0) # Spostamento all'inizio del file
try:
buff = file.read(chunk_size) # Lettura del primo chunk
chunks_sent = 0
msg = 'ARET' + str(n_chunks).zfill(6) # Risposta alla richiesta di download, deve contenere ARET ed il numero di chunks che saranno inviati
print 'Upload Message: ' + msg
self.conn.sendall(msg)
print 'Sending chunks...'
while len(buff) == chunk_size: # Invio dei chunks
try:
msg = str(len(buff)).zfill(5) + buff
self.conn.sendall(msg) # Invio di
chunks_sent += 1
helpers.update_progress(chunks_sent, n_chunks, 'Uploading ' + file.name) # Stampa a video del progresso dell'upload
buff = file.read(chunk_size) # Lettura chunk successivo
except IOError:
print "Connection error due to the death of the peer!!!\n"
if len(buff) != 0: # Invio dell'eventuale resto, se più piccolo di chunk_size
msg = str(len(buff)).zfill(5) + buff
self.conn.sendall(msg)
print "\nUpload Completed"
file.close() # Chiusura del file
except EOFError:
print "You have read a EOF char"
else:
print "Error: unknown directory response.\n"
self.conn.shutdown(1) # Segnalazione di fine comunicazione
self.conn.close() # Chiusura comunicazione
def get_file(session_id, host_ipv4, host_ipv6, host_port, file, directory):
"""
Effettua il download di un file da un altro peer
:param session_id: id sessione corrente assegnato dalla directory
:type session_id: str
:param host_ipv4: indirizzo ipv4 del peer da cui scaricare il file
:type host_ipv4: str
:param host_ipv6: indirizzo ipv6 del peer da cui scaricare il file
:type host_ipv6: str
:param host_port: porta del peer da cui scaricare il file
:type host_port: str
:param file: file da scaricare
:type file: file
:param directory: socket verso la directory (per la segnalazione del download)
:type directory: object
"""
c = Connection.Connection(host_ipv4, host_ipv6, host_port) # Inizializzazione della connessione verso il peer
c.connect()
download = c.socket
msg = 'RETR' + file.md5
print 'Download Message: ' + msg
try:
download.send(msg) # Richiesta di download al peer
print 'Message sent, waiting for response...'
response_message = download.recv(10) # Risposta del peer, deve contenere il codice ARET seguito dalle parti del file
except socket.error as e:
print 'Error: ' + e.message
except Exception as e:
print 'Error: ' + e.message
else:
if response_message[:4] == 'ARET':
n_chunks = response_message[4:10] # Numero di parti del file da scaricare
#tmp = 0
filename = file.name
fout = open('received/' + filename, "wb") # Apertura di un nuovo file in write byte mode (sovrascrive se già esistente)
n_chunks = int(str(n_chunks).lstrip('0')) # Rimozione gli 0 dal numero di parti e converte in intero
for i in range(0, n_chunks):
if i == 0:
print 'Download started...'
helpers.update_progress(i, n_chunks, 'Downloading ' + fout.name) # Stampa a video del progresso del download
try:
chunk_length = recvall(download, 5) # Ricezione dal peer la lunghezza della parte di file
data = recvall(download, int(chunk_length)) # Ricezione dal peer la parte del file
fout.write(data) # Scrittura della parte su file
except socket.error as e:
print 'Socket Error: ' + e.message
break
except IOError as e:
print 'IOError: ' + e.message
break
except Exception as e:
print 'Error: ' + e.message
break
fout.close() # Chiusura file a scrittura ultimata
print '\nDownload completed'
warns_directory(session_id, file.md5, directory) # Invocazione del metododo che segnala il download alla directory
print 'Checking file integrity...'
downloaded_md5 = helpers.hashfile(open(fout.name, 'rb'), hashlib.md5()) # Controllo dell'integrità del file appena scarcato tramite md5
if file.md5 == downloaded_md5:
print 'The downloaded file is intact'
else:
print 'Something is wrong. Check the downloaded file'
else:
print 'Error: unknown response from directory.\n'
def Train(self, train_input, train_target, train_classes = None, auxiliary = False, verbose = True, nb_epochs = 50, batch_size=250, device='cpu', evolution = False, test_input = None, test_target = None):
""" Training of the siamese module.
if not auxiliary:
usual training
if auxiliary:
first split the sample into the two images and use the branch to try to classify the numbers, compute the
loss wrt to the classes of digits and update the parameters of the branch.
Then run the entire sample trough the sample and compute the loss against the train_target and update
Parameters
----------
train_input : [n,2,14,14]
two images [14,14] representing two numbers
train_target : [n,2]
is the first digit bigger than the second
train_classes : [n,2]
values of the two digits
auxiliary : bool
use of the auxiliary loss (the default is False).
verbose : bool
if True print the training `verbose` (the default is True).
nb_epochs : int
epochs to train (the default is 50).
device : torch.device
Torch device (the default is 'cpu').
"""
if auxiliary:
if verbose:
print("training with auxiliary loss with {} epochs".format(nb_epochs))
if train_classes is None: print("Error: if auxiliary loss, the model needs the classes of the training set")
elif verbose:
print("training with no auxiliary losses with {} epochs".format(nb_epochs))
criterion = nn.CrossEntropyLoss()
criterion = criterion.to(device)
optimizer = optim.Adam(self.parameters(),lr = 0.005)
if evolution:
acc_train = []
acc_test = []
for e in range(nb_epochs):
for input, targets in zip(train_input.split(batch_size), train_target.split(batch_size)):
if auxiliary:
#first pass
#separating the data so that we train on the two images separatly, and learn to classify them properly
train_input1,train_classes1,train_input2,train_classes2 = split_channels(train_input, train_classes)
#use the branch to perform handwritten digits classification
out1 = self.branch(train_input1)
out2 = self.branch(train_input2)
#auxiliary loss: learn to detect the handwritten digits directly
loss_aux = criterion(out1,train_classes1) + criterion(out2,train_classes2)
#optimize based on this
self.zero_grad()
loss_aux.backward(retain_graph=True)
optimizer.step()
#second pass
#loss and optimization of the whole model
response = self.forward(train_input)
loss = criterion(response,train_target)
self.zero_grad()
loss.backward()
optimizer.step()
else:
response = self.forward(train_input)
loss = criterion(response,train_target)
self.zero_grad()
loss.backward()
optimizer.step()
if verbose:
acc = accuracy(self, train_input, train_target)
print("epoch {:3}, loss {:7.4}, accuracy {:.2%}".format(e,loss,acc))
else:
update_progress((e+1)/nb_epochs, message="")
if evolution:
acc_train.append(accuracy(self, train_input, train_target))
acc_test.append( accuracy(self, test_input, test_target))
if evolution:
return acc_train,acc_test