def push():
print "Pushing"
s = getSocket();
email,password = getCredentials()
protocol.send_one_message(s, json.dumps({'email':email,'password':password, 'req_type':'push', 'dir_parse_json': custom_tools.parse_dir(settings.main_dir)}))
if (protocol.recv_one_message(s)=='1'):
print "Pushing Files..."
print "Files in directory are:"
for i in custom_tools.parse_dir(settings.main_dir+'/'):
print i['filename'],i['foldername'],i['timestamp']
num_files = int(protocol.recv_one_message(s))
print 'Number of files to send is ', num_files
details = json.loads(protocol.recv_one_message(s))
filename_list = details["filename_list"]
foldername_list = details["foldername_list"]
print filename_list
s.close()
for x in range(0,len(filename_list)):
if(foldername_list[x] == ''):
filename=settings.main_dir+'/'+filename_list[x]
else:
filename=settings.main_dir+'/'+foldername_list[x]+'/'+filename_list[x]
print('Sending file with filename '+filename)
if ( os.path.isdir(filename) ):
add_folder(filename)
else:
add_file(filename)
else:
print "You are not Authenticated"
def add_file(filename):
if (not os.path.isfile(filename)):
return
s = getSocket()
a=filename.split('/');
root=settings.main_dir.split('/');
folder = '.'
if ( len(a) > len(root) + 1 ):
folder = a[len(a)-2]
print "Folder: ", folder
email,password = getCredentials()
url = urllib.pathname2url(filename)
mimetype=mimetypes.guess_type(url)
mimetype=str(mimetype[0])
statinfo = os.stat(filename)
timestamp = datetime.datetime.fromtimestamp(statinfo.st_mtime).strftime('%Y-%m-%d %H:%M:%S')
print "Transferring file ", filename
protocol.send_one_message(s, json.dumps({'filename' : a[-1],'foldername' : folder,'email':email,'password':password, 'req_type':'add_file','file_size':statinfo.st_size,'file_timestamp':timestamp,'content_type':mimetype,'dir_parse_json':custom_tools.parse_dir(settings.main_dir+'/')}))
if (protocol.recv_one_message(s)=='1'):
protocol.send_one_file(s,filename)
reply = protocol.recv_one_message(s)
print "Server Replied", reply
s.close()
else:
print "You are not Authenticated"
def remove_file(filename):
s = getSocket()
a=filename.split('/');
email,password = getCredentials()
timestamp = time.strftime('%Y-%m-%d %H:%M:%S')
print "Deleting File ", filename
root=settings.main_dir.split('/');
folder = '.'
if ( len(a) > len(root) + 1 ):
folder = a[len(a)-2]
print "Folder: ", folder
protocol.send_one_message(s, json.dumps({'filename' : a[-1], 'foldername' : folder, 'email':email,'password':password, 'req_type':'delete_file','file_timestamp':timestamp}))
if (protocol.recv_one_message(s)=='1'):
reply = protocol.recv_one_message(s)
print "Server Replied: ", reply
s.close()
else:
print "You are not Authenticated"
def pull():
s = getSocket();
email,password = getCredentials()
protocol.send_one_message(s, json.dumps({'email':email,'password':password, 'req_type':'pull', 'dir_parse_json': custom_tools.parse_dir(settings.main_dir)}))
if (protocol.recv_one_message(s)=='1'):
print "Pulling Files..."
print "Files in directory are:"
for i in custom_tools.parse_dir(settings.main_dir+'/'):
print i['filename'],i['foldername'],i['timestamp']
num_files = int(protocol.recv_one_message(s))
print 'Number of files to receive is ', num_files
details = json.loads(protocol.recv_one_message(s))
filename_list = details["filename_list"]
foldername_list = details["foldername_list"]
print filename_list
print foldername_list
for x in range(0,len(filename_list)):
if(foldername_list[x] == ''):
filename=settings.main_dir+'/'+filename_list[x]
else:
directory = settings.main_dir+'/'+foldername_list[x]+'/'
if not os.path.exists(directory):
os.makedirs(directory)
filename=settings.main_dir+'/'+foldername_list[x]+'/'+filename_list[x]
if(protocol.recv_one_message(s)=='1'):
print('Receiving file with filename '+filename)
protocol.recv_one_file(s,filename)
print "received"
else:
print('Will not receive file with filename '+filename)
s.close()
else:
print "You are not Authenticated"
def demander(q, j):
informer("QUESTION : " + q)
protocol.send_one_message(clients[j.numero], messages.Question(q).toJson())
r = protocol.recv_one_message(clients[j.numero])
r = messages.deserialize(r)
informer("REPONSE DONNEE : " + str(r.content))
return str(r.content)
def remove_folder(folder):
s=getSocket()
a=folder.split('/');
email,password = getCredentials()
print "Removing folder", a[-1]
protocol.send_one_message(s, json.dumps({'foldername' : a[-1],'email':email,'password':password, 'req_type':'delete_folder'}))
if (protocol.recv_one_message(s)=='1'):
reply = protocol.recv_one_message(s)
print "Server Replied", reply
s.close()
else:
print "You are not Authenticated"
def worker_thread(worker_HOST):
global model
global workers_lipz_coeffs
global ps_grads
worker_grads = Epoch_Params_Grad_Holder()
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect(worker_HOST)
with sess.as_default():
while True:
if converged:
break
batch = train_set.get()
with model_lock:
epoch = model.epoch()
params = model.read()
prot.send_one_message(s, batch[0].tobytes())
prot.send_one_message(s, batch[1].tobytes())
prot.send_one_message(s, params.tobytes())
data_grad = prot.recv_one_message(s)
grad = np.frombuffer(data_grad, np.float32)
worker_grads.add((epoch, params, grad))
with model_lock:
candidate_lipz_coeffs = copy.deepcopy(workers_lipz_coeffs)
candidate_lipz_coeffs[
worker_HOST[1]] = worker_grads.lipz_coeff()
ps_grads.add((model.epoch(), model.read(), grad))
lipz_coeff = ps_grads.lipz_coeff()
delay = time.time() - t_start
if delay < 3:
model.update(grad)
workers_lipz_coeffs = candidate_lipz_coeffs
print(workers_lipz_coeffs)
print('accepted1', worker_HOST[1], ps_grads.prev_epoch(),
ps_grads.epoch())
else:
if lipz_filters_in(lipz_coeff, candidate_lipz_coeffs):
model.update(grad)
workers_lipz_coeffs = candidate_lipz_coeffs
print('accepted:', worker_HOST[1],
ps_grads.prev_epoch(), ps_grads.epoch())
else:
worker_grads.revert()
ps_grads.revert()
print('rejected:', worker_HOST[1],
ps_grads.prev_epoch(), ps_grads.epoch())
s.shutdown(socket.SHUT_RDWR)
s.close()
def worker_thread(worker_HOST):
global model
global converged
global accs
global epochs
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect(worker_HOST)
with sess.as_default():
while True:
if converged:
break
batch = train_set.get()
with model_lock:
epoch = model.epoch()
params = model.read()
prot.send_one_message(s, batch[0].tobytes())
prot.send_one_message(s, batch[1].tobytes())
prot.send_one_message(s, params.tobytes())
data_grad = prot.recv_one_message(s)
grad = np.frombuffer(data_grad, np.float32)
with model_lock:
if converged:
break
model.update(grad)
new_epoch = model.epoch()
acc = model.eval(*test_set.get())[0]
staleness = new_epoch - epoch
epochs.append(new_epoch)
accs.append(acc)
stales.append(staleness)
print(worker_HOST[1], epoch, '=>', new_epoch, acc, staleness)
if acc >= max_train_accur or new_epoch > max_train_epoch:
converged = True
s.shutdown(socket.SHUT_RDWR)
s.close()
while True:
# receive observations
data_x = prot.recv_one_message(conn)
if not data_x:
break
x = np.frombuffer(data_x, np.float32)
x = x.reshape((50, 784))
# receive labels
data_y = prot.recv_one_message(conn)
y = np.frombuffer(data_y, np.int32)
print('received batch')
# receive parameters
data_params = prot.recv_one_message(conn)
params = np.frombuffer(data_params, np.float32)
print('received parameters')
# calculate gradient
model.write(params)
grad = model.backprop(x, y)
# send gradient
prot.send_one_message(conn, grad.tobytes())
print('sent gradient\n')
# close connection
conn.shutdown(socket.SHUT_RDWR)
conn.close()
print('connection closed')
print(str(acc))
if acc >= max_train_accur or epoch >= max_train_epoch:
break
# Train
it = iter(train_set)
for i in range(0, l, n):
grads = []
# send batch, parameters to all workers
for j in range(i, i + n):
p = int(j / l * 100.)
sys.stdout.write("\rTraining... " + str(p) + "%")
sys.stdout.flush()
s = sockets[j % n]
batch = next(it)
prot.send_one_message(s, batch[0].tobytes())
prot.send_one_message(s, batch[1].tobytes())
prot.send_one_message(s, model.read().tobytes())
# receive gradients
for s in sockets:
data_grad = prot.recv_one_message(s)
grad = np.frombuffer(data_grad, np.float32)
grads.append(grad)
# update model
model.update(aggregate(grads))
print("\rTraining... 100%")
t_end = time.time()
print('time: ', t_end - t_start)
def answer(self):
r = messages.Response(str(randrange(2)))
protocol.send_one_message(self.socket, r.toJson())
def message(texte, jos):
for j in jos:
protocol.send_one_message(clients[j.numero],
messages.Information(texte).toJson())