def _handshake_to_new_node(client, sid):
## add client info into children document
_add_children(client, sid)
s_obj = WebServerModel.objects(sid=sid)
logger.info("client id:"+client['nid']+" has joined the system")
message = json.dumps({
'action':api._ACTION_KEYS[0],
'data':
{
'server_id':s_obj.scalar('sid').get(),
'server_public_key':s_obj.scalar('public_key').get(),
'common_key_private':s_obj.scalar('common_key_private').get()
}
})
encrypted_message = api.encrypt_msg(client['public_key'], message)
childs = s_obj.get().children
message = {}
for child in childs:
if not message:
message = {
'action':api._ACTION_KEYS[1],
'data':[
{
'nid':child.nid,
'public_key':child.public_key
}]
}
else:
message['data'].append(
{
'nid':child.nid,
'public_key':child.public_key
})
## TODO: set try and except
retry = 0
while retry < api.MAX_TRY:
try:
proxy = callme.Proxy(
server_id=client['nid'],
amqp_host="localhost",
timeout=3)
proxy.use_server(client['nid']).execute_command(encrypted_message)
retry = api.MAX_TRY
except:
retry+=1
print "passing master server info failed... {} times".format(retry)
else:
## broadcast of a new client to all nodes
_broadcast_to_nodes(api.EXCHANGE_FOR_ALL,
api.encrypt_msg(s_obj.scalar('common_key_public').get(),
json.dumps(message)))
def execute_command(self, encrypted_action):
print "server calling rpc"
data = json.loads(
api.decrypt_msg(self.model.private_key, encrypted_action))
action = data['action']
print action
## response from master server has arrived, stop trying to reach other servers
if self.timer.is_alive():
self.timer.stop()
## server sent info
if (action == api._ACTION_KEYS[0]):
self._store_server_info(data['data'], self.model.nid)
## add comments
elif(action == api._ACTION_KEYS[5]):
data['nid'] = self.model.nid
"""data['content'].update(
{
'nid':self.model.nid,
'sid':data['sid']
})"""
return api.encrypt_msg(AppModel.objects(
nid=self.model.nid).get().server_public_key,
app_api.add_comments(**data))
def _refresh_common_key(sid):
s_obj = WebServerModel.objects(sid=sid)
if s_obj.get().children:
logger.info("common key timeout")
new_prk = api.generate_private_key()
message = {
"action":api._ACTION_KEYS[3],
"data":
{
"common_key_private":new_prk.exportKey().decode('utf-8')
}
}
try:
if _broadcast_to_nodes(api.EXCHANGE_FOR_ALL,
api.encrypt_msg(s_obj.scalar('common_key_public').get(),
message)) == False:
logger.error("Message broadcast failed")
#else:
res = s_obj.update_one(
common_key_public=api.generate_public_key(new_prk).exportKey().decode('utf-8'),
common_key_private=message['data']['common_key_private'])
s_obj.get().save()
## make log of updated common key
_make_log(api._ACTION_KEYS[3], sid, "common key updated")
except:
logger.error("try again")
threading.Timer(api.COMMON_KEY_TIMEOUT, _refresh_common_key, [sid]).start()
def _clean_failed_garbages(sid):
garbages = q.get_custom_list(api.FAILED_NODES)
message = json.dumps(
{
'action':api._ACTION_KEYS[2],
'data':garbages
})
_broadcast_to_nodes(
api.EXCHANGE_FOR_ALL,
api.encrypt_msg(
WebServerModel.objects(
sid=sid).scalar(
'common_key_public').get(),
message))
if WebServerModel.objects(sid=sid).update(pull_all__children=
map(lambda x:{
'nid':x
}, garbages))>1:
logger.info("cleaned failed nodes")
else:
logger.error("cleanning failed or nothing to clean")
threading.Timer(api.CLEAN_GARBAGE_NODES,
_clean_failed_garbages, [sid]).start()
def _create_usersession(uname, sid): sk = api._generate_session_key() mUserSession = UserSession( session_id=sk, username = uname ) pk = WebServerModel.objects(sid=sid).get().public_key print pk return api.encrypt_msg(pk, sk)
def _send_add_comment_call(**data):
if data['nid'] == None:
return False
retry = 0
prm = None
while retry < api.MAX_TRY:
try:
# do not retry pushing node id inth the queue over as it failed first time
if retry == 0:
prm = _coordinate_acc_to_res(
sid=data['sid'],
nid=data['nid'])
proxy = callme.Proxy(
server_id=data['nid'],
amqp_host="localhost",
timeout=3)
for c in WebServerModel.objects(sid=data['sid']).get().children:
if c.nid == data['nid']:
response = proxy.use_server(c.nid).execute_command(
api.encrypt_msg(c.public_key, {
'action':api._ACTION_KEYS[5],
## access to shared resource has been permitted
'permission':prm,
'sid':data['sid'],
'content':
{
'by':data['by'],
'session_id':data['session_id'],
'comment':data['comment'],
'created_at':data['created_at']
}
}))
return response
## given node is not matching with any of registered node
retry+=1
print "trying {}".format(retry)
except:
retry+=1
print "node {} is not responding.. trying {} times...".format(data['nid'], retry)
# finally
return False
def _inspect_inactive_children(sid):
#print "inspection...{}".format(sid)
s_obj = WebServerModel.objects(sid=sid)
children = s_obj.get().children
if children:
for child in children:
#logger.info(child.last_access)
#logger.info(api._get_current_time())
if api._get_unix_from_datetime(
api._get_current_time())-api._get_unix_from_datetime(
child.last_access)>api.TIME_DETERMINE_INACTIVE:
logger.info(child.nid+" -> user logged out")
## pull inactive user
s_obj.update_one(pull__children__nid=child.nid)
s_obj.get().save()
## notify all clients about inactive user
message = json.dumps(
{
'action':api._ACTION_KEYS[2],
'data':{
"nid":child.nid,
}
})
## remove lock on shared resource if the inactive child is holding it
if q.get(api.RES_HOLDER) == child.nid:
logger.info("logging out user had locks")
_coordinate_acc_to_res(sid, child.nid,
api._ACTION_KEYS[6])
_broadcast_to_nodes(api.EXCHANGE_FOR_ALL,
api.encrypt_msg(s_obj.scalar('common_key_public').get(),
message))
_make_log(api._ACTION_KEYS[2], sid, child.nid)
## TODO: try deamon thread
threading.Timer(api.INSPECTION_TIME_INTERVAL, _inspect_inactive_children, [sid]).start()
def _coordinate_acc_to_res(sid=None, nid=None, action=None):
## done working message from a node who has access to the shared resource
if action == api._ACTION_KEYS[6]:
q.pop()
q.delete(api.RES_HOLDER)
_make_log("resource_released", sid, nid)
## next node in the queue
nxt = q.bottom()
if nxt:
logger.info("next node exists")
logger.info(nxt)
# if next resource accessor is master node himself
if nxt == sid:
_upload_user_sessions(sid)
_coordinate_acc_to_res(sid, sid,
api._ACTION_KEYS[6])
## save as node which holding lock to the resource at present
q.append(api.RES_HOLDER, nxt)
for child in WebServerModel.objects(sid=sid).get().children:
if child.nid == nxt:
_send_direct_msg(nxt, api.encrypt_msg(child.public_key, {
'action':api._ACTION_KEYS[8], # action : access permission
'by':sid
}))
break
_make_log("resource_lock", sid, nxt)
else:
logger.info("queue is empty")
return True
elif action == None and nid == None:
if q.length() == 0:
q.push(sid)
#q.push(nid) # for debugging
q.append(api.RES_HOLDER, sid)
_make_log("resource_lock", sid, sid)
return True
else:
q.push(sid)
return False
elif action == None:
for child in WebServerModel.objects(sid=sid).get().children:
## node exists
if child.nid == nid:
if q.length() == 0:
q.push(nid)
#q.push(nid) # for debugging
q.append(api.RES_HOLDER, nid)
_make_log("resource_lock", sid, nid)
return True
else:
q.push(nid)
return False
## not registered node
logger.error("not registered node")
return False