def _get_next_worker(sid):
s_obj = WebServerModel.objects(sid=sid)
lowest_time = api._get_current_time()
for child in s_obj.get().children:
## list of nodes failed once
if child.nid in q.get_custom_list(api.FAILED_NODES):
print "failed nodes"
print q.get_custom_list(api.FAILED_NODES)
pass
elif not 'last_work' in child:
s_obj.filter(
children__nid=child.nid).update_one(
set__children__S__last_work=lowest_time) # set as now if new server
s_obj.get().save()
next_worker_id = child.nid
elif child.last_work < lowest_time:
lowest_time = child.last_work
next_worker_id = child.nid
try:
s_obj.filter(
children__nid=next_worker_id).update_one(
set__children__S__last_work=api._get_current_time())
s_obj.get().save()
print "next worker is... {}".format(next_worker_id)
return next_worker_id
except NameError:
logger.error("no workers are available")
return None
def _clean_session_garbages(): garbage = [] for s in Sessions.objects[:]: if s.expiration < api._get_current_time(): garbage.append(s.ssid) if garbage: for s in garbage: Sessions.objects(ssid=s).delete()
def get_user_list(sid):
if _coordinate_acc_to_res(sid=sid):
if _upload_user_sessions():
## TODO: try with mongodb query but this requires to
## handle obtain 'modified' and convert into unix timestamp to compare
## within the query
users = map(lambda x:{
'user_name':json.loads(x.user_data)['user_name'],
'modified':x['modified'],
'ssid':x['ssid']
} if api._get_unix_from_datetime(
api._get_current_time())-api._get_unix_from_datetime(
x.modified)<api.TIME_DETERMINE_USER_ACTIVE else None,
UserSessions.objects.all())
# release resource lock
_coordinate_acc_to_res(sid, sid,
api._ACTION_KEYS[6])
# return list of lately active users by removing None values
return filter(None, users)
else:
# release resource lock
_coordinate_acc_to_res(sid, sid,
api._ACTION_KEYS[6])
else:
users = map(lambda x:{
'user_name':json.loads(x.user_data)['user_name'],
'modified':x['modified'],
'ssid':x['ssid']
} if api._get_unix_from_datetime(
api._get_current_time())-api._get_unix_from_datetime(
x.modified)<api.TIME_DETERMINE_USER_ACTIVE else None,
Sessions.objects.all())
return filter(None, users)
return None
def _update_last_access_time(nid, sid):
collection = WebServerModel._get_collection()
result = collection.update_one(
{
"_id":sid,
"children._id":nid
},
{
"$set":
{
"children.$.last_access":api._get_current_time()
}
})
return True if result.modified_count == 1 and result.matched_count == 1 else False
def open_session(self, app, request):
ssid = request.cookies.get(app.session_cookie_name)
if ssid:
stored_session = self.sessions.find_one({'ssid': ssid})
#print "matched_session {}".format(stored_session)
if stored_session:
if stored_session.get('expiration') > api._get_current_time():
print "session id not expired: {}".format(ssid)
return MongoSession(
initial=stored_session['user_data'],
ssid=stored_session['ssid'])
ssid = api._generate_session_id()
print "new ssid: {}".format(ssid)
return MongoSession(ssid=ssid)
def _inspect_inactive_children(sid):
#print "inspection...{}".format(sid)
s_obj = WebServerModel.objects(sid=sid)
children = s_obj.get().children
if children:
for child in children:
#logger.info(child.last_access)
#logger.info(api._get_current_time())
if api._get_unix_from_datetime(
api._get_current_time())-api._get_unix_from_datetime(
child.last_access)>api.TIME_DETERMINE_INACTIVE:
logger.info(child.nid+" -> user logged out")
## pull inactive user
s_obj.update_one(pull__children__nid=child.nid)
s_obj.get().save()
## notify all clients about inactive user
message = json.dumps(
{
'action':api._ACTION_KEYS[2],
'data':{
"nid":child.nid,
}
})
## remove lock on shared resource if the inactive child is holding it
if q.get(api.RES_HOLDER) == child.nid:
logger.info("logging out user had locks")
_coordinate_acc_to_res(sid, child.nid,
api._ACTION_KEYS[6])
_broadcast_to_nodes(api.EXCHANGE_FOR_ALL,
api.encrypt_msg(s_obj.scalar('common_key_public').get(),
message))
_make_log(api._ACTION_KEYS[2], sid, child.nid)
## TODO: try deamon thread
threading.Timer(api.INSPECTION_TIME_INTERVAL, _inspect_inactive_children, [sid]).start()
def _send_heartbeat(self, server_id, nid):
retry = 0
while retry < api.MAX_TRY:
try:
proxy = callme.Proxy(
server_id=server_id,
amqp_host="localhost",
amqp_port=5672,
timeout=3)
proxy.use_server(server_id).receive_heartbeats(nid, server_id)
# try again
retry = api.MAX_TRY
## record heartbeat locally as well
AppModel.objects(nid=nid).update_one(last_access=api._get_current_time())
is_master_failing = False
# try daemon thread
#threading.Timer(3, _send_heart_beat).setDaemon(True)
threading.Timer(api.CHECK_HEARTBEAT, self._send_heartbeat, [server_id, nid]).start()
except:
retry += 1
is_master_failing = True
print "Server: {} is not responding, trying {} times...".format(server_id, str(retry))
#print "Master server is not responding"
## master server failed
if is_master_failing:
self.timer = ConnTimeout(api._TIMEOUT,
app_api._send_handshake_msg,
servers=3,
args=[nid,
AppModel.objects(nid=nid).get().public_key,
AppModel.objects(nid=nid).get().created_at.isoformat()])
self.timer.start()
"""_send_handshake_msg(
def save_session(self, app, session, response):
domain = self.get_cookie_domain(app)
if not session:
## in web_server session is not defined
print "delete cookies"
response.delete_cookie(app.session_cookie_name, domain=domain)
return session
if self.get_expiration_time(app, session):
expiration = self.get_expiration_time(app, session)
else:
expiration = api._get_expiration_time()
print "expiration has been extended {}".format(expiration)
res = self.sessions.update_one(
{'ssid': session.ssid},
{
'$set':{
'by':self.by,
'ssid': session.ssid,
'user_data': session,
'expiration': expiration,
'modified':api._get_current_time()
}
}, upsert=True) ## upsert true
if res.upserted_id:
logger.info("New session id added to the system")
elif res.matched_count == 1 and res.modified_count == 1:
logger.info("Existing session id updated")
else:
logger.critical("Something wrong with user session")
response.set_cookie(app.session_cookie_name,
session.ssid,
expires=self.get_expiration_time(app, session),
httponly=True, domain=domain)
def process_msg_add_request(sid, uname, ssid, comment):
## if master fails to find slave node, try MAX_TRY times until it
## response 503
retry = 0
while retry < api.MAX_TRY:
try:
node = _get_next_worker(sid)
if node == None:
break
s_time = api._get_current_time().isoformat()
response = _send_add_comment_call(**{
'nid':node,
'sid':sid,
'by':uname,
'session_id':ssid,
'comment':comment,
'created_at':s_time
})
if not response:
retry+=1
q.custom_push(api.FAILED_NODES, node)
logger.error("failed... trying another node...")
## execution succeed
else:
return api.decrypt_msg(
WebServerModel.objects(sid=sid).get().private_key, response)
except:
retry+=1
logger.error("failed... trying another node...")
return {
'errorMsg':"chatting is not available at the moment"
}
