def run(self, serverState, request, response):
#get the connection params for this node
newParams = json.loads(request.getParam("connectionParams"))
conf = ServerConf()
nodes = conf.getNodes()
if nodes.exists(newParams['serverId']):
node = nodes.get(newParams['serverId'])
node.setHostname(newParams['hostname'])
node.setServerSecurePort(newParams['server_secure_port'])
node.setClientSecurePort(newParams['client_secure_port'])
node.setQualifiedName(newParams['fqdn'])
#Needed so that we write changes to conf file
conf.removeNode(node.server_id)
conf.addNode(node)
#update the network topology
ServerToServerMessage.getNetworkTopology(resetCache=True)
response.add("Updated connection parameters")
log.info("Updated connection params for %s" % node.toString())
else:
response.add("Requested update for node %s but this node " \
"is not a neigbouring node "%newParams[
'serverId'],
status="ERROR")
log.error("Failed updating connection params for %s" %
newParams['serverId'])
def run(self, serverState, request, response):
#get the connection params for this node
newParams =json.loads(request.getParam("connectionParams"))
conf = ServerConf()
nodes = conf.getNodes()
if nodes.exists(newParams['serverId']):
node = nodes.get(newParams['serverId'])
node.setHostname(newParams['hostname'])
node.setServerSecurePort(newParams['server_secure_port'])
node.setClientSecurePort(newParams['client_secure_port'])
node.setQualifiedName(newParams['fqdn'])
#Needed so that we write changes to conf file
conf.removeNode(node.server_id)
conf.addNode(node)
#update the network topology
ServerToServerMessage.getNetworkTopology(resetCache=True)
response.add("Updated connection parameters")
log.info("Updated connection params for %s"%node.toString())
else:
response.add("Requested update for node %s but this node " \
"is not a neigbouring node "%newParams[
'serverId'],
status="ERROR")
log.error("Failed updating connection params for %s"%newParams['serverId'])
def _sendMessage(self,node,fields,files = [],headers=dict()):
self.node = node
#TODO find nicer way to do this
#a bit backwards to initialize the parent object here.
ServerToServerMessage.__init__(self,node.getId())
#self.initialize(node.server_id)
headers['end-node'] = node.getHostname()
headers['end-node-port'] = node.getServerSecurePort()
msg = ServerRequest.prepareRequest(fields,[],headers)
resp = self.putRequest(msg)
def _sendMessage(self, node, fields, files=[], headers=dict()):
self.node = node
# TODO find nicer way to do this
# a bit backwards to initialize the parent object here.
ServerToServerMessage.__init__(self, node.getId())
# self.initialize(node.server_id)
headers["end-node"] = node.getHostname()
headers["end-node-port"] = node.getServerSecurePort()
msg = ServerRequest.prepareRequest(fields, [], headers)
resp = self.putRequest(msg)
def sendKeepAlive(conf):
"""
Sends a message for each connected node in order to keep the
connection alive
"""
log.log(cpc.util.log.TRACE, "Starting keep alive thread")
#first get the network topology. by doing this we know that the network
# topology is fetched and resides in the cache.
#since we are later on fetching all connections from the connection pool
# there will be no connection left to do this call thus we must be sure
# that its already cached.
ServerToServerMessage.getNetworkTopology()
while True:
log.log(cpc.util.log.TRACE, "Starting to send keep alive")
sentRequests = 0
for node in conf.getNodes().nodes.itervalues():
if node.isConnectedOutbound():
try:
connections = ServerConnectionPool().getAllConnections(
node)
for conn in connections:
try:
message = DirectServerMessage(node, conf)
message.conn = conn
message.pingServer(node.getId())
log.log(cpc.util.log.TRACE,
"keepAlive sent to %s" % node.toString())
except ServerConnectionError:
log.error("Connection to %s is broken" %
node.toString())
sentRequests += 1
except ConnectionPoolEmptyError:
#this just mean that no connections are available in the
# pool i.e they are used and communicated on thus we do
# not need to send keep alive messages on them
continue
keepAliveInterval = conf.getKeepAliveInterval()
log.log(
cpc.util.log.TRACE, "sent keep alive to %s nodes "
"will resend in %s seconds" % (sentRequests, keepAliveInterval))
time.sleep(keepAliveInterval)
def testBroadCastMessage(self):
self.create5NodeNetwork()
#this way we initialize the configs for a specific server which lets us test method calls as as specific server
clientConf = ConnectionBundle(confdir=self.serverConfs[0], reload=True)
serverConf = ServerConf(confdir=self.serverConfs[0], reload=True)
broadCastMessage = BroadcastMessage()
#logs should mention that we are setting elements to cache
broadCastMessage.updateNetworkTopology()
clientConf = ConnectionBundle(confdir=self.serverConfs[1], reload=True)
serverConf = ServerConf(confdir=self.serverConfs[1], reload=True)
#this should be brought from the cache(check in the logs)
ServerToServerMessage.getNetworkTopology()
def testBroadCastMessage(self):
self.create5NodeNetwork()
#this way we initialize the configs for a specific server which lets us test method calls as as specific server
clientConf = ConnectionBundle(confdir=self.serverConfs[0],reload=True)
serverConf = ServerConf(confdir=self.serverConfs[0],reload=True)
broadCastMessage = BroadcastMessage()
#logs should mention that we are setting elements to cache
broadCastMessage.updateNetworkTopology()
clientConf = ConnectionBundle(confdir=self.serverConfs[1],reload=True)
serverConf = ServerConf(confdir=self.serverConfs[1],reload=True)
#this should be brought from the cache(check in the logs)
ServerToServerMessage.getNetworkTopology()
def sendKeepAlive(conf):
"""
Sends a message for each connected node in order to keep the
connection alive
"""
log.log(cpc.util.log.TRACE,"Starting keep alive thread")
#first get the network topology. by doing this we know that the network
# topology is fetched and resides in the cache.
#since we are later on fetching all connections from the connection pool
# there will be no connection left to do this call thus we must be sure
# that its already cached.
ServerToServerMessage.getNetworkTopology()
while True:
log.log(cpc.util.log.TRACE,"Starting to send keep alive")
sentRequests = 0
for node in conf.getNodes().nodes.itervalues():
if node.isConnectedOutbound():
try:
connections = ServerConnectionPool().getAllConnections(node)
for conn in connections:
try:
message = DirectServerMessage(node,conf)
message.conn = conn
message.pingServer(node.getId())
log.log(cpc.util.log.TRACE,"keepAlive sent to %s"%node
.toString())
except ServerConnectionError:
log.error("Connection to %s is broken"%node.toString())
sentRequests+=1
except ConnectionPoolEmptyError:
#this just mean that no connections are available in the
# pool i.e they are used and communicated on thus we do
# not need to send keep alive messages on them
continue
keepAliveInterval = conf.getKeepAliveInterval()
log.log(cpc.util.log.TRACE,"sent keep alive to %s nodes "
"will resend in %s seconds"%(sentRequests,
keepAliveInterval))
time.sleep(keepAliveInterval)
def broadcast(self, fields, files=[], headers=dict()):
topology = ServerToServerMessage.getNetworkTopology()
if not topology:
log.error("Cannot find network topology.")
return
# we dont want to broadcast to ourself
node = Node.getSelfNode(ServerConf())
topology.removeNode(node.getId())
for node in topology.nodes.itervalues():
self._sendMessage(node, fields, files, headers)
def broadcast(self,fields,files = [],headers=dict()):
topology = ServerToServerMessage.getNetworkTopology()
if not topology:
log.error("Cannot find network topology.")
return
#we dont want to broadcast to ourself
node = Node.getSelfNode(ServerConf())
topology.removeNode(node.getId())
for node in topology.nodes.itervalues():
self._sendMessage(node,fields,files,headers)
def run(self, serverState, request, response):
"""
Used when a server wants to generate a network topology
"""
topology = Nodes()
if request.hasParam('topology'):
topology = json.loads(request.getParam('topology'),
object_hook=json_serializer.fromJson)
topology = ServerToServerMessage.requestNetworkTopology(topology,serverState)
response.add("", topology)
log.info("Returned network topology size %d" % topology.size())
def establishConnectionsAndBroadcastTopology(serverState,nodesArr):
#try to establish secure connections to a server
for node in nodesArr:
server_state.establishInboundConnection(node,serverState)
server_state.establishOutboundConnection(node)
NetworkTopologyCache().remove()
log.debug("starting broadcast!")
topology = ServerToServerMessage.getNetworkTopology()
## #recalculate the network topology and broadcast it
broadCastMessage = BroadcastMessage()
broadCastMessage.updateNetworkTopology()
def establishConnectionsAndBroadcastTopology(serverState, nodesArr):
#try to establish secure connections to a server
for node in nodesArr:
server_state.establishInboundConnection(node, serverState)
server_state.establishOutboundConnection(node)
NetworkTopologyCache().remove()
log.debug("starting broadcast!")
topology = ServerToServerMessage.getNetworkTopology()
## #recalculate the network topology and broadcast it
broadCastMessage = BroadcastMessage()
broadCastMessage.updateNetworkTopology()
def run(self, serverState, request, response):
"""
Used when a server wants to generate a network topology
"""
topology = Nodes()
if request.hasParam('topology'):
topology = json.loads(request.getParam('topology'),
object_hook=json_serializer.fromJson)
topology = ServerToServerMessage.requestNetworkTopology(
topology, serverState)
response.add("", topology)
log.info("Returned network topology size %d" % topology.size())
def updateNetworkTopology(self):
topology = ServerToServerMessage.getNetworkTopology()
cmdstring="network-topology-update"
fields = []
input = Input('cmd', cmdstring)
fields.append(input)
if topology != None:
input2 = Input('topology',
json.dumps(topology,default = json_serializer.toJson,indent=4)) # a json structure that needs to be dumped
fields.append(input2)
self.broadcast(fields,[])
def updateNetworkTopology(self):
topology = ServerToServerMessage.getNetworkTopology()
cmdstring = "network-topology-update"
fields = []
input = Input("cmd", cmdstring)
fields.append(input)
if topology != None:
input2 = Input(
"topology", json.dumps(topology, default=json_serializer.toJson, indent=4)
) # a json structure that needs to be dumped
fields.append(input2)
self.broadcast(fields, [])
def run(self, serverState, request, response):
"""
When a client requests a network topology.
ServerToServerMessage.getNetworkTopology() tries to first get it from
the cache, if it cant find it there it starts generating a topology
"""
topology = ServerToServerMessage.getNetworkTopology()
if not topology:
log.error("Cannot get network topology.")
return
response.add("", topology)
log.info("Returned network topology size %d" % topology.size())
def run(self, serverState, request, response):
"""
When a client requests a network topology.
ServerToServerMessage.getNetworkTopology() tries to first get it from
the cache, if it cant find it there it starts generating a topology
"""
topology = ServerToServerMessage.getNetworkTopology()
if not topology:
log.error("Cannot get network topology.")
return
response.add("", topology)
log.info("Returned network topology size %d" % topology.size())
def run(self, serverState, request, response):
ret_dict = {}
# handle project status
user = self.getUser(request)
explicit_project=self.getProject(request, serverState, only_explicit=True)
if explicit_project is not None:
projects = [explicit_project.getName()]
else:
if user.isSuperuser():
projects = lst=serverState.getProjectList().list()
else:
projects = UserHandler().getProjectListForUser(user)
ret_prj_dict = {}
for prj_str in projects:
ret_prj_dict[prj_str] = dict()
queue = {'queue' : [], 'running': []}
state_count = {}
err_list=[]
warn_list=[]
prj_obj = serverState.getProjectList().get(prj_str)
# we iterate over the childred rather than calling _traverseInstance
# here to avoid the project itself being counted as an instance
for child in prj_obj.getSubValueIterList():
self._traverseInstance(prj_obj.getSubValue([child]),
state_count, queue, err_list,
warn_list)
ret_prj_dict[prj_str]['states'] = state_count
ret_prj_dict[prj_str]['queue'] = queue
ret_prj_dict[prj_str]['errors'] = err_list
ret_prj_dict[prj_str]['warnings'] = warn_list
if prj_str == request.session.get('default_project_name', None):
ret_prj_dict[prj_str]['default']=True
ret_dict['projects'] = ret_prj_dict
if explicit_project is not None:
# client only want info for this project, return with that.
response.add("", ret_dict)
return
# handle network
topology = ServerToServerMessage.getNetworkTopology()
if not topology:
log.error("Cannot get network topology.")
return
numWorkers = 0
numServers = 0
numLocalWorkers =len(WorkerStateHandler.getConnectedWorkers(serverState.workerStates))
numNotConnectedLocalServers = 0
numLocalServers = 0
for id,node in ServerConf().getNodes().nodes.iteritems():
if node.isConnected():
numLocalServers +=1
else:
numNotConnectedLocalServers+=1
for name, node in topology.nodes.iteritems():
numWorkers += len(node.workerStates)
numServers += 1
numWorkers +=numLocalWorkers
ret_dict['network'] = {
'workers': numWorkers,
'servers': numServers,
'local_workers': numLocalWorkers,
'local_servers': numLocalServers,
'not_connected_local_servers':numNotConnectedLocalServers
}
response.add("", ret_dict)
def run(self, serverState, request, response):
ret_dict = {}
# handle project status
user = self.getUser(request)
explicit_project = self.getProject(request,
serverState,
only_explicit=True)
if explicit_project is not None:
projects = [explicit_project.getName()]
else:
if user.isSuperuser():
projects = lst = serverState.getProjectList().list()
else:
projects = UserHandler().getProjectListForUser(user)
ret_prj_dict = {}
for prj_str in projects:
ret_prj_dict[prj_str] = dict()
queue = {'queue': [], 'running': []}
state_count = {}
err_list = []
warn_list = []
prj_obj = serverState.getProjectList().get(prj_str)
# we iterate over the childred rather than calling _traverseInstance
# here to avoid the project itself being counted as an instance
for child in prj_obj.getSubValueIterList():
self._traverseInstance(prj_obj.getSubValue([child]),
state_count, queue, err_list, warn_list)
ret_prj_dict[prj_str]['states'] = state_count
ret_prj_dict[prj_str]['queue'] = queue
ret_prj_dict[prj_str]['errors'] = err_list
ret_prj_dict[prj_str]['warnings'] = warn_list
if prj_str == request.session.get('default_project_name', None):
ret_prj_dict[prj_str]['default'] = True
ret_dict['projects'] = ret_prj_dict
if explicit_project is not None:
# client only want info for this project, return with that.
response.add("", ret_dict)
return
# handle network
topology = ServerToServerMessage.getNetworkTopology()
if not topology:
log.error("Cannot get network topology.")
return
numWorkers = 0
numServers = 0
numLocalWorkers = len(
WorkerStateHandler.getConnectedWorkers(serverState.workerStates))
numNotConnectedLocalServers = 0
numLocalServers = 0
for id, node in ServerConf().getNodes().nodes.iteritems():
if node.isConnected():
numLocalServers += 1
else:
numNotConnectedLocalServers += 1
for name, node in topology.nodes.iteritems():
numWorkers += len(node.workerStates)
numServers += 1
numWorkers += numLocalWorkers
ret_dict['network'] = {
'workers': numWorkers,
'servers': numServers,
'local_workers': numLocalWorkers,
'local_servers': numLocalServers,
'not_connected_local_servers': numNotConnectedLocalServers
}
response.add("", ret_dict)
