def fromJson(jsonObj): if 'class' in jsonObj: if jsonObj['class'] == 'Node': node = Node(jsonObj['server_id'], int(jsonObj['client_secure_port']), int(jsonObj['server_secure_port']), jsonObj['qualified_name'], jsonObj['hostname']) if "nodes" in jsonObj: node.setNodes(jsonObj['nodes']) if "priority" in jsonObj: node.setPriority(jsonObj['priority']) if "workerStates" in jsonObj: node.workerStates = jsonObj['workerStates'] return node if jsonObj['class'] == 'WorkerState': return WorkerState(jsonObj['host'], jsonObj['state'], jsonObj['workerId']) if jsonObj['class'] == 'Nodes': nodes = Nodes() for node in jsonObj['nodes'].itervalues(): nodes.addNode(node) return nodes if jsonObj['class'] == 'NodeConnectRequest': return NodeConnectRequest(jsonObj['server_id'], jsonObj['client_secure_port'], jsonObj['server_secure_port'], jsonObj['key'], jsonObj['qualified_name'], jsonObj['hostname']) return jsonObj
def testCacheObjects(self): node1 = Node("testhost", 8080, 9090) node2 = Node("testhost", 8081, 9091) nodes = Nodes() nodes.addNode(node1) nodes.addNode(node2) Cache().add("network-topology", nodes) cachedNodes = Cache().get("network-topology") self.assertEquals(2, cachedNodes.size())
def fromJson(jsonObj): if "class" in jsonObj: if jsonObj["class"] == "Node": node = Node( jsonObj["server_id"], int(jsonObj["client_secure_port"]), int(jsonObj["server_secure_port"]), jsonObj["qualified_name"], jsonObj["hostname"], ) if "nodes" in jsonObj: node.setNodes(jsonObj["nodes"]) if "priority" in jsonObj: node.setPriority(jsonObj["priority"]) if "workerStates" in jsonObj: node.workerStates = jsonObj["workerStates"] return node if jsonObj["class"] == "WorkerState": return WorkerState(jsonObj["host"], jsonObj["state"], jsonObj["workerId"]) if jsonObj["class"] == "Nodes": nodes = Nodes() for node in jsonObj["nodes"].itervalues(): nodes.addNode(node) return nodes if jsonObj["class"] == "NodeConnectRequest": return NodeConnectRequest( jsonObj["server_id"], jsonObj["client_secure_port"], jsonObj["server_secure_port"], jsonObj["key"], jsonObj["qualified_name"], jsonObj["hostname"], ) return jsonObj
def run(self, serverState, request, response): # first read platform capabilities and executables rdr=cpc.command.platform_exec_reader.PlatformExecutableReader() workerData=request.getParam('worker') if request.hasParam('worker-id'): workerID=request.getParam('worker-id') else: workerID='(none)' log.debug("Worker platform + executables: %s"%workerData) rdr.readString(workerData,"Worker-reported platform + executables") # match queued commands to executables. cwm=CommandWorkerMatcher(rdr.getPlatforms(), rdr.getExecutableList(), rdr.getWorkerRequirements()) cmds=cwm.getWork(serverState.getCmdQueue()) if not cwm.isDepleted(): # now sleep for 5 seconds to give the dataflow time to react to any # new state. time.sleep(5) cmds.extend(cwm.getWork(serverState.getCmdQueue())) # now check the forwarded variables conf=serverState.conf originatingServer=None heartbeatInterval=None try: # check whether there is an originating server. If not, we're it if self.forwarded: if 'originating-server-id' in request.headers: originatingServer = request.headers['originating-server-id'] # check the expected heartbeat time. log.debug("Forwarded message") if request.hasParam('heartbeat-interval'): heartbeatInterval = int(request.getParam('heartbeat-interval')) log.debug("Forwarded heartbeat interval is %d"% heartbeatInterval) except NameError: # self.forwarded does not exist. Treat it as if self.forwarded == False pass if originatingServer is None: # If the originating server property has not been set, the # request hasn't been forwarded, therefore we are the originating # server selfNode=Node.getSelfNode(conf) originatingServer = selfNode.getId() # we only store worker state in the server the worker connects to serverState.setWorkerState(WorkerStatus.WORKER_STATUS_CONNECTED,workerID, request.headers['originating-client']) if heartbeatInterval is None: heartbeatInterval = conf.getHeartbeatTime() log.debug("worker identified %s"%request.headers['originating-client'] ) if len(cmds) > 0: # first add them to the running list so they never get lost runningCmdList=serverState.getRunningCmdList() runningCmdList.add(cmds, originatingServer, heartbeatInterval) # construct the tar file with the workloads. tff=tempfile.TemporaryFile() tf=tarfile.open(fileobj=tff, mode="w:gz") # make the commands ready for cmd in cmds: log.debug("Adding command id %s to tar file."%cmd.id) # write the command description to the command's directory task=cmd.getTask() #log.debug(cmd) project=task.getProject() taskDir = "task_%s"%task.getID() cmddir=cmd.getDir() if not os.path.exists(cmddir): log.debug("cmddir %s did not exist. Created directory."%cmd.id) os.mkdir(cmddir) arcdir="%s"%(cmd.id) log.debug("cmddir=%s"%cmddir) outf=open(os.path.join(cmddir, "command.xml"), "w") cmd.writeWorkerXML(outf) outf.close() tf.add(cmddir, arcname=arcdir, recursive=True) # set the state of the command. tf.close() del(tf) tff.seek(0) # now send it back response.setFile(tff,'application/x-tar') #project.writeTasks() # the file is closed after the response is sent. log.info("Did direct worker-ready") else: nodes = conf.getNodes().getNodesByPriority() topology = Nodes() if request.hasParam('topology'): topology = json.loads(request.getParam('topology') ,object_hook = json_serializer.fromJson) thisNode = Node.getSelfNode(conf) thisNode.nodes = conf.getNodes() topology.addNode(thisNode) hasJob =False # temporary flag that should be removed for node in nodes: if topology.exists(node.getId()) == False: clnt=ServerMessage(node.getId()) clientResponse=clnt.workerReadyForwardedRequest(workerID, workerData, topology, originatingServer, heartbeatInterval, request.headers['originating-client']) if clientResponse.getType() == 'application/x-tar': log.log(cpc.util.log.TRACE, 'got work from %s'% (clientResponse.headers[ 'originating-server-id'])) hasJob=True # we need to rewrap the message #TODO stupid intermediary step because the mmap form # clientresponse is prematurely closed tmp = tempfile.TemporaryFile('w+b') message = clientResponse.getRawData() tmp.write(message.read(len(message))) tmp.seek(0) #for key in clientResponse.headers: # print "%s:%s"%(key,clientResponse.headers[key]) response.setFile(tmp,'application/x-tar') response.headers['originating-server-id']=\ clientResponse.headers[ 'originating-server-id'] #OPTIMIZE leads to a lot of folding and unfolding of #packages if not hasJob: response.add("No command") log.info("Did delegated worker-ready")