def fromJson(jsonObj):
if 'class' in jsonObj:
if jsonObj['class'] == 'Node':
node = Node(jsonObj['server_id'],
int(jsonObj['client_secure_port']),
int(jsonObj['server_secure_port']),
jsonObj['qualified_name'], jsonObj['hostname'])
if "nodes" in jsonObj:
node.setNodes(jsonObj['nodes'])
if "priority" in jsonObj:
node.setPriority(jsonObj['priority'])
if "workerStates" in jsonObj:
node.workerStates = jsonObj['workerStates']
return node
if jsonObj['class'] == 'WorkerState':
return WorkerState(jsonObj['host'], jsonObj['state'],
jsonObj['workerId'])
if jsonObj['class'] == 'Nodes':
nodes = Nodes()
for node in jsonObj['nodes'].itervalues():
nodes.addNode(node)
return nodes
if jsonObj['class'] == 'NodeConnectRequest':
return NodeConnectRequest(jsonObj['server_id'],
jsonObj['client_secure_port'],
jsonObj['server_secure_port'],
jsonObj['key'],
jsonObj['qualified_name'],
jsonObj['hostname'])
return jsonObj
def testCacheObjects(self):
node1 = Node("testhost", 8080, 9090)
node2 = Node("testhost", 8081, 9091)
nodes = Nodes()
nodes.addNode(node1)
nodes.addNode(node2)
Cache().add("network-topology", nodes)
cachedNodes = Cache().get("network-topology")
self.assertEquals(2, cachedNodes.size())
def testCacheObjects(self):
node1 = Node("testhost", 8080, 9090)
node2 = Node("testhost", 8081, 9091)
nodes = Nodes()
nodes.addNode(node1)
nodes.addNode(node2)
Cache().add("network-topology", nodes)
cachedNodes = Cache().get("network-topology")
self.assertEquals(2, cachedNodes.size())
def fromJson(jsonObj):
if "class" in jsonObj:
if jsonObj["class"] == "Node":
node = Node(
jsonObj["server_id"],
int(jsonObj["client_secure_port"]),
int(jsonObj["server_secure_port"]),
jsonObj["qualified_name"],
jsonObj["hostname"],
)
if "nodes" in jsonObj:
node.setNodes(jsonObj["nodes"])
if "priority" in jsonObj:
node.setPriority(jsonObj["priority"])
if "workerStates" in jsonObj:
node.workerStates = jsonObj["workerStates"]
return node
if jsonObj["class"] == "WorkerState":
return WorkerState(jsonObj["host"], jsonObj["state"], jsonObj["workerId"])
if jsonObj["class"] == "Nodes":
nodes = Nodes()
for node in jsonObj["nodes"].itervalues():
nodes.addNode(node)
return nodes
if jsonObj["class"] == "NodeConnectRequest":
return NodeConnectRequest(
jsonObj["server_id"],
jsonObj["client_secure_port"],
jsonObj["server_secure_port"],
jsonObj["key"],
jsonObj["qualified_name"],
jsonObj["hostname"],
)
return jsonObj
def run(self, serverState, request, response):
# first read platform capabilities and executables
rdr=cpc.command.platform_exec_reader.PlatformExecutableReader()
workerData=request.getParam('worker')
if request.hasParam('worker-id'):
workerID=request.getParam('worker-id')
else:
workerID='(none)'
log.debug("Worker platform + executables: %s"%workerData)
rdr.readString(workerData,"Worker-reported platform + executables")
# match queued commands to executables.
cwm=CommandWorkerMatcher(rdr.getPlatforms(),
rdr.getExecutableList(),
rdr.getWorkerRequirements())
cmds=cwm.getWork(serverState.getCmdQueue())
if not cwm.isDepleted():
# now sleep for 5 seconds to give the dataflow time to react to any
# new state.
time.sleep(5)
cmds.extend(cwm.getWork(serverState.getCmdQueue()))
# now check the forwarded variables
conf=serverState.conf
originatingServer=None
heartbeatInterval=None
try:
# check whether there is an originating server. If not, we're it
if self.forwarded:
if 'originating-server-id' in request.headers:
originatingServer = request.headers['originating-server-id']
# check the expected heartbeat time.
log.debug("Forwarded message")
if request.hasParam('heartbeat-interval'):
heartbeatInterval = int(request.getParam('heartbeat-interval'))
log.debug("Forwarded heartbeat interval is %d"%
heartbeatInterval)
except NameError:
# self.forwarded does not exist. Treat it as if self.forwarded == False
pass
if originatingServer is None:
# If the originating server property has not been set, the
# request hasn't been forwarded, therefore we are the originating
# server
selfNode=Node.getSelfNode(conf)
originatingServer = selfNode.getId()
# we only store worker state in the server the worker connects to
serverState.setWorkerState(WorkerStatus.WORKER_STATUS_CONNECTED,workerID,
request.headers['originating-client'])
if heartbeatInterval is None:
heartbeatInterval = conf.getHeartbeatTime()
log.debug("worker identified %s"%request.headers['originating-client'] )
if len(cmds) > 0:
# first add them to the running list so they never get lost
runningCmdList=serverState.getRunningCmdList()
runningCmdList.add(cmds, originatingServer, heartbeatInterval)
# construct the tar file with the workloads.
tff=tempfile.TemporaryFile()
tf=tarfile.open(fileobj=tff, mode="w:gz")
# make the commands ready
for cmd in cmds:
log.debug("Adding command id %s to tar file."%cmd.id)
# write the command description to the command's directory
task=cmd.getTask()
#log.debug(cmd)
project=task.getProject()
taskDir = "task_%s"%task.getID()
cmddir=cmd.getDir()
if not os.path.exists(cmddir):
log.debug("cmddir %s did not exist. Created directory."%cmd.id)
os.mkdir(cmddir)
arcdir="%s"%(cmd.id)
log.debug("cmddir=%s"%cmddir)
outf=open(os.path.join(cmddir, "command.xml"), "w")
cmd.writeWorkerXML(outf)
outf.close()
tf.add(cmddir, arcname=arcdir, recursive=True)
# set the state of the command.
tf.close()
del(tf)
tff.seek(0)
# now send it back
response.setFile(tff,'application/x-tar')
#project.writeTasks()
# the file is closed after the response is sent.
log.info("Did direct worker-ready")
else:
nodes = conf.getNodes().getNodesByPriority()
topology = Nodes()
if request.hasParam('topology'):
topology = json.loads(request.getParam('topology')
,object_hook = json_serializer.fromJson)
thisNode = Node.getSelfNode(conf)
thisNode.nodes = conf.getNodes()
topology.addNode(thisNode)
hasJob =False # temporary flag that should be removed
for node in nodes:
if topology.exists(node.getId()) == False:
clnt=ServerMessage(node.getId())
clientResponse=clnt.workerReadyForwardedRequest(workerID,
workerData,
topology,
originatingServer,
heartbeatInterval,
request.headers['originating-client'])
if clientResponse.getType() == 'application/x-tar':
log.log(cpc.util.log.TRACE,
'got work from %s'%
(clientResponse.headers[
'originating-server-id']))
hasJob=True
# we need to rewrap the message
#TODO stupid intermediary step because the mmap form
# clientresponse is prematurely closed
tmp = tempfile.TemporaryFile('w+b')
message = clientResponse.getRawData()
tmp.write(message.read(len(message)))
tmp.seek(0)
#for key in clientResponse.headers:
# print "%s:%s"%(key,clientResponse.headers[key])
response.setFile(tmp,'application/x-tar')
response.headers['originating-server-id']=\
clientResponse.headers[
'originating-server-id']
#OPTIMIZE leads to a lot of folding and unfolding of
#packages
if not hasJob:
response.add("No command")
log.info("Did delegated worker-ready")
