def __connect(self, require_certificate_authentication=None, disable_cookies=False):
'''
inputs:
require_certificate_authentication:boolean requires a certificate from the server
'''
if require_certificate_authentication is not None:
require_certificate_authentication = require_certificate_authentication
else:
try:
if self.require_certificate_authentication is not None:
require_certificate_authentication = self.require_certificate_authentication
else:
require_certificate_authentication = True
except AttributeError:
require_certificate_authentication = True
try:
if require_certificate_authentication:
log.log(cpc.util.log.TRACE,"Connecting HTTPS with cert authentication")
self.conn=client_connection.ClientConnectionRequireCert(self.conf)
else:
log.log(cpc.util.log.TRACE,"Connecting HTTPS with no cert authentication")
self.conn=client_connection.ClientConnectionNoCertRequired(
self.conf, disable_cookies)
self.conn.connect(self.host,self.port)
except httplib.HTTPException as e:
raise ClientConnectionError(e,self.host,self.port)
except socket.error as e:
raise ClientConnectionError(e,self.host,self.port)
def initialize(self,endNodeId):
topology=self.getNetworkTopology()
if not topology:
log.error("Cannot get network topology")
return
# this is myself:
startNode = Node.getSelfNode(self.conf)
self.endNode = topology.nodes.get(endNodeId)
log.log(cpc.util.log.TRACE,"Finding route between %s(%s %s) and %s(%s "
"%s"")"%(startNode.server_id,startNode.getHostname(),
startNode.getServerSecurePort(),
self.endNode.server_id,
self.endNode.getHostname(),self.endNode.getServerSecurePort()))
route = Nodes.findRoute(startNode, self.endNode,topology)
self.hostNode = route[1] #route[0] is the current host
self.host = self.hostNode.getHostname()
self.port = self.hostNode.getServerSecurePort()
self.serverId = self.hostNode.getId()
log.log(cpc.util.log.TRACE,"Server-to-server connecting to %s(%s:%s)"%
(self.serverId,self.host,self.port))
def putConnection(self,connection,host,port):
#create a queue in case we dont have one
self.getOrCreateQueue(host,port)
key = self.getKey(host, port)
with self.listlock:
self.pool[key].put(connection,False)
log.log(cpc.util.log.TRACE,"put back connection in pool for host:%s:%s"%(host,port))
def getCommandOutputData(cmdID, workerServer):
log.log(cpc.util.log.TRACE,"Trying to pull command output from %s"%
workerServer)
s2smsg=ServerMessage(workerServer)
rundata_response = s2smsg.pullAssetRequest(cmdID, Asset.cmdOutput())
if rundata_response.getType() != "application/x-tar":
log.error("Incorrect response type: %s, should be %s"%
(rundata_response.getType(), 'application/x-tar'))
if rundata_response.getType() == "text/json":
errormsg=rundata_response.message.read(len(rundata_response.
message))
presp=ProcessedResponse(rundata_response)
if not presp.isOK():
log.error('Response from worker server not OK: %s'%
errormsg)
else:
s2smsg.clearAssetRequest(cmdID)
log.log(cpc.util.log.TRACE,
"Successfully pulled command output data from %s."%
workerServer)
return rundata_response
#runfile = rundata_response.getRawData()
#this doesnt work because the mmap closes as it is returned
return None
def __init__(self):
self.__dict__ = self.__shared_state
if len(self.__shared_state)>0:
return
ConnectionPool.__init__(self)
log.log(cpc.util.log.TRACE,"instantiation of Server connection pool")
def __init__(self):
self.__dict__ = self.__shared_state
if len(self.__shared_state)>0:
return
self.cacheLock = threading.Lock()
log.log(cpc.util.log.TRACE,"instantiation of cache")
self.cache = {}
def connect(self,host,port):
self.host = host
self.port = port
log.log(cpc.util.log.TRACE,"Connecting HTTPS with no cert req to host %s, port %s"%(
self.host,self.port))
self.conn = HttpsConnectionNoCertReq(self.host,self.port)
self.conn.connect()
self.connected=True
def getOrCreateQueue(self,host,port):
key = self.getKey(host, port)
if key not in self.pool:
log.log(cpc.util.log.TRACE,"Instantiating server https "
"connection pool for host:%s:%s"%(host,port))
q = Queue()
self.pool[key] = q
else:
q= self.pool[key]
return q
def connect(self, host, port):
self.host = host
self.port = port
log.log(
cpc.util.log.TRACE,
"Connecting HTTPS with no cert req to host %s, port %s" %
(self.host, self.port))
self.conn = HttpsConnectionNoCertReq(self.host, self.port)
self.conn.connect()
self.connected = True
def add(self,key,value,ttl=None):
'''
ttl : time to live in seconds, if set to none the cache lifetime is infinite
'''
with self.cacheLock:
if ttl:
log.log(cpc.util.log.TRACE,'adding object %s to cache with ttl %s'%(key,ttl))
else:
log.log(cpc.util.log.TRACE,'adding object %s to cache'%key)
if(ttl):
ttl = int(time.time())+ttl
self.cache[key] = (value,ttl)
def handleSinglePart(headers,message):
contentLength = long(headers['content-length'])
if headers['content-type'] == 'application/x-www-form-urlencoded' or headers['content-type'] == 'application/x-www-form-urlencoded; charset=UTF-8': #TODO generalize
msg = message.read(contentLength)
log.log(cpc.util.log.TRACE,'RAW msg is %s'%msg)
parsedDict = urlparse.parse_qs(msg) #Note values here are stored in lists, this is so one can handle many inputs with same name, for now we dont want that as our multipart parsing does not support it
params = dict()
for k,v in parsedDict.iteritems():
params[k] = v[0]
request = ServerRequest(headers,None,params)
return request
def handleSinglePart(headers,message):
contentLength = long(headers['content-length'])
if headers['content-type'] == 'application/x-www-form-urlencoded' or headers['content-type'] == 'application/x-www-form-urlencoded; charset=UTF-8': #TODO generalize
msg = message.read(contentLength)
parsedDict = urlparse.parse_qs(msg) #Note values here are stored in lists, this is so one can handle many inputs with same name, for now we dont want that as our multipart parsing does not support it
params = dict()
for k,v in parsedDict.iteritems():
params[k] = v[0]
log.log(cpc.util.log.TRACE,'msg is %s'%params)
request = ServerRequest(headers,None,params)
return request
def connect(self, host, port):
self.host = host
self.port = port
privateKey = self.conf.getPrivateKey()
keyChain = self.conf.getCaChainFile()
cert = self.conf.getCertFile()
log.log(
cpc.util.log.TRACE,
"Connecting VerHTTPS to host %s, port %s" % (self.host, self.port))
self.conn = HttpsConnectionWithCertReq(self.host, self.port,
privateKey, keyChain, cert)
self.conn.connect()
self.connected = True
def run(self, serverState, request, response):
cmdID=request.getParam('cmd_id')
assetType=request.getParam('asset_type')
try:
runfile=serverState.getLocalAssets().getAsset(cmdID,
assetType).getData()
except:
log.error("Local asset cmdid=%s not found!"%cmdID)
response.add("Command output data from cmdID %s not found on this server (%s)."%
(cmdID,serverState.conf.getHostName()),
status="ERROR")
else:
asset=serverState.getLocalAssets().getCmdOutputAsset(cmdID)
log.log(cpc.util.log.TRACE,"Local asset cmdid=%s \nproject server=%s"%
(asset.cmdID, asset.projectServer))
response.setFile(runfile,'application/x-tar')
log.info("Pulled asset %s/%s"%(cmdID, assetType))
def connect(self,host,port):
self.host = host
self.port = port
privateKey = self.conf.getPrivateKey()
keyChain = self.conf.getCaChainFile()
cert = self.conf.getCertFile()
log.log(cpc.util.log.TRACE,"Connecting VerHTTPS to host %s, port %s"%(
self.host,self.port))
self.conn = HttpsConnectionWithCertReq(self.host,
self.port,
privateKey,
keyChain,
cert)
self.conn.connect()
self.connected=True
def getAllConnections(self,host,port):
connections = []
#check if we have a queue instantiated for that host
with self.listlock:
q = self.getOrCreateQueue(host,port)
#we try yo get all available connections from the pool
while not q.empty():
connections.append(q.get())
if len(connections)==0:
log.log(cpc.util.log.TRACE,"No connections available in pool for "
"host:%s:%s"%(host,port))
raise ConnectionPoolEmptyError("No connections available in pool for "
"host:%s:%s"%(host,port))
return connections
def run(self, serverState, request, response):
workerID=request.getParam('worker_id')
workerDir=request.getParam('worker_dir')
iteration=request.getParam('iteration')
itemsXML=request.getParam('heartbeat_items')
log.log(cpc.util.log.TRACE, 'items: %s'%itemsXML)
hwr=cpc.command.heartbeat.HeartbeatItemReader()
hwr.readString(itemsXML, "worker heartbeat items")
faultyItems=[]
Nhandled=len(hwr.getItems())
ret=serverState.getRunningCmdList().ping(workerID, workerDir, iteration,
hwr.getItems(), False,
faultyItems)
if len(faultyItems)==0:
response.add('', data=serverState.conf.getHeartbeatTime())
else:
response.add('Heatbeat NOT OK', status="ERROR", data=faultyItems)
log.info("Handled %d forwarded heartbeat signal items."%(Nhandled))
def getConnection(self,host,port):
#check if we have a queue instantiated for that host
with self.listlock:
q = self.getOrCreateQueue(host,port)
try:
#we wait to until we get hold of a connection,
# if we do not get a connection after timeout then all available
# connections must have died.
connection = q.get(block=True,timeout=15)
log.log(cpc.util.log.TRACE,"Got a connection from pool for "
"host:%s:%s"%(host,port))
except Empty as e:
log.log(cpc.util.log.TRACE,"No connections available in pool for "
"host:%s:%s"%(host,port))
raise ConnectionPoolEmptyError(e)
return connection
def run(self, serverState, request, response):
cmdID = request.getParam('cmd_id')
assetType = request.getParam('asset_type')
try:
runfile = serverState.getLocalAssets().getAsset(
cmdID, assetType).getData()
except:
log.error("Local asset cmdid=%s not found!" % cmdID)
response.add(
"Command output data from cmdID %s not found on this server (%s)."
% (cmdID, serverState.conf.getHostName()),
status="ERROR")
else:
asset = serverState.getLocalAssets().getCmdOutputAsset(cmdID)
log.log(
cpc.util.log.TRACE,
"Local asset cmdid=%s \nproject server=%s" %
(asset.cmdID, asset.projectServer))
response.setFile(runfile, 'application/x-tar')
log.info("Pulled asset %s/%s" % (cmdID, assetType))
def matchCommandWorker(matcher, command):
"""Function to use in queue.getUntil() to get a number of commands from
the queue.
TODO: this is where performance tuning results should be used."""
cont = True # whether to continue getting commands from the queue
# whether to use this command: make sure we only have a single type
use = False
execID = matcher.getExecID(command)
log.log(cpc.util.log.TRACE, "exec id is %s" % execID)
log.debug("Type: %s" % command.getTask().getFunctionName())
if execID is not None:
use = matcher.checkType(command.getTask().getFunctionName()) and matcher.checkWorkerRequirements(command)
log.debug("Should use: %s " % use)
if use:
if matcher.checkAddResources(command):
use = True
else:
use = False
cont = False
return (cont, use)
def matchCommandWorker(matcher, command):
"""Function to use in queue.getUntil() to get a number of commands from
the queue.
TODO: this is where performance tuning results should be used."""
cont=True # whether to continue getting commands from the queue
# whether to use this command: make sure we only have a single type
use=False
execID=matcher.getExecID(command)
log.log(cpc.util.log.TRACE,'exec id is %s'%execID)
log.debug("Type: %s"%command.getTask().getFunctionName())
if execID is not None:
use=(matcher.checkType(command.getTask().getFunctionName()) and
matcher.checkWorkerRequirements(command))
log.debug("Should use: %s "%use)
if use:
if matcher.checkAddResources(command):
use=True
else:
use=False
cont=False
return (cont, use)
def get(self,key):
"""
returns: the cacheobject if it exists, False otherwise
"""
with self.cacheLock:
val = False
if key in self.cache:
now = int(time.time())
val,ttl = self.cache[key]
if ttl<now:
val = False
else:
log.log(cpc.util.log.TRACE,'object %s in cache has expired'%key)
elif val == False:
log.log(cpc.util.log.TRACE,'did not find object %s in cache'%key)
else:
log.log(cpc.util.log.TRACE,'getting object %s from cache'%key)
return val
def run(self, serverState, request, response):
# first read platform capabilities and executables
rdr=cpc.command.platform_exec_reader.PlatformExecutableReader()
workerData=request.getParam('worker')
if request.hasParam('worker-id'):
workerID=request.getParam('worker-id')
else:
workerID='(none)'
log.debug("Worker platform + executables: %s"%workerData)
rdr.readString(workerData,"Worker-reported platform + executables")
# match queued commands to executables.
cwm=CommandWorkerMatcher(rdr.getPlatforms(),
rdr.getExecutableList(),
rdr.getWorkerRequirements())
cmds=cwm.getWork(serverState.getCmdQueue())
if not cwm.isDepleted():
# now sleep for 5 seconds to give the dataflow time to react to any
# new state.
time.sleep(5)
cmds.extend(cwm.getWork(serverState.getCmdQueue()))
# now check the forwarded variables
conf=serverState.conf
originatingServer=None
heartbeatInterval=None
try:
# check whether there is an originating server. If not, we're it
if self.forwarded:
if 'originating-server-id' in request.headers:
originatingServer = request.headers['originating-server-id']
# check the expected heartbeat time.
log.debug("Forwarded message")
if request.hasParam('heartbeat-interval'):
heartbeatInterval = int(request.getParam('heartbeat-interval'))
log.debug("Forwarded heartbeat interval is %d"%
heartbeatInterval)
except NameError:
# self.forwarded does not exist. Treat it as if self.forwarded == False
pass
if originatingServer is None:
# If the originating server property has not been set, the
# request hasn't been forwarded, therefore we are the originating
# server
selfNode=Node.getSelfNode(conf)
originatingServer = selfNode.getId()
# we only store worker state in the server the worker connects to
serverState.setWorkerState(WorkerStatus.WORKER_STATUS_CONNECTED,workerID,
request.headers['originating-client'])
if heartbeatInterval is None:
heartbeatInterval = conf.getHeartbeatTime()
log.debug("worker identified %s"%request.headers['originating-client'] )
if len(cmds) > 0:
# first add them to the running list so they never get lost
runningCmdList=serverState.getRunningCmdList()
runningCmdList.add(cmds, originatingServer, heartbeatInterval)
# construct the tar file with the workloads.
tff=tempfile.TemporaryFile()
tf=tarfile.open(fileobj=tff, mode="w:gz")
# make the commands ready
for cmd in cmds:
log.debug("Adding command id %s to tar file."%cmd.id)
# write the command description to the command's directory
task=cmd.getTask()
#log.debug(cmd)
project=task.getProject()
taskDir = "task_%s"%task.getID()
cmddir=cmd.getDir()
if not os.path.exists(cmddir):
log.debug("cmddir %s did not exist. Created directory."%cmd.id)
os.mkdir(cmddir)
arcdir="%s"%(cmd.id)
log.debug("cmddir=%s"%cmddir)
outf=open(os.path.join(cmddir, "command.xml"), "w")
cmd.writeWorkerXML(outf)
outf.close()
tf.add(cmddir, arcname=arcdir, recursive=True)
# set the state of the command.
tf.close()
del(tf)
tff.seek(0)
# now send it back
response.setFile(tff,'application/x-tar')
#project.writeTasks()
# the file is closed after the response is sent.
log.info("Did direct worker-ready")
else:
nodes = conf.getNodes().getNodesByPriority()
topology = Nodes()
if request.hasParam('topology'):
topology = json.loads(request.getParam('topology')
,object_hook = json_serializer.fromJson)
thisNode = Node.getSelfNode(conf)
thisNode.nodes = conf.getNodes()
topology.addNode(thisNode)
hasJob =False # temporary flag that should be removed
for node in nodes:
if topology.exists(node.getId()) == False:
clnt=ServerMessage(node.getId())
clientResponse=clnt.workerReadyForwardedRequest(workerID,
workerData,
topology,
originatingServer,
heartbeatInterval,
request.headers['originating-client'])
if clientResponse.getType() == 'application/x-tar':
log.log(cpc.util.log.TRACE,
'got work from %s'%
(clientResponse.headers[
'originating-server-id']))
hasJob=True
# we need to rewrap the message
#TODO stupid intermediary step because the mmap form
# clientresponse is prematurely closed
tmp = tempfile.TemporaryFile('w+b')
message = clientResponse.getRawData()
tmp.write(message.read(len(message)))
tmp.seek(0)
#for key in clientResponse.headers:
# print "%s:%s"%(key,clientResponse.headers[key])
response.setFile(tmp,'application/x-tar')
response.headers['originating-server-id']=\
clientResponse.headers[
'originating-server-id']
#OPTIMIZE leads to a lot of folding and unfolding of
#packages
if not hasJob:
response.add("No command")
log.info("Did delegated worker-ready")
def __init__(self):
log.log(cpc.util.log.TRACE,"instantiation of connectionPool")
self.pool = {}
self.listlock=threading.Lock()
def handleMultipart(mainHeaders,msgStream):
files = dict()
params = dict()
BOUNDARY = "--"+HttpMethodParser.extractBoundary(mainHeaders)
stopBoundary = BOUNDARY+"--"
terminateBoundary = ''
msgStream.readline() #has an empty line at start that we want to get rid of
while terminateBoundary != stopBoundary:
headers = mimetools.Message(msgStream)
terminateBoundary = ''
log.log(cpc.util.log.TRACE,'multipart headers are %s'%headers.headers)
if(ServerRequest.isFile(headers['Content-Disposition'])):
file = tempfile.TemporaryFile(mode="w+b")
name = ServerRequest.getFieldName(headers['Content-Disposition'])
notused,contentDispositionParams = cgi.parse_header(headers['Content-Disposition'])
name = contentDispositionParams['name']
#if we have a content length we just read it and store the data
contentLength = headers.getheader('Content-Length')
if(contentLength): # If a content length is sent we parse the nice way
bytes = int(contentLength)
if(ServerRequest.isFile(headers['Content-Disposition'])):
file.write(msgStream.read(bytes))
else:
line = msgStream.read(bytes)
log.log(cpc.util.log.TRACE,"line is "+line)
params[name] = line
msgStream.readline() ## we will have a trailin CRLF that we just want to get rid of
if(ServerRequest.isFile(headers['Content-Disposition'])):
readBytes = 0
while(True):
line = msgStream.readline()
if re.search(BOUNDARY,line):
#time to wrap it up
if(line[-2:] == '\r\n'):
line = line[:-2]
elif(line[-1:] == '\n'):
line = line[:-1]
terminateBoundary = line
file.seek(0)
skipBytes = 2
realFile = tempfile.TemporaryFile(mode="w+b")
realFile.write(file.read(readBytes-skipBytes))
file.close()
realFile.seek(0)
#For testing during dev only!!
#runTest(realFile)
files[name]= realFile
break
else:
readBytes +=len(line)
file.write(line)
else:
while(True):
line = msgStream.readline()
if(line[-2:] == '\r\n'):
line = line[:-2]
elif(line[-1:] == '\n'):
line = line[:-1]
if re.search(BOUNDARY,line):
terminateBoundary = line
break;
else:
if name in params:
params[name]+= line
else:
params[name] = line
return ServerRequest(mainHeaders,None,params,files)
def remove(self,key):
with self.cacheLock:
if key in self.cache:
log.log(cpc.util.log.TRACE,'removing object %s from cache'%key)
del(self.cache[key])
def cleanAll(self):
with self.cacheLock:
self.cache={}
log.log(cpc.util.log.TRACE,'cleaning all objects from cache')
def handleMultipart(mainHeaders,msgStream):
files = dict()
params = dict()
BOUNDARY = "--"+HttpMethodParser.extractBoundary(mainHeaders)
stopBoundary = BOUNDARY+"--"
terminateBoundary = ''
msgStream.readline() #has an empty line at start that we want to get rid of
while terminateBoundary != stopBoundary:
headers = mimetools.Message(msgStream)
terminateBoundary = ''
log.log(cpc.util.log.TRACE,'multipart headers are %s'%headers.headers)
if(ServerRequest.isFile(headers['Content-Disposition'])):
file = tempfile.TemporaryFile(mode="w+b")
name = ServerRequest.getFieldName(headers['Content-Disposition'])
notused,contentDispositionParams = cgi.parse_header(headers['Content-Disposition'])
name = contentDispositionParams['name']
#if we have a content length we just read it and store the data
contentLength = headers.getheader('Content-Length')
if(contentLength): # If a content length is sent we parse the nice way
bytes = int(contentLength)
if(ServerRequest.isFile(headers['Content-Disposition'])):
file.write(msgStream.read(bytes))
else:
line = msgStream.read(bytes)
log.log(cpc.util.log.TRACE,"line is "+line)
params[name] = line
msgStream.readline() ## we will have a trailin CRLF that we just want to get rid of
if(ServerRequest.isFile(headers['Content-Disposition'])):
readBytes = 0
while(True):
line = msgStream.readline()
if re.search(BOUNDARY,line):
#time to wrap it up
if(line[-2:] == '\r\n'):
line = line[:-2]
elif(line[-1:] == '\n'):
line = line[:-1]
terminateBoundary = line
file.seek(0)
skipBytes = 2
realFile = tempfile.TemporaryFile(mode="w+b")
realFile.write(file.read(readBytes-skipBytes))
file.close()
realFile.seek(0)
#For testing during dev only!!
#runTest(realFile)
files[name]= realFile
break
else:
readBytes +=len(line)
file.write(line)
else:
while(True):
line = msgStream.readline()
if(line[-2:] == '\r\n'):
line = line[:-2]
elif(line[-1:] == '\n'):
line = line[:-1]
if re.search(BOUNDARY,line):
terminateBoundary = line
break;
else:
if name in params:
params[name]+= line
else:
params[name] = line
return ServerRequest(mainHeaders,None,params,files)