示例#1
0
    def __connect(self, require_certificate_authentication=None, disable_cookies=False):

        '''
        inputs:
             require_certificate_authentication:boolean  requires a certificate from the server
        '''
        if require_certificate_authentication is not None:
            require_certificate_authentication = require_certificate_authentication
        else:
            try:
                if self.require_certificate_authentication is not None:
                    require_certificate_authentication = self.require_certificate_authentication
                else:
                    require_certificate_authentication = True
            except AttributeError:
                require_certificate_authentication = True
        try:
            if require_certificate_authentication:
                log.log(cpc.util.log.TRACE,"Connecting HTTPS with cert authentication")
                self.conn=client_connection.ClientConnectionRequireCert(self.conf)
            else:
                log.log(cpc.util.log.TRACE,"Connecting HTTPS with no cert authentication")
                self.conn=client_connection.ClientConnectionNoCertRequired(
                            self.conf, disable_cookies)
            self.conn.connect(self.host,self.port)
        except httplib.HTTPException as e:
            raise ClientConnectionError(e,self.host,self.port)
        except socket.error as e:
            raise ClientConnectionError(e,self.host,self.port)
    def initialize(self,endNodeId):

        topology=self.getNetworkTopology()

        if not topology:
            log.error("Cannot get network topology")
            return

        # this is myself:
        startNode = Node.getSelfNode(self.conf)

        self.endNode = topology.nodes.get(endNodeId)

        log.log(cpc.util.log.TRACE,"Finding route between %s(%s %s) and %s(%s "
                                   "%s"")"%(startNode.server_id,startNode.getHostname(),
                                    startNode.getServerSecurePort(),
                                    self.endNode.server_id,
                                    self.endNode.getHostname(),self.endNode.getServerSecurePort()))
        route = Nodes.findRoute(startNode, self.endNode,topology)

        self.hostNode = route[1]   #route[0] is the current host
        self.host = self.hostNode.getHostname()
        self.port = self.hostNode.getServerSecurePort()
        self.serverId = self.hostNode.getId()
        log.log(cpc.util.log.TRACE,"Server-to-server connecting to %s(%s:%s)"%
                (self.serverId,self.host,self.port))
 def putConnection(self,connection,host,port):
     #create a queue in case we dont have one
     self.getOrCreateQueue(host,port)
     key = self.getKey(host, port)
     with self.listlock:
         self.pool[key].put(connection,False)
         log.log(cpc.util.log.TRACE,"put back connection in pool for host:%s:%s"%(host,port))
示例#4
0
 def getCommandOutputData(cmdID, workerServer):
     log.log(cpc.util.log.TRACE,"Trying to pull command output from %s"%
             workerServer)
     s2smsg=ServerMessage(workerServer)  
     rundata_response = s2smsg.pullAssetRequest(cmdID, Asset.cmdOutput())
     
     if rundata_response.getType() != "application/x-tar":
         log.error("Incorrect response type: %s, should be %s"%
                   (rundata_response.getType(), 'application/x-tar'))
         if rundata_response.getType() == "text/json":
             errormsg=rundata_response.message.read(len(rundata_response.
                                                        message))
             presp=ProcessedResponse(rundata_response)
             if not presp.isOK():
                 log.error('Response from worker server not OK: %s'%
                           errormsg)
     else:
         s2smsg.clearAssetRequest(cmdID)
         log.log(cpc.util.log.TRACE,
                 "Successfully pulled command output data from %s."%
                 workerServer)
         return rundata_response
         #runfile = rundata_response.getRawData()
         #this doesnt work because the mmap closes as it is returned
     return None
    def __init__(self):
        self.__dict__ = self.__shared_state

        if len(self.__shared_state)>0:
            return

        ConnectionPool.__init__(self)
        log.log(cpc.util.log.TRACE,"instantiation of Server connection pool")
示例#6
0
    def __init__(self):
        self.__dict__ = self.__shared_state
        
        if len(self.__shared_state)>0:
            return

        self.cacheLock = threading.Lock()
        log.log(cpc.util.log.TRACE,"instantiation of cache")
        self.cache = {}
示例#7
0
    def connect(self,host,port):

        self.host = host
        self.port = port

        log.log(cpc.util.log.TRACE,"Connecting HTTPS with no cert req to host %s, port %s"%(
            self.host,self.port))
        self.conn = HttpsConnectionNoCertReq(self.host,self.port)
        self.conn.connect()
        self.connected=True
    def getOrCreateQueue(self,host,port):
        key = self.getKey(host, port)
        if key not in self.pool:
            log.log(cpc.util.log.TRACE,"Instantiating server https "
                                       "connection pool for host:%s:%s"%(host,port))
            q =  Queue()
            self.pool[key] = q
        else:
            q= self.pool[key]

        return q
    def connect(self, host, port):

        self.host = host
        self.port = port

        log.log(
            cpc.util.log.TRACE,
            "Connecting HTTPS with no cert req to host %s, port %s" %
            (self.host, self.port))
        self.conn = HttpsConnectionNoCertReq(self.host, self.port)
        self.conn.connect()
        self.connected = True
示例#10
0
    def add(self,key,value,ttl=None):
        '''
        ttl : time to live in seconds, if set to none the cache lifetime is infinite
        '''
        with self.cacheLock:
            if ttl:
                log.log(cpc.util.log.TRACE,'adding object %s to cache with ttl %s'%(key,ttl))
            else:
                log.log(cpc.util.log.TRACE,'adding object %s to cache'%key)

            if(ttl):
                ttl = int(time.time())+ttl
            self.cache[key] = (value,ttl)
示例#11
0
    def handleSinglePart(headers,message):
        contentLength = long(headers['content-length'])
        if headers['content-type'] == 'application/x-www-form-urlencoded' or headers['content-type'] == 'application/x-www-form-urlencoded; charset=UTF-8': #TODO generalize
            msg =  message.read(contentLength)
            log.log(cpc.util.log.TRACE,'RAW msg is %s'%msg)
            parsedDict = urlparse.parse_qs(msg)  #Note values here are stored in lists, this is so one can handle many inputs with same name, for now we dont want that as our multipart parsing does not support it
            params = dict()
            for k,v in parsedDict.iteritems():
                params[k] = v[0]

            request = ServerRequest(headers,None,params)

        return request
示例#12
0
    def handleSinglePart(headers,message):
        contentLength = long(headers['content-length'])
        if headers['content-type'] == 'application/x-www-form-urlencoded' or headers['content-type'] == 'application/x-www-form-urlencoded; charset=UTF-8': #TODO generalize
            msg =  message.read(contentLength)
            parsedDict = urlparse.parse_qs(msg)  #Note values here are stored in lists, this is so one can handle many inputs with same name, for now we dont want that as our multipart parsing does not support it
            params = dict()
            for k,v in parsedDict.iteritems():
                params[k] = v[0]

            log.log(cpc.util.log.TRACE,'msg is %s'%params)
            request = ServerRequest(headers,None,params)

        return request
示例#13
0
    def connect(self, host, port):

        self.host = host
        self.port = port
        privateKey = self.conf.getPrivateKey()
        keyChain = self.conf.getCaChainFile()
        cert = self.conf.getCertFile()

        log.log(
            cpc.util.log.TRACE,
            "Connecting VerHTTPS to host %s, port %s" % (self.host, self.port))
        self.conn = HttpsConnectionWithCertReq(self.host, self.port,
                                               privateKey, keyChain, cert)
        self.conn.connect()
        self.connected = True
示例#14
0
 def run(self, serverState, request, response):
     cmdID=request.getParam('cmd_id')
     assetType=request.getParam('asset_type')
     try:
         runfile=serverState.getLocalAssets().getAsset(cmdID, 
                                                       assetType).getData()
     except:
         log.error("Local asset cmdid=%s not found!"%cmdID)
         response.add("Command output data from cmdID %s not found on this server (%s)."%
                      (cmdID,serverState.conf.getHostName()), 
                      status="ERROR")
     else:
         asset=serverState.getLocalAssets().getCmdOutputAsset(cmdID)
         log.log(cpc.util.log.TRACE,"Local asset cmdid=%s \nproject server=%s"%
                                    (asset.cmdID, asset.projectServer))
         response.setFile(runfile,'application/x-tar')
     log.info("Pulled asset %s/%s"%(cmdID, assetType))
示例#15
0
    def connect(self,host,port):

        self.host = host
        self.port = port
        privateKey = self.conf.getPrivateKey()
        keyChain = self.conf.getCaChainFile()
        cert = self.conf.getCertFile()

        log.log(cpc.util.log.TRACE,"Connecting VerHTTPS to host %s, port %s"%(
            self.host,self.port))
        self.conn = HttpsConnectionWithCertReq(self.host,
                                            self.port,
                                            privateKey,
                                            keyChain,
                                            cert)
        self.conn.connect()
        self.connected=True
    def getAllConnections(self,host,port):
        connections = []
        #check if we have a queue instantiated for that host
        with self.listlock:
            q = self.getOrCreateQueue(host,port)

            #we try yo get all available connections from the pool
            while not q.empty():
                connections.append(q.get())

            if len(connections)==0:
                log.log(cpc.util.log.TRACE,"No connections available in pool for "
                                       "host:%s:%s"%(host,port))
                raise ConnectionPoolEmptyError("No connections available in pool for "
                                           "host:%s:%s"%(host,port))

            return connections
示例#17
0
 def run(self, serverState, request, response):
     workerID=request.getParam('worker_id')
     workerDir=request.getParam('worker_dir')
     iteration=request.getParam('iteration')
     itemsXML=request.getParam('heartbeat_items')
     log.log(cpc.util.log.TRACE, 'items: %s'%itemsXML)
     hwr=cpc.command.heartbeat.HeartbeatItemReader()
     hwr.readString(itemsXML, "worker heartbeat items")
     faultyItems=[]
     Nhandled=len(hwr.getItems())
     ret=serverState.getRunningCmdList().ping(workerID, workerDir, iteration,
                                              hwr.getItems(), False,
                                              faultyItems)
     if len(faultyItems)==0:
         response.add('', data=serverState.conf.getHeartbeatTime())
     else:
         response.add('Heatbeat NOT OK', status="ERROR", data=faultyItems)
     log.info("Handled %d forwarded heartbeat signal items."%(Nhandled))
    def getConnection(self,host,port):

        #check if we have a queue instantiated for that host
        with self.listlock:
            q = self.getOrCreateQueue(host,port)
        try:
            #we wait to until we get hold of a connection,
            # if we do not get a connection after timeout then all available
            # connections must have died.
            connection = q.get(block=True,timeout=15)
            log.log(cpc.util.log.TRACE,"Got a connection from pool for "
                                       "host:%s:%s"%(host,port))

        except Empty as e:
            log.log(cpc.util.log.TRACE,"No connections available in pool for "
                                       "host:%s:%s"%(host,port))
            raise ConnectionPoolEmptyError(e)

        return connection
示例#19
0
 def run(self, serverState, request, response):
     cmdID = request.getParam('cmd_id')
     assetType = request.getParam('asset_type')
     try:
         runfile = serverState.getLocalAssets().getAsset(
             cmdID, assetType).getData()
     except:
         log.error("Local asset cmdid=%s not found!" % cmdID)
         response.add(
             "Command output data from cmdID %s not found on this server (%s)."
             % (cmdID, serverState.conf.getHostName()),
             status="ERROR")
     else:
         asset = serverState.getLocalAssets().getCmdOutputAsset(cmdID)
         log.log(
             cpc.util.log.TRACE,
             "Local asset cmdid=%s \nproject server=%s" %
             (asset.cmdID, asset.projectServer))
         response.setFile(runfile, 'application/x-tar')
     log.info("Pulled asset %s/%s" % (cmdID, assetType))
示例#20
0
def matchCommandWorker(matcher, command):
    """Function to use in queue.getUntil() to get a number of commands from
       the queue.
       TODO: this is where performance tuning results should be used."""
    cont = True  # whether to continue getting commands from the queue
    # whether to use this command: make sure we only have a single type
    use = False
    execID = matcher.getExecID(command)
    log.log(cpc.util.log.TRACE, "exec id is %s" % execID)
    log.debug("Type: %s" % command.getTask().getFunctionName())
    if execID is not None:
        use = matcher.checkType(command.getTask().getFunctionName()) and matcher.checkWorkerRequirements(command)

    log.debug("Should use: %s " % use)
    if use:
        if matcher.checkAddResources(command):
            use = True
        else:
            use = False
            cont = False
    return (cont, use)
示例#21
0
def matchCommandWorker(matcher, command):
    """Function to use in queue.getUntil() to get a number of commands from
       the queue.
       TODO: this is where performance tuning results should be used."""
    cont=True # whether to continue getting commands from the queue
    # whether to use this command: make sure we only have a single type
    use=False
    execID=matcher.getExecID(command)
    log.log(cpc.util.log.TRACE,'exec id is %s'%execID)
    log.debug("Type: %s"%command.getTask().getFunctionName())
    if execID is not None:
        use=(matcher.checkType(command.getTask().getFunctionName()) and
             matcher.checkWorkerRequirements(command))

    log.debug("Should use: %s "%use)
    if use:
        if matcher.checkAddResources(command):
            use=True
        else:
            use=False
            cont=False
    return (cont, use)
示例#22
0
 def get(self,key):
     """
     returns: the cacheobject if it exists, False otherwise
     """
     with self.cacheLock:
         val = False
         if key in self.cache:
             now = int(time.time())
             val,ttl = self.cache[key]
             if ttl<now:
                 val = False
             else:
                 log.log(cpc.util.log.TRACE,'object %s in cache has expired'%key)
         elif val == False:
             log.log(cpc.util.log.TRACE,'did not find object %s in cache'%key)
         else:
             log.log(cpc.util.log.TRACE,'getting object %s from cache'%key)
         return val
示例#23
0
    def run(self, serverState, request, response):
        # first read platform capabilities and executables
        rdr=cpc.command.platform_exec_reader.PlatformExecutableReader()
        workerData=request.getParam('worker')
        if request.hasParam('worker-id'):
            workerID=request.getParam('worker-id')
        else:
            workerID='(none)'
        log.debug("Worker platform + executables: %s"%workerData)
        rdr.readString(workerData,"Worker-reported platform + executables")
        # match queued commands to executables.
        cwm=CommandWorkerMatcher(rdr.getPlatforms(),
                                 rdr.getExecutableList(),
                                 rdr.getWorkerRequirements())
        cmds=cwm.getWork(serverState.getCmdQueue())
        if not cwm.isDepleted():
            # now sleep for 5 seconds to give the dataflow time to react to any
            # new state.
            time.sleep(5)
            cmds.extend(cwm.getWork(serverState.getCmdQueue()))
        # now check the forwarded variables
        conf=serverState.conf
        originatingServer=None
        heartbeatInterval=None
        try:
            # check whether there is an originating server. If not, we're it
            if self.forwarded:
                if 'originating-server-id' in request.headers:
                    originatingServer = request.headers['originating-server-id']
                # check the expected heartbeat time.
                log.debug("Forwarded message")
                if request.hasParam('heartbeat-interval'):
                    heartbeatInterval = int(request.getParam('heartbeat-interval'))
                    log.debug("Forwarded heartbeat interval is %d"%
                            heartbeatInterval)
        except NameError:
            # self.forwarded does not exist. Treat it as if self.forwarded == False
            pass

        if originatingServer is None:
            # If the originating server property has not been set,  the
            # request hasn't been forwarded, therefore we are the originating
            # server
            selfNode=Node.getSelfNode(conf)
            originatingServer = selfNode.getId()
            # we only store worker state in the server the worker connects to
            serverState.setWorkerState(WorkerStatus.WORKER_STATUS_CONNECTED,workerID,
                                       request.headers['originating-client'])
        if heartbeatInterval is None:
            heartbeatInterval = conf.getHeartbeatTime()
        log.debug("worker identified %s"%request.headers['originating-client'] )

        if len(cmds) > 0:
            # first add them to the running list so they never get lost
            runningCmdList=serverState.getRunningCmdList()
            runningCmdList.add(cmds, originatingServer, heartbeatInterval)
            # construct the tar file with the workloads.
            tff=tempfile.TemporaryFile()
            tf=tarfile.open(fileobj=tff, mode="w:gz")
            # make the commands ready
            for cmd in cmds:
                log.debug("Adding command id %s to tar file."%cmd.id)
                # write the command description to the command's directory
                task=cmd.getTask()
                #log.debug(cmd)
                project=task.getProject()
                taskDir = "task_%s"%task.getID()
                cmddir=cmd.getDir()
                if not os.path.exists(cmddir):
                    log.debug("cmddir %s did not exist. Created directory."%cmd.id)
                    os.mkdir(cmddir)
                arcdir="%s"%(cmd.id)
                log.debug("cmddir=%s"%cmddir)
                outf=open(os.path.join(cmddir, "command.xml"), "w")
                cmd.writeWorkerXML(outf)
                outf.close()
                tf.add(cmddir, arcname=arcdir, recursive=True)
                # set the state of the command.
            tf.close()
            del(tf)
            tff.seek(0)
            # now send it back
            response.setFile(tff,'application/x-tar')
            #project.writeTasks()
            # the file is closed after the response is sent.
            log.info("Did direct worker-ready")
        else:
            nodes = conf.getNodes().getNodesByPriority()

            topology = Nodes()
            if request.hasParam('topology'):
                topology = json.loads(request.getParam('topology')
                                      ,object_hook = json_serializer.fromJson)

            thisNode = Node.getSelfNode(conf)
            thisNode.nodes = conf.getNodes()
            topology.addNode(thisNode)

            hasJob =False # temporary flag that should be removed
            for node in nodes:
                if topology.exists(node.getId()) == False:
                    clnt=ServerMessage(node.getId())

                    clientResponse=clnt.workerReadyForwardedRequest(workerID,
                                        workerData,
                                        topology,
                                        originatingServer,
                                        heartbeatInterval,
                                        request.headers['originating-client'])

                    if clientResponse.getType() == 'application/x-tar':

                        log.log(cpc.util.log.TRACE,
                                'got work from %s'%
                                (clientResponse.headers[
                                     'originating-server-id']))
                        hasJob=True
                        # we need to rewrap the message

                        #TODO stupid intermediary step because the mmap form
                        # clientresponse is prematurely closed
                        tmp = tempfile.TemporaryFile('w+b')

                        message = clientResponse.getRawData()

                        tmp.write(message.read(len(message)))
                        tmp.seek(0)

                        #for key in clientResponse.headers:
                        #    print "%s:%s"%(key,clientResponse.headers[key])

                        response.setFile(tmp,'application/x-tar')
                        response.headers['originating-server-id']=\
                                  clientResponse.headers[
                                      'originating-server-id']
                    #OPTIMIZE leads to a lot of folding and unfolding of
                    #packages
            if not hasJob:
                response.add("No command")
            log.info("Did delegated worker-ready")
 def __init__(self):
     log.log(cpc.util.log.TRACE,"instantiation of connectionPool")
     self.pool = {}
     self.listlock=threading.Lock()
示例#25
0
    def handleMultipart(mainHeaders,msgStream):
        files = dict()
        params = dict()

        BOUNDARY = "--"+HttpMethodParser.extractBoundary(mainHeaders)        
        stopBoundary = BOUNDARY+"--"
        terminateBoundary = ''
        
        msgStream.readline() #has an empty line at start that we want to get rid of
          
        while terminateBoundary != stopBoundary:
            headers = mimetools.Message(msgStream)
            
            terminateBoundary = ''
            log.log(cpc.util.log.TRACE,'multipart headers are %s'%headers.headers)
            
            if(ServerRequest.isFile(headers['Content-Disposition'])):
                file = tempfile.TemporaryFile(mode="w+b")
            
            name =  ServerRequest.getFieldName(headers['Content-Disposition'])            
            notused,contentDispositionParams = cgi.parse_header(headers['Content-Disposition'])                        
            name = contentDispositionParams['name']
             
            
            #if we have a content length we just read it and store the data
            
            contentLength = headers.getheader('Content-Length')
            if(contentLength):   # If a content length is sent we parse the nice way
                bytes = int(contentLength)
                if(ServerRequest.isFile(headers['Content-Disposition'])):
                    file.write(msgStream.read(bytes))
                    
                else: 
                    line  = msgStream.read(bytes)
                    log.log(cpc.util.log.TRACE,"line is "+line)
                    params[name] = line  
                    
                msgStream.readline()    ## we will have a trailin CRLF that we just want to get rid of
            
            if(ServerRequest.isFile(headers['Content-Disposition'])):
                readBytes = 0
                while(True):
                    line = msgStream.readline()                    
                    if re.search(BOUNDARY,line):
                        #time to wrap it up
                        
                        if(line[-2:] == '\r\n'):                          
                            line = line[:-2]
                        elif(line[-1:] == '\n'):                     
                            line = line[:-1]
                        
                        terminateBoundary = line                                               
                        
                        file.seek(0)
                        skipBytes = 2

                        realFile = tempfile.TemporaryFile(mode="w+b")
                        realFile.write(file.read(readBytes-skipBytes))
                        
                        file.close()                        
                        realFile.seek(0)
                        
                        #For testing during dev only!!
                        #runTest(realFile)                                                      
                        
                        files[name]= realFile
                        break
                    else:
                        readBytes +=len(line)
                        file.write(line)   
                
            else:  
                while(True):
                    line = msgStream.readline()

                    if(line[-2:] == '\r\n'):                      
                        line = line[:-2]
                    elif(line[-1:] == '\n'):                     
                        line = line[:-1]
                                            
                    if re.search(BOUNDARY,line):       
                        terminateBoundary = line   
                        break;                                 
                    
                    else:
                        if name in params:
                            params[name]+= line
                        else: 
                            params[name] = line    
            
        return ServerRequest(mainHeaders,None,params,files)
示例#26
0
 def remove(self,key):
     with self.cacheLock:
         if key in self.cache:
             log.log(cpc.util.log.TRACE,'removing object %s from cache'%key)
             del(self.cache[key])
示例#27
0
 def cleanAll(self):
     with self.cacheLock:
         self.cache={}
         log.log(cpc.util.log.TRACE,'cleaning all objects from cache')
示例#28
0
    def handleMultipart(mainHeaders,msgStream):
        files = dict()
        params = dict()

        BOUNDARY = "--"+HttpMethodParser.extractBoundary(mainHeaders)        
        stopBoundary = BOUNDARY+"--"
        terminateBoundary = ''
        
        msgStream.readline() #has an empty line at start that we want to get rid of
          
        while terminateBoundary != stopBoundary:
            headers = mimetools.Message(msgStream)
            
            terminateBoundary = ''
            log.log(cpc.util.log.TRACE,'multipart headers are %s'%headers.headers)
            
            if(ServerRequest.isFile(headers['Content-Disposition'])):
                file = tempfile.TemporaryFile(mode="w+b")
            
            name =  ServerRequest.getFieldName(headers['Content-Disposition'])            
            notused,contentDispositionParams = cgi.parse_header(headers['Content-Disposition'])                        
            name = contentDispositionParams['name']
             
            
            #if we have a content length we just read it and store the data
            
            contentLength = headers.getheader('Content-Length')
            if(contentLength):   # If a content length is sent we parse the nice way
                bytes = int(contentLength)
                if(ServerRequest.isFile(headers['Content-Disposition'])):
                    file.write(msgStream.read(bytes))
                    
                else: 
                    line  = msgStream.read(bytes)
                    log.log(cpc.util.log.TRACE,"line is "+line)
                    params[name] = line  
                    
                msgStream.readline()    ## we will have a trailin CRLF that we just want to get rid of
            
            if(ServerRequest.isFile(headers['Content-Disposition'])):
                readBytes = 0
                while(True):
                    line = msgStream.readline()                    
                    if re.search(BOUNDARY,line):
                        #time to wrap it up
                        
                        if(line[-2:] == '\r\n'):                          
                            line = line[:-2]
                        elif(line[-1:] == '\n'):                     
                            line = line[:-1]
                        
                        terminateBoundary = line                                               
                        
                        file.seek(0)
                        skipBytes = 2

                        realFile = tempfile.TemporaryFile(mode="w+b")
                        realFile.write(file.read(readBytes-skipBytes))
                        
                        file.close()                        
                        realFile.seek(0)
                        
                        #For testing during dev only!!
                        #runTest(realFile)                                                      
                        
                        files[name]= realFile
                        break
                    else:
                        readBytes +=len(line)
                        file.write(line)   
                
            else:  
                while(True):
                    line = msgStream.readline()

                    if(line[-2:] == '\r\n'):                      
                        line = line[:-2]
                    elif(line[-1:] == '\n'):                     
                        line = line[:-1]
                                            
                    if re.search(BOUNDARY,line):       
                        terminateBoundary = line   
                        break;                                 
                    
                    else:
                        if name in params:
                            params[name]+= line
                        else: 
                            params[name] = line    
            
        return ServerRequest(mainHeaders,None,params,files)