def __init__(self, endNodeId): #TODO perhaps node should be input node = ServerConf().getNodes().get(endNodeId) ServerConnection.__init__(self,node,ServerConf()) self.initialize(endNodeId) """Connect to a server opening a connection"""
def _handleSession(self, request): handler_base._handleSession(self, request) if 'user' not in request.session \ and not ServerConf().getServerVerification() \ and self.headers.has_key('originating-server-id'): if ServerConf().getNodes().exists( self.headers['originating-server-id']): request.session['user'] = User(1, 'root', UserLevel.SUPERUSER)
def testStart2Servers(self): numServers = 2 self.createConfFolders(numServers) hostname = gethostname() node0HttpsPort = 13807 node1HttpsPort = 13808 node0HttpPort = 14807 node1HttpPort = 14808 for i in range(numServers): args = [ '../../../../cpc-server', '-c', self.serverConfs[i], 'start' ] #doing cpc.server.server.forkAndRun(cf, debug ) directly here will will for some strange reason mess up things when shutting down, the process wont shutdown subprocess.call(args) time.sleep(2) #connect node 0 to node 1 args = [ '../../../../cpc-server', '-c', self.serverConfs[0], 'connnect-server', hostname, str(node1HttpPort) ] #doing cpc.server.server.forkAndRun(cf, debug ) directly here will will for some strange reason mess up things when shutting down, the process wont shutdown subprocess.call(args) args = [ '../../../../cpc-server', '-c', self.serverConfs[1], 'trust', hostname, str(node0HttpsPort) ] #doing cpc.server.server.forkAndRun(cf, debug ) directly here will will for some strange reason mess up things when shutting down, the process wont shutdown subprocess.call(args) #verify existense of of nodes in each conf file conf1 = ServerConf(confdir=self.serverConfs[0], reload=True) node0Nodes = conf1.getNodes() self.assertTrue(node0Nodes.exists(hostname, node1HttpsPort)) conf2 = ServerConf(confdir=self.serverConfs[1], reload=True) node1Nodes = conf2.getNodes() self.assertTrue(node1Nodes.exists(hostname, node0HttpsPort)) #do a network topology call conf = ConnectionBundle(confdir=self.serverConfs[0], reload=True) client = ClientMessage() topology = ProcessedResponse(client.networkTopology()).getData() self.assertEquals(topology.size(), 2)
def readDir(self, bindir, platforms): """Read a directory (usually in the search path) for all executables. bindir = the directory name. platforms = the list of availble platforms.""" reader = ExecutableReader(bindir) try: files = os.listdir(bindir) except OSError: files = [] for file in files: try: basedir = os.path.join(bindir, file) log.debug("basedir is %s" % basedir) pfile = os.path.join(basedir, "plugin") log.debug("pfile is %s" % pfile) nfile = os.path.join(basedir, "executable.xml") log.debug("exec xml is %s" % nfile) pl = None # check whether this is in fact a plugin if (not os.path.isdir(basedir)) and os.access( basedir, os.X_OK): plf = basedir pl = cpc.util.plugin.ExecutablePlugin(basedir, conf=ServerConf()) # or it contains a plugin elif (not os.path.isdir(pfile)) and os.access(pfile, os.X_OK): plf = basedir pl = cpc.util.plugin.ExecutablePlugin(pfile, conf=ServerConf()) if pl is not None: # and run the plugin if it is one for platform in platforms: (retcode, retst) = pl.run(plf, platform.getName()) log.debug("returned: %s" % retst) if retcode == 0: reader.readString(retst, "executable plugin output", basedir) elif os.path.exists(nfile): try: # otherwise just read the executable xml reader.read(nfile, basedir) except IOError: pass except IOError: pass self.executables.extend(reader.getExecutables())
def testInit(self): #init the server conf conf = ServerConf(confdir=self.confDir) self.assertEquals(conf.get('conf_dir'), self.confDir) self.assertEquals(self.confFile, conf.get('conf_file')) conf.getServerKeyDir( ) #just to make sure it has been initiated the proper way conf.setServerHost("testhost") conf2 = ServerConf( confdir=self.confDir ) #creating a new instance reinitiates the config and reads parameters from config file self.assertEquals('testhost', conf2.get('server_host'))
def do_GET(self): self.log.log(cpc.util.log.TRACE, '%s %s' % (self.command, self.path)) # if the path starts with application root + / or ? we have a message to process #otherwise we should just strip any request params and keep the resource reference if (self.isApplicationRoot()): request = HttpMethodParser.parseGET(self.headers.dict, self.path) self.processMessage(request) #take the input and put it into a request object else: if self.path == "/": self.path += "index.html" else: self.path = self.path.split('?', 1)[0] #strip trailing '?...' webDir = ServerConf().getWebRootPath() resourcePath = webDir + self.path if not os.path.isfile(resourcePath): self.responseCode = 404 resourcePath = webDir + '/404.html' response = ServerResponse() file = open(resourcePath, 'rb') response.setFile(file, mimetypes.guess_type(resourcePath)) self._sendResponse(response)
def establishInboundConnection(node, serverState): conf = ServerConf() for i in range(0, conf.getNumPersistentConnections()): try: message = PersistentServerMessage(node, conf) socket = message.persistIncomingConnection() serverState.addReadableSocket(socket) node.addInboundConnection() except ServerConnectionError as e: #The node is not reachable at this moment, # no need to throw an exception since we are marking the node # as unreachable ins ServerConnectionHandler log.log( cpc.util.log.TRACE, "Exception when establishing " "inbound connections: %s " % e) break if node.isConnectedInbound(): log.log( cpc.util.log.TRACE, "Established inbound " "connections to server " "%s" % node.toString()) else: log.log( cpc.util.log.TRACE, "Could not establish inbound " "connections to server " "%s" % node.toString())
def establishOutboundConnection(node): conf = ServerConf() for i in range(0, conf.getNumPersistentConnections()): try: #This will make a regular call, the connection pool will take # care of the rest message = PersistentServerMessage(node, conf) resp = message.persistOutgoingConnection() node.addOutboundConnection() except ServerConnectionError as e: #The node is not reachable at this moment, # no need to throw an exception since we are marking the node # as unreachable in ServerConnection log.log( cpc.util.log.TRACE, "Exception when establishing " "outgoing connections: %s " % e) break if node.isConnectedOutbound(): log.log( cpc.util.log.TRACE, "Established outgoing " "connections to server " "%s" % node.toString()) else: log.log( cpc.util.log.TRACE, "Could not establish outgoing " "connections to %s" % node.toString())
def saveProject(self, project): self.taskExecThreads.acquire() conf = ServerConf() try: self.taskExecThreads.pause() self._write() projectFolder = "%s/%s" % (conf.getRunDir(), project) if (os.path.isdir(projectFolder)): #tar the project folder but keep the old files also, this is # only a backup!!! #copy _state.xml to _state.bak.xml stateBackupFile = "%s/_state.bak.xml" % projectFolder shutil.copyfile("%s/_state.xml" % projectFolder, stateBackupFile) tff = tempfile.TemporaryFile() tf = tarfile.open(fileobj=tff, mode="w:gz") tf.add(projectFolder, arcname=".", recursive=True) tf.close() del (tf) tff.seek(0) os.remove(stateBackupFile) self.taskExecThreads.cont() else: self.taskExecThreads.cont() raise Exception("Project does not exist") finally: self.taskExecThreads.release() return tff
def __persistConnection(self, direction, headers=dict()): headers['persistent-connection'] = direction #message body is actually irrellevant and is not read on the other # side. #we just need to conform to the http protocol fields = [] fields.append(Input('cmd', "persist-connection")) #sending along the connection parameters for this server conf = ServerConf() connectionParams = dict() connectionParams['serverId'] = conf.getServerId() connectionParams['hostname'] = conf.getHostName() connectionParams['fqdn'] = conf.getFqdn() connectionParams['client_secure_port'] = conf\ .getClientSecurePort() connectionParams['server_secure_port'] = conf\ .getServerSecurePort() input2 = Input( 'connectionParams', json.dumps(connectionParams, default=json_serializer.toJson, indent=4)) # a json structure that needs to be dumped fields.append(input2) response = self.putRequest( ServerRequest.prepareRequest(fields, [], headers)) return response
def sendAddNodeRequest(self, host): """ """ conf = ServerConf() cmdstring = 'connect-server-request' fields = [] input = Input('cmd', cmdstring) inf = open(conf.getCACertFile(), "r") key = inf.read() nodeConnectRequest = NodeConnectRequest(conf.getServerId(), conf.getClientSecurePort(), conf.getServerSecurePort(), key, conf.getFqdn(), conf.getHostName()) input2 = Input( 'nodeConnectRequest', json.dumps(nodeConnectRequest, default=json_serializer.toJson, indent=4)) input3 = Input('unqalifiedDomainName', host) fields.append(input) fields.append(input2) fields.append(input3) fields.append(Input('version', "1")) # this goes over the client Secure Port, and we don't want the server to use # cookies response = self.postRequest(ServerRequest.prepareRequest(fields), require_certificate_authentication=False, disable_cookies=True) return response
def readProjectState(self, projectName): prj = self.projects[projectName] conf = ServerConf() projectBaseDir = "%s/%s" % (conf.getRunDir(), projectName) #We have a couple of hardcoded paths that might not be valid anymore # Think of the case when we are moving projects to another server # here we replace those old paths with new valid paths stateBakXML = "%s/%s" % (projectBaseDir, "_state.bak.xml") #/get the state_bak.xml file = open(stateBakXML, 'r') content = file.read() file.close() #find first occurence of <env ..... base_dir=<BASE_DIR> m = re.search('<env .* base_dir="(.*)".*>', content) if m != None: #only if we have a project with active tasks oldBaseDir = m.group(1) new_content = re.sub(oldBaseDir, projectBaseDir, content) file = open(stateBakXML, "w") file.write(new_content) file.close() #reread project state prj.readState(stateFile="_state.bak.xml")
def run(self, serverState, request, response): conf = ServerConf() sentConnectRequests = conf.getSentNodeConnectRequests() node = json.loads(request.getParam('connectRequest'), object_hook=json_serializer.fromJson) if (sentConnectRequests.exists(node.getId())): nodeToAdd = sentConnectRequests.get(node.getId()) conf.addNode( Node(node.server_id, node.getClientSecurePort(), node.getServerSecurePort(), node.getQualifiedName(), nodeToAdd.getHostname())) #conf.addNode(nodeToAdd) openssl = OpenSSL(conf) openssl.addCa(node.key) sentConnectRequests.removeNode(node.getId()) conf.set('sent_node_connect_requests', sentConnectRequests) # need to send back a status in the data notifying ok response.add('Connection to node %s established' % node.toString()) log.info("Node connection accepted") #add it to the node list else: response.add('No previous node request sent for host %s' % node.toString()) log.info("Node connection not accepted")
def run(self, serverState, request, response): #get the connection params for this node newParams = json.loads(request.getParam("connectionParams")) conf = ServerConf() nodes = conf.getNodes() if nodes.exists(newParams['serverId']): node = nodes.get(newParams['serverId']) node.setHostname(newParams['hostname']) node.setServerSecurePort(newParams['server_secure_port']) node.setClientSecurePort(newParams['client_secure_port']) node.setQualifiedName(newParams['fqdn']) #Needed so that we write changes to conf file conf.removeNode(node.server_id) conf.addNode(node) #update the network topology ServerToServerMessage.getNetworkTopology(resetCache=True) response.add("Updated connection parameters") log.info("Updated connection params for %s" % node.toString()) else: response.add("Requested update for node %s but this node " \ "is not a neigbouring node "%newParams[ 'serverId'], status="ERROR") log.error("Failed updating connection params for %s" % newParams['serverId'])
def grant(key): #key is server-id conf = ServerConf() nodeConnectRequests = conf.getNodeConnectRequests() if nodeConnectRequests.exists(key): nodeToAdd = nodeConnectRequests.get( key) #this returns a nodeConnectRequest object serv = RawServerMessage(nodeToAdd.getHostname(), nodeToAdd.getClientSecurePort()) #letting the requesting node know that it is accepted #also sending this servers connection parameters resp = serv.addNodeAccepted() conf.addNode( Node(nodeToAdd.server_id, nodeToAdd.getClientSecurePort(), nodeToAdd.getServerSecurePort(), nodeToAdd.getQualifiedName(), nodeToAdd.getHostname())) #trust the key openssl = OpenSSL(conf) openssl.addCa(nodeToAdd.key) nodeConnectRequests.removeNode(nodeToAdd.getId()) conf.set('node_connect_requests', nodeConnectRequests) return True else: return False
def run(self, serverState, request, response): conf = ServerConf() host = request.getParam('host') client_secure_port = request.getParam('client_secure_port') result = dict() #do we have a server with this hostname or fqdn? connectedNodes = conf.getNodes() if (connectedNodes.hostnameOrFQDNExists(host) == False): serv = RawServerMessage(host, client_secure_port) resp = ProcessedResponse(serv.sendAddNodeRequest(host)) if resp.isOK(): result = resp.getData() nodeConnectRequest = NodeConnectRequest( result['serverId'], int(client_secure_port), None, None, result['fqdn'], host) conf.addSentNodeConnectRequest(nodeConnectRequest) result['nodeConnectRequest'] = nodeConnectRequest log.info("Added node %s" % host) response.add('', result) else: response.add("Remote server said: %s" % resp.getMessage(), status="ERROR") else: errorMessage = "%s is already trusted" % host response.add(errorMessage, status="ERROR") log.info(errorMessage)
def run(self, serverState, request, response): projectName = request.getParam("project") user = self.getUser(request) if (request.haveFile("projectFile")): projectBundle = request.getFile('projectFile') try: serverState.getProjectList().add(projectName) UserHandler().addUserToProject(user, projectName) extractPath = "%s/%s" % (ServerConf().getRunDir(), projectName) tar = tarfile.open(fileobj=projectBundle, mode="r") tar.extractall(path=extractPath) tar.close() del (tar) serverState.readProjectState(projectName) gc.collect() except: response.add("No project file provided", status="ERROR") return response.add("Project restored as %s" % projectName) log.info("Project load %s" % (projectName)) else: response.add("No project file provided", status="ERROR") log.info("Project load %s failed" % (projectName))
def testBroadCastMessage(self): self.create5NodeNetwork() #this way we initialize the configs for a specific server which lets us test method calls as as specific server clientConf = ConnectionBundle(confdir=self.serverConfs[0], reload=True) serverConf = ServerConf(confdir=self.serverConfs[0], reload=True) broadCastMessage = BroadcastMessage() #logs should mention that we are setting elements to cache broadCastMessage.updateNetworkTopology() clientConf = ConnectionBundle(confdir=self.serverConfs[1], reload=True) serverConf = ServerConf(confdir=self.serverConfs[1], reload=True) #this should be brought from the cache(check in the logs) ServerToServerMessage.getNetworkTopology()
def addNodeAccepted(self): conf = ServerConf() inf = open(conf.getCACertFile(), "r") key = inf.read() #only sending fqdn the requesting should already know the unqualified # hostname node = NodeConnectRequest(conf.getServerId(), conf.getClientSecurePort(), conf.getServerSecurePort(), key, conf.getFqdn(), None) cmdstring = 'node-connection-accepted' fields = [] input = Input('cmd', cmdstring) input2 = Input( 'connectRequest', json.dumps(node, default=json_serializer.toJson, indent=4)) fields.append(input) fields.append(input2) fields.append(Input('version', "1")) # this goes over client secure port , and we don't want the server to use # cookies response = self.postRequest(ServerRequest.prepareRequest(fields), require_certificate_authentication=False, disable_cookies=True) return response
def test_createConf(self): self.createConfFolder(0) conf = ServerConf(None, self.serverConfs[0]) test = conf.getServerKeyDir() self.assertTrue(os.path.isdir(conf.getServerKeyDir()))
def serverLoop(conf, serverState): """The main loop of the server process.""" cpc.util.log.initServerLog(conf, log_mode=ServerConf().getMode()) th2 = Thread(target=serveHTTPSWithCertAuthentication, args=[serverState]) th2.daemon = True th2.start() serveHTTPSWithNoCertReq(serverState)
def serveHTTPSWithNoCertReq(serverState): try: httpd = HTTPSServerNoCertAuthentication( request_handler.handlerForRequestWithNoCertReq, ServerConf(), serverState) sa = httpd.socket.getsockname() log.info("Serving HTTPS for client communication on %s port %s..." % (sa[0], sa[1])) httpd.serve_forever() except KeyboardInterrupt: print "Interrupted" serverState.doQuit() except Exception: #TODO better error handling of server errors during startup print "HTTPS port %s already taken" % ServerConf().getClientSecurePort( ) serverState.doQuit()
def run(self, serverState, request, response): priority = int(request.getParam("priority")) nodeId = request.getParam("nodeId") conf = ServerConf() nodes = conf.getNodes() nodes.changePriority(nodeId, priority) conf.write() response.add("", nodes.getNodesByPriority()) log.info("Changed %s node priority to %d" % (nodeId, priority))
def establishOutBoundConnections(): conf = ServerConf() log.log(cpc.util.log.TRACE, "Starting to establish outgoing connections") for node in conf.getNodes().nodes.itervalues(): establishOutboundConnection(node) log.log(cpc.util.log.TRACE, "Finished establishing outgoing " "connections")
def testAddNodes(self): conf = ServerConf(confdir=self.confDir) conf.addNode('localhost1') conf.addNode('localhost2') nodes = conf.getNodes() self.assertEquals(nodes.size(), 2) self.assertTrue(nodes.exists("localhost1", "13807")) self.assertTrue(nodes.exists("localhost2", "13807"))
def run(self, serverState, request, response): nodeConnectRequest = json.loads(request.getParam('nodeConnectRequest'), object_hook=json_serializer.fromJson) conf = ServerConf() conf.addNodeConnectRequest(nodeConnectRequest) result = dict() result['serverId'] = conf.getServerId() result['fqdn'] = conf.getFqdn() response.add("", result) log.info("Handled add node request")
def __init__(self, host=None, port=None): self.conf = ServerConf() self.host = host self.port = port if self.host == None: self.host = self.conf.getServerHost() if self.port == None: self.port = self.conf.getServerSecurePort() self.privateKey = self.conf.getPrivateKey() self.keychain = self.conf.getCaChainFile()
def shutdown_request(self, request): if (hasattr(request, "revertSocket") and request.revertSocket == True): node = ServerConf().getNodes().get(request.serverId) request = self.wrapHttpsConnectionWithCertReq(request) ServerConnectionPool().putConnection(request, node) node.addOutboundConnection() log.log( cpc.util.log.TRACE, "Socket not closed, " "saved as persistent outgoing" " connection") else: #This is how we distinguish socket that comes from a server if (hasattr(request, "serverId")): node = ServerConf().getNodes().get(request.serverId) node.reduceInboundConnection() log.log(cpc.util.log.TRACE, "Closing socket for %s " % node.toString()) SocketServer.TCPServer.shutdown_request(self, request)
def initTracker(): conf = ServerConf() dirs= Asset.getDirs() try: os.makedirs(conf.getLocalAssetsDir()) except: pass for dir in dirs: try: os.makedirs(os.path.join(conf.getLocalAssetsDir(), dir)) except: pass
def establishInboundConnections(serverState): """ for each node that is not connected try to establish an inbound connection """ conf = ServerConf() log.log(cpc.util.log.TRACE, "Starting to establish incoming connections") for node in conf.getNodes().nodes.itervalues(): establishInboundConnection(node, serverState) log.log(cpc.util.log.TRACE, "Finished establishing incoming " "connections")