def __init__(self, endNodeId):
#TODO perhaps node should be input
node = ServerConf().getNodes().get(endNodeId)
ServerConnection.__init__(self,node,ServerConf())
self.initialize(endNodeId)
"""Connect to a server opening a connection"""
def _handleSession(self, request):
handler_base._handleSession(self, request)
if 'user' not in request.session \
and not ServerConf().getServerVerification() \
and self.headers.has_key('originating-server-id'):
if ServerConf().getNodes().exists(
self.headers['originating-server-id']):
request.session['user'] = User(1, 'root', UserLevel.SUPERUSER)
def testStart2Servers(self):
numServers = 2
self.createConfFolders(numServers)
hostname = gethostname()
node0HttpsPort = 13807
node1HttpsPort = 13808
node0HttpPort = 14807
node1HttpPort = 14808
for i in range(numServers):
args = [
'../../../../cpc-server', '-c', self.serverConfs[i], 'start'
] #doing cpc.server.server.forkAndRun(cf, debug ) directly here will will for some strange reason mess up things when shutting down, the process wont shutdown
subprocess.call(args)
time.sleep(2)
#connect node 0 to node 1
args = [
'../../../../cpc-server', '-c', self.serverConfs[0],
'connnect-server', hostname,
str(node1HttpPort)
] #doing cpc.server.server.forkAndRun(cf, debug ) directly here will will for some strange reason mess up things when shutting down, the process wont shutdown
subprocess.call(args)
args = [
'../../../../cpc-server', '-c', self.serverConfs[1], 'trust',
hostname,
str(node0HttpsPort)
] #doing cpc.server.server.forkAndRun(cf, debug ) directly here will will for some strange reason mess up things when shutting down, the process wont shutdown
subprocess.call(args)
#verify existense of of nodes in each conf file
conf1 = ServerConf(confdir=self.serverConfs[0], reload=True)
node0Nodes = conf1.getNodes()
self.assertTrue(node0Nodes.exists(hostname, node1HttpsPort))
conf2 = ServerConf(confdir=self.serverConfs[1], reload=True)
node1Nodes = conf2.getNodes()
self.assertTrue(node1Nodes.exists(hostname, node0HttpsPort))
#do a network topology call
conf = ConnectionBundle(confdir=self.serverConfs[0], reload=True)
client = ClientMessage()
topology = ProcessedResponse(client.networkTopology()).getData()
self.assertEquals(topology.size(), 2)
def readDir(self, bindir, platforms):
"""Read a directory (usually in the search path) for all executables.
bindir = the directory name.
platforms = the list of availble platforms."""
reader = ExecutableReader(bindir)
try:
files = os.listdir(bindir)
except OSError:
files = []
for file in files:
try:
basedir = os.path.join(bindir, file)
log.debug("basedir is %s" % basedir)
pfile = os.path.join(basedir, "plugin")
log.debug("pfile is %s" % pfile)
nfile = os.path.join(basedir, "executable.xml")
log.debug("exec xml is %s" % nfile)
pl = None
# check whether this is in fact a plugin
if (not os.path.isdir(basedir)) and os.access(
basedir, os.X_OK):
plf = basedir
pl = cpc.util.plugin.ExecutablePlugin(basedir,
conf=ServerConf())
# or it contains a plugin
elif (not os.path.isdir(pfile)) and os.access(pfile, os.X_OK):
plf = basedir
pl = cpc.util.plugin.ExecutablePlugin(pfile,
conf=ServerConf())
if pl is not None:
# and run the plugin if it is one
for platform in platforms:
(retcode, retst) = pl.run(plf, platform.getName())
log.debug("returned: %s" % retst)
if retcode == 0:
reader.readString(retst,
"executable plugin output",
basedir)
elif os.path.exists(nfile):
try:
# otherwise just read the executable xml
reader.read(nfile, basedir)
except IOError:
pass
except IOError:
pass
self.executables.extend(reader.getExecutables())
def testInit(self):
#init the server conf
conf = ServerConf(confdir=self.confDir)
self.assertEquals(conf.get('conf_dir'), self.confDir)
self.assertEquals(self.confFile, conf.get('conf_file'))
conf.getServerKeyDir(
) #just to make sure it has been initiated the proper way
conf.setServerHost("testhost")
conf2 = ServerConf(
confdir=self.confDir
) #creating a new instance reinitiates the config and reads parameters from config file
self.assertEquals('testhost', conf2.get('server_host'))
def do_GET(self):
self.log.log(cpc.util.log.TRACE, '%s %s' % (self.command, self.path))
# if the path starts with application root + / or ? we have a message to process
#otherwise we should just strip any request params and keep the resource reference
if (self.isApplicationRoot()):
request = HttpMethodParser.parseGET(self.headers.dict, self.path)
self.processMessage(request)
#take the input and put it into a request object
else:
if self.path == "/":
self.path += "index.html"
else:
self.path = self.path.split('?', 1)[0] #strip trailing '?...'
webDir = ServerConf().getWebRootPath()
resourcePath = webDir + self.path
if not os.path.isfile(resourcePath):
self.responseCode = 404
resourcePath = webDir + '/404.html'
response = ServerResponse()
file = open(resourcePath, 'rb')
response.setFile(file, mimetypes.guess_type(resourcePath))
self._sendResponse(response)
def establishInboundConnection(node, serverState):
conf = ServerConf()
for i in range(0, conf.getNumPersistentConnections()):
try:
message = PersistentServerMessage(node, conf)
socket = message.persistIncomingConnection()
serverState.addReadableSocket(socket)
node.addInboundConnection()
except ServerConnectionError as e:
#The node is not reachable at this moment,
# no need to throw an exception since we are marking the node
# as unreachable ins ServerConnectionHandler
log.log(
cpc.util.log.TRACE, "Exception when establishing "
"inbound connections: %s " % e)
break
if node.isConnectedInbound():
log.log(
cpc.util.log.TRACE, "Established inbound "
"connections to server "
"%s" % node.toString())
else:
log.log(
cpc.util.log.TRACE, "Could not establish inbound "
"connections to server "
"%s" % node.toString())
def establishOutboundConnection(node):
conf = ServerConf()
for i in range(0, conf.getNumPersistentConnections()):
try:
#This will make a regular call, the connection pool will take
# care of the rest
message = PersistentServerMessage(node, conf)
resp = message.persistOutgoingConnection()
node.addOutboundConnection()
except ServerConnectionError as e:
#The node is not reachable at this moment,
# no need to throw an exception since we are marking the node
# as unreachable in ServerConnection
log.log(
cpc.util.log.TRACE, "Exception when establishing "
"outgoing connections: %s " % e)
break
if node.isConnectedOutbound():
log.log(
cpc.util.log.TRACE, "Established outgoing "
"connections to server "
"%s" % node.toString())
else:
log.log(
cpc.util.log.TRACE, "Could not establish outgoing "
"connections to %s" % node.toString())
def saveProject(self, project):
self.taskExecThreads.acquire()
conf = ServerConf()
try:
self.taskExecThreads.pause()
self._write()
projectFolder = "%s/%s" % (conf.getRunDir(), project)
if (os.path.isdir(projectFolder)):
#tar the project folder but keep the old files also, this is
# only a backup!!!
#copy _state.xml to _state.bak.xml
stateBackupFile = "%s/_state.bak.xml" % projectFolder
shutil.copyfile("%s/_state.xml" % projectFolder,
stateBackupFile)
tff = tempfile.TemporaryFile()
tf = tarfile.open(fileobj=tff, mode="w:gz")
tf.add(projectFolder, arcname=".", recursive=True)
tf.close()
del (tf)
tff.seek(0)
os.remove(stateBackupFile)
self.taskExecThreads.cont()
else:
self.taskExecThreads.cont()
raise Exception("Project does not exist")
finally:
self.taskExecThreads.release()
return tff
def __persistConnection(self, direction, headers=dict()):
headers['persistent-connection'] = direction
#message body is actually irrellevant and is not read on the other
# side.
#we just need to conform to the http protocol
fields = []
fields.append(Input('cmd', "persist-connection"))
#sending along the connection parameters for this server
conf = ServerConf()
connectionParams = dict()
connectionParams['serverId'] = conf.getServerId()
connectionParams['hostname'] = conf.getHostName()
connectionParams['fqdn'] = conf.getFqdn()
connectionParams['client_secure_port'] = conf\
.getClientSecurePort()
connectionParams['server_secure_port'] = conf\
.getServerSecurePort()
input2 = Input(
'connectionParams',
json.dumps(connectionParams,
default=json_serializer.toJson,
indent=4)) # a json structure that needs to be dumped
fields.append(input2)
response = self.putRequest(
ServerRequest.prepareRequest(fields, [], headers))
return response
def sendAddNodeRequest(self, host):
"""
"""
conf = ServerConf()
cmdstring = 'connect-server-request'
fields = []
input = Input('cmd', cmdstring)
inf = open(conf.getCACertFile(), "r")
key = inf.read()
nodeConnectRequest = NodeConnectRequest(conf.getServerId(),
conf.getClientSecurePort(),
conf.getServerSecurePort(),
key, conf.getFqdn(),
conf.getHostName())
input2 = Input(
'nodeConnectRequest',
json.dumps(nodeConnectRequest,
default=json_serializer.toJson,
indent=4))
input3 = Input('unqalifiedDomainName', host)
fields.append(input)
fields.append(input2)
fields.append(input3)
fields.append(Input('version', "1"))
# this goes over the client Secure Port, and we don't want the server to use
# cookies
response = self.postRequest(ServerRequest.prepareRequest(fields),
require_certificate_authentication=False,
disable_cookies=True)
return response
def readProjectState(self, projectName):
prj = self.projects[projectName]
conf = ServerConf()
projectBaseDir = "%s/%s" % (conf.getRunDir(), projectName)
#We have a couple of hardcoded paths that might not be valid anymore
# Think of the case when we are moving projects to another server
# here we replace those old paths with new valid paths
stateBakXML = "%s/%s" % (projectBaseDir, "_state.bak.xml")
#/get the state_bak.xml
file = open(stateBakXML, 'r')
content = file.read()
file.close()
#find first occurence of <env ..... base_dir=<BASE_DIR>
m = re.search('<env .* base_dir="(.*)".*>', content)
if m != None: #only if we have a project with active tasks
oldBaseDir = m.group(1)
new_content = re.sub(oldBaseDir, projectBaseDir, content)
file = open(stateBakXML, "w")
file.write(new_content)
file.close()
#reread project state
prj.readState(stateFile="_state.bak.xml")
def run(self, serverState, request, response):
conf = ServerConf()
sentConnectRequests = conf.getSentNodeConnectRequests()
node = json.loads(request.getParam('connectRequest'),
object_hook=json_serializer.fromJson)
if (sentConnectRequests.exists(node.getId())):
nodeToAdd = sentConnectRequests.get(node.getId())
conf.addNode(
Node(node.server_id, node.getClientSecurePort(),
node.getServerSecurePort(), node.getQualifiedName(),
nodeToAdd.getHostname()))
#conf.addNode(nodeToAdd)
openssl = OpenSSL(conf)
openssl.addCa(node.key)
sentConnectRequests.removeNode(node.getId())
conf.set('sent_node_connect_requests', sentConnectRequests)
# need to send back a status in the data notifying ok
response.add('Connection to node %s established' % node.toString())
log.info("Node connection accepted")
#add it to the node list
else:
response.add('No previous node request sent for host %s' %
node.toString())
log.info("Node connection not accepted")
def run(self, serverState, request, response):
#get the connection params for this node
newParams = json.loads(request.getParam("connectionParams"))
conf = ServerConf()
nodes = conf.getNodes()
if nodes.exists(newParams['serverId']):
node = nodes.get(newParams['serverId'])
node.setHostname(newParams['hostname'])
node.setServerSecurePort(newParams['server_secure_port'])
node.setClientSecurePort(newParams['client_secure_port'])
node.setQualifiedName(newParams['fqdn'])
#Needed so that we write changes to conf file
conf.removeNode(node.server_id)
conf.addNode(node)
#update the network topology
ServerToServerMessage.getNetworkTopology(resetCache=True)
response.add("Updated connection parameters")
log.info("Updated connection params for %s" % node.toString())
else:
response.add("Requested update for node %s but this node " \
"is not a neigbouring node "%newParams[
'serverId'],
status="ERROR")
log.error("Failed updating connection params for %s" %
newParams['serverId'])
def grant(key): #key is server-id
conf = ServerConf()
nodeConnectRequests = conf.getNodeConnectRequests()
if nodeConnectRequests.exists(key):
nodeToAdd = nodeConnectRequests.get(
key) #this returns a nodeConnectRequest object
serv = RawServerMessage(nodeToAdd.getHostname(),
nodeToAdd.getClientSecurePort())
#letting the requesting node know that it is accepted
#also sending this servers connection parameters
resp = serv.addNodeAccepted()
conf.addNode(
Node(nodeToAdd.server_id, nodeToAdd.getClientSecurePort(),
nodeToAdd.getServerSecurePort(),
nodeToAdd.getQualifiedName(), nodeToAdd.getHostname()))
#trust the key
openssl = OpenSSL(conf)
openssl.addCa(nodeToAdd.key)
nodeConnectRequests.removeNode(nodeToAdd.getId())
conf.set('node_connect_requests', nodeConnectRequests)
return True
else:
return False
def run(self, serverState, request, response):
conf = ServerConf()
host = request.getParam('host')
client_secure_port = request.getParam('client_secure_port')
result = dict()
#do we have a server with this hostname or fqdn?
connectedNodes = conf.getNodes()
if (connectedNodes.hostnameOrFQDNExists(host) == False):
serv = RawServerMessage(host, client_secure_port)
resp = ProcessedResponse(serv.sendAddNodeRequest(host))
if resp.isOK():
result = resp.getData()
nodeConnectRequest = NodeConnectRequest(
result['serverId'], int(client_secure_port), None, None,
result['fqdn'], host)
conf.addSentNodeConnectRequest(nodeConnectRequest)
result['nodeConnectRequest'] = nodeConnectRequest
log.info("Added node %s" % host)
response.add('', result)
else:
response.add("Remote server said: %s" % resp.getMessage(),
status="ERROR")
else:
errorMessage = "%s is already trusted" % host
response.add(errorMessage, status="ERROR")
log.info(errorMessage)
def run(self, serverState, request, response):
projectName = request.getParam("project")
user = self.getUser(request)
if (request.haveFile("projectFile")):
projectBundle = request.getFile('projectFile')
try:
serverState.getProjectList().add(projectName)
UserHandler().addUserToProject(user, projectName)
extractPath = "%s/%s" % (ServerConf().getRunDir(), projectName)
tar = tarfile.open(fileobj=projectBundle, mode="r")
tar.extractall(path=extractPath)
tar.close()
del (tar)
serverState.readProjectState(projectName)
gc.collect()
except:
response.add("No project file provided", status="ERROR")
return
response.add("Project restored as %s" % projectName)
log.info("Project load %s" % (projectName))
else:
response.add("No project file provided", status="ERROR")
log.info("Project load %s failed" % (projectName))
def testBroadCastMessage(self):
self.create5NodeNetwork()
#this way we initialize the configs for a specific server which lets us test method calls as as specific server
clientConf = ConnectionBundle(confdir=self.serverConfs[0], reload=True)
serverConf = ServerConf(confdir=self.serverConfs[0], reload=True)
broadCastMessage = BroadcastMessage()
#logs should mention that we are setting elements to cache
broadCastMessage.updateNetworkTopology()
clientConf = ConnectionBundle(confdir=self.serverConfs[1], reload=True)
serverConf = ServerConf(confdir=self.serverConfs[1], reload=True)
#this should be brought from the cache(check in the logs)
ServerToServerMessage.getNetworkTopology()
def addNodeAccepted(self):
conf = ServerConf()
inf = open(conf.getCACertFile(), "r")
key = inf.read()
#only sending fqdn the requesting should already know the unqualified
# hostname
node = NodeConnectRequest(conf.getServerId(),
conf.getClientSecurePort(),
conf.getServerSecurePort(), key,
conf.getFqdn(), None)
cmdstring = 'node-connection-accepted'
fields = []
input = Input('cmd', cmdstring)
input2 = Input(
'connectRequest',
json.dumps(node, default=json_serializer.toJson, indent=4))
fields.append(input)
fields.append(input2)
fields.append(Input('version', "1"))
# this goes over client secure port , and we don't want the server to use
# cookies
response = self.postRequest(ServerRequest.prepareRequest(fields),
require_certificate_authentication=False,
disable_cookies=True)
return response
def test_createConf(self):
self.createConfFolder(0)
conf = ServerConf(None, self.serverConfs[0])
test = conf.getServerKeyDir()
self.assertTrue(os.path.isdir(conf.getServerKeyDir()))
def serverLoop(conf, serverState):
"""The main loop of the server process."""
cpc.util.log.initServerLog(conf, log_mode=ServerConf().getMode())
th2 = Thread(target=serveHTTPSWithCertAuthentication, args=[serverState])
th2.daemon = True
th2.start()
serveHTTPSWithNoCertReq(serverState)
def serveHTTPSWithNoCertReq(serverState):
try:
httpd = HTTPSServerNoCertAuthentication(
request_handler.handlerForRequestWithNoCertReq, ServerConf(),
serverState)
sa = httpd.socket.getsockname()
log.info("Serving HTTPS for client communication on %s port %s..." %
(sa[0], sa[1]))
httpd.serve_forever()
except KeyboardInterrupt:
print "Interrupted"
serverState.doQuit()
except Exception:
#TODO better error handling of server errors during startup
print "HTTPS port %s already taken" % ServerConf().getClientSecurePort(
)
serverState.doQuit()
def run(self, serverState, request, response):
priority = int(request.getParam("priority"))
nodeId = request.getParam("nodeId")
conf = ServerConf()
nodes = conf.getNodes()
nodes.changePriority(nodeId, priority)
conf.write()
response.add("", nodes.getNodesByPriority())
log.info("Changed %s node priority to %d" % (nodeId, priority))
def establishOutBoundConnections():
conf = ServerConf()
log.log(cpc.util.log.TRACE, "Starting to establish outgoing connections")
for node in conf.getNodes().nodes.itervalues():
establishOutboundConnection(node)
log.log(cpc.util.log.TRACE, "Finished establishing outgoing "
"connections")
def testAddNodes(self):
conf = ServerConf(confdir=self.confDir)
conf.addNode('localhost1')
conf.addNode('localhost2')
nodes = conf.getNodes()
self.assertEquals(nodes.size(), 2)
self.assertTrue(nodes.exists("localhost1", "13807"))
self.assertTrue(nodes.exists("localhost2", "13807"))
def run(self, serverState, request, response):
nodeConnectRequest = json.loads(request.getParam('nodeConnectRequest'),
object_hook=json_serializer.fromJson)
conf = ServerConf()
conf.addNodeConnectRequest(nodeConnectRequest)
result = dict()
result['serverId'] = conf.getServerId()
result['fqdn'] = conf.getFqdn()
response.add("", result)
log.info("Handled add node request")
def __init__(self, host=None, port=None):
self.conf = ServerConf()
self.host = host
self.port = port
if self.host == None:
self.host = self.conf.getServerHost()
if self.port == None:
self.port = self.conf.getServerSecurePort()
self.privateKey = self.conf.getPrivateKey()
self.keychain = self.conf.getCaChainFile()
def shutdown_request(self, request):
if (hasattr(request, "revertSocket") and request.revertSocket == True):
node = ServerConf().getNodes().get(request.serverId)
request = self.wrapHttpsConnectionWithCertReq(request)
ServerConnectionPool().putConnection(request, node)
node.addOutboundConnection()
log.log(
cpc.util.log.TRACE, "Socket not closed, "
"saved as persistent outgoing"
" connection")
else:
#This is how we distinguish socket that comes from a server
if (hasattr(request, "serverId")):
node = ServerConf().getNodes().get(request.serverId)
node.reduceInboundConnection()
log.log(cpc.util.log.TRACE,
"Closing socket for %s " % node.toString())
SocketServer.TCPServer.shutdown_request(self, request)
def initTracker():
conf = ServerConf()
dirs= Asset.getDirs()
try:
os.makedirs(conf.getLocalAssetsDir())
except:
pass
for dir in dirs:
try:
os.makedirs(os.path.join(conf.getLocalAssetsDir(), dir))
except:
pass
def establishInboundConnections(serverState):
"""
for each node that is not connected
try to establish an inbound connection
"""
conf = ServerConf()
log.log(cpc.util.log.TRACE, "Starting to establish incoming connections")
for node in conf.getNodes().nodes.itervalues():
establishInboundConnection(node, serverState)
log.log(cpc.util.log.TRACE, "Finished establishing incoming "
"connections")
