def establishOutboundConnection(node):
conf = ServerConf()
for i in range(0, conf.getNumPersistentConnections()):
try:
#This will make a regular call, the connection pool will take
# care of the rest
message = PersistentServerMessage(node, conf)
resp = message.persistOutgoingConnection()
node.addOutboundConnection()
except ServerConnectionError as e:
#The node is not reachable at this moment,
# no need to throw an exception since we are marking the node
# as unreachable in ServerConnection
log.log(
cpc.util.log.TRACE, "Exception when establishing "
"outgoing connections: %s " % e)
break
if node.isConnectedOutbound():
log.log(
cpc.util.log.TRACE, "Established outgoing "
"connections to server "
"%s" % node.toString())
else:
log.log(
cpc.util.log.TRACE, "Could not establish outgoing "
"connections to %s" % node.toString())
def saveProject(self, project):
self.taskExecThreads.acquire()
conf = ServerConf()
try:
self.taskExecThreads.pause()
self._write()
projectFolder = "%s/%s" % (conf.getRunDir(), project)
if (os.path.isdir(projectFolder)):
#tar the project folder but keep the old files also, this is
# only a backup!!!
#copy _state.xml to _state.bak.xml
stateBackupFile = "%s/_state.bak.xml" % projectFolder
shutil.copyfile("%s/_state.xml" % projectFolder,
stateBackupFile)
tff = tempfile.TemporaryFile()
tf = tarfile.open(fileobj=tff, mode="w:gz")
tf.add(projectFolder, arcname=".", recursive=True)
tf.close()
del (tf)
tff.seek(0)
os.remove(stateBackupFile)
self.taskExecThreads.cont()
else:
self.taskExecThreads.cont()
raise Exception("Project does not exist")
finally:
self.taskExecThreads.release()
return tff
def establishOutboundConnection(node):
conf = ServerConf()
for i in range(0, conf.getNumPersistentConnections()):
try:
#This will make a regular call, the connection pool will take
# care of the rest
message = PersistentServerMessage(node, conf)
resp = message.persistOutgoingConnection()
node.addOutboundConnection()
except ServerConnectionError as e:
#The node is not reachable at this moment,
# no need to throw an exception since we are marking the node
# as unreachable in ServerConnection
log.log(cpc.util.log.TRACE, "Exception when establishing "
"outgoing connections: %s " % e)
break
if node.isConnectedOutbound():
log.log(cpc.util.log.TRACE,"Established outgoing "
"connections to server "
"%s"%node.toString())
else:
log.log(cpc.util.log.TRACE,"Could not establish outgoing "
"connections to %s"%node.toString())
def __persistConnection(self,direction,headers = dict()):
headers['persistent-connection'] = direction
#message body is actually irrellevant and is not read on the other
# side.
#we just need to conform to the http protocol
fields = []
fields.append(Input('cmd', "persist-connection"))
#sending along the connection parameters for this server
conf = ServerConf()
connectionParams = dict()
connectionParams['serverId'] = conf.getServerId()
connectionParams['hostname'] = conf.getHostName()
connectionParams['fqdn'] = conf.getFqdn()
connectionParams['client_secure_port'] = conf\
.getClientSecurePort()
connectionParams['server_secure_port'] = conf\
.getServerSecurePort()
input2 = Input('connectionParams',
json.dumps(connectionParams,default = json_serializer.toJson,
indent=4)) # a json structure that needs to be dumped
fields.append(input2)
response= self.putRequest(ServerRequest.prepareRequest(fields, [],
headers))
return response
def __persistConnection(self, direction, headers=dict()):
headers['persistent-connection'] = direction
#message body is actually irrellevant and is not read on the other
# side.
#we just need to conform to the http protocol
fields = []
fields.append(Input('cmd', "persist-connection"))
#sending along the connection parameters for this server
conf = ServerConf()
connectionParams = dict()
connectionParams['serverId'] = conf.getServerId()
connectionParams['hostname'] = conf.getHostName()
connectionParams['fqdn'] = conf.getFqdn()
connectionParams['client_secure_port'] = conf\
.getClientSecurePort()
connectionParams['server_secure_port'] = conf\
.getServerSecurePort()
input2 = Input(
'connectionParams',
json.dumps(connectionParams,
default=json_serializer.toJson,
indent=4)) # a json structure that needs to be dumped
fields.append(input2)
response = self.putRequest(
ServerRequest.prepareRequest(fields, [], headers))
return response
def readProjectState(self, projectName):
prj = self.projects[projectName]
conf = ServerConf()
projectBaseDir = "%s/%s" % (conf.getRunDir(), projectName)
#We have a couple of hardcoded paths that might not be valid anymore
# Think of the case when we are moving projects to another server
# here we replace those old paths with new valid paths
stateBakXML = "%s/%s" % (projectBaseDir, "_state.bak.xml")
#/get the state_bak.xml
file = open(stateBakXML, 'r')
content = file.read()
file.close()
#find first occurence of <env ..... base_dir=<BASE_DIR>
m = re.search('<env .* base_dir="(.*)".*>', content)
if m != None: #only if we have a project with active tasks
oldBaseDir = m.group(1)
new_content = re.sub(oldBaseDir, projectBaseDir, content)
file = open(stateBakXML, "w")
file.write(new_content)
file.close()
#reread project state
prj.readState(stateFile="_state.bak.xml")
def run(self, serverState, request, response):
conf = ServerConf()
host = request.getParam('host')
client_secure_port = request.getParam('client_secure_port')
result = dict()
#do we have a server with this hostname or fqdn?
connectedNodes = conf.getNodes()
if (connectedNodes.hostnameOrFQDNExists(host) == False):
serv = RawServerMessage(host, client_secure_port)
resp = ProcessedResponse(serv.sendAddNodeRequest(host))
if resp.isOK():
result = resp.getData()
nodeConnectRequest = NodeConnectRequest(
result['serverId'], int(client_secure_port), None, None,
result['fqdn'], host)
conf.addSentNodeConnectRequest(nodeConnectRequest)
result['nodeConnectRequest'] = nodeConnectRequest
log.info("Added node %s" % host)
response.add('', result)
else:
response.add("Remote server said: %s" % resp.getMessage(),
status="ERROR")
else:
errorMessage = "%s is already trusted" % host
response.add(errorMessage, status="ERROR")
log.info(errorMessage)
def run(self, serverState, request, response):
conf = ServerConf()
host = request.getParam('host')
client_secure_port = request.getParam('client_secure_port')
result = dict()
#do we have a server with this hostname or fqdn?
connectedNodes = conf.getNodes()
if (connectedNodes.hostnameOrFQDNExists(host) == False):
serv = RawServerMessage(host, client_secure_port)
resp = ProcessedResponse(serv.sendAddNodeRequest(host))
if resp.isOK():
result = resp.getData()
nodeConnectRequest = NodeConnectRequest(result['serverId'],
int(client_secure_port),None,None,result['fqdn'],host)
conf.addSentNodeConnectRequest(nodeConnectRequest)
result['nodeConnectRequest']=nodeConnectRequest
log.info("Added node %s" % host)
response.add('', result)
else:
response.add("Remote server said: %s"%resp.getMessage(),
status="ERROR")
else:
errorMessage = "%s is already trusted" % host
response.add(errorMessage, status="ERROR")
log.info(errorMessage)
def readProjectState(self, projectName):
prj = self.projects[projectName]
conf = ServerConf()
projectBaseDir = "%s/%s" % (conf.getRunDir(), projectName)
# We have a couple of hardcoded paths that might not be valid anymore
# Think of the case when we are moving projects to another server
# here we replace those old paths with new valid paths
stateBakXML = "%s/%s" % (projectBaseDir, "_state.bak.xml")
# /get the state_bak.xml
file = open(stateBakXML, "r")
content = file.read()
file.close()
# find first occurence of <env ..... base_dir=<BASE_DIR>
m = re.search('<env .* base_dir="(.*)".*>', content)
if m != None: # only if we have a project with active tasks
oldBaseDir = m.group(1)
new_content = re.sub(oldBaseDir, projectBaseDir, content)
file = open(stateBakXML, "w")
file.write(new_content)
file.close()
# reread project state
prj.readState(stateFile="_state.bak.xml")
def establishInboundConnection(node, serverState):
conf=ServerConf()
for i in range(0, conf.getNumPersistentConnections()):
try:
message = PersistentServerMessage(node, conf)
socket = message.persistIncomingConnection()
serverState.addReadableSocket(socket)
node.addInboundConnection()
except ServerConnectionError as e:
#The node is not reachable at this moment,
# no need to throw an exception since we are marking the node
# as unreachable ins ServerConnectionHandler
log.log(cpc.util.log.TRACE, "Exception when establishing "
"inbound connections: %s " % e)
break
if node.isConnectedInbound():
log.log(cpc.util.log.TRACE, "Established inbound "
"connections to server "
"%s" % node.toString())
else:
log.log(cpc.util.log.TRACE, "Could not establish inbound "
"connections to server "
"%s" % node.toString())
def saveProject(self,project):
self.taskExecThreads.acquire()
conf = ServerConf()
try:
self.taskExecThreads.pause()
self._write()
projectFolder = "%s/%s"%(conf.getRunDir(),project)
if(os.path.isdir(projectFolder)):
#tar the project folder but keep the old files also, this is
# only a backup!!!
#copy _state.xml to _state.bak.xml
stateBackupFile = "%s/_state.bak.xml"%projectFolder
shutil.copyfile("%s/_state.xml"%projectFolder,stateBackupFile)
tff=tempfile.TemporaryFile()
tf=tarfile.open(fileobj=tff, mode="w:gz")
tf.add(projectFolder, arcname=".", recursive=True)
tf.close()
del(tf)
tff.seek(0)
os.remove(stateBackupFile)
self.taskExecThreads.cont()
else:
self.taskExecThreads.cont()
raise Exception("Project does not exist")
finally:
self.taskExecThreads.release()
return tff
def establishInboundConnection(node, serverState):
conf = ServerConf()
for i in range(0, conf.getNumPersistentConnections()):
try:
message = PersistentServerMessage(node, conf)
socket = message.persistIncomingConnection()
serverState.addReadableSocket(socket)
node.addInboundConnection()
except ServerConnectionError as e:
#The node is not reachable at this moment,
# no need to throw an exception since we are marking the node
# as unreachable ins ServerConnectionHandler
log.log(
cpc.util.log.TRACE, "Exception when establishing "
"inbound connections: %s " % e)
break
if node.isConnectedInbound():
log.log(
cpc.util.log.TRACE, "Established inbound "
"connections to server "
"%s" % node.toString())
else:
log.log(
cpc.util.log.TRACE, "Could not establish inbound "
"connections to server "
"%s" % node.toString())
def test_createConf(self):
self.createConfFolder(0)
conf = ServerConf(None, self.serverConfs[0])
test = conf.getServerKeyDir()
self.assertTrue(os.path.isdir(conf.getServerKeyDir()))
def test_createConf(self):
self.createConfFolder(0)
conf = ServerConf(None,self.serverConfs[0])
test =conf.getServerKeyDir()
self.assertTrue(os.path.isdir(conf.getServerKeyDir()))
def __init__(self, endNodeId):
#TODO perhaps node should be input
node = ServerConf().getNodes().get(endNodeId)
ServerConnection.__init__(self,node,ServerConf())
self.initialize(endNodeId)
"""Connect to a server opening a connection"""
def testAddNodes(self):
conf = ServerConf(confdir=self.confDir)
conf.addNode('localhost1')
conf.addNode('localhost2')
nodes = conf.getNodes()
self.assertEquals(nodes.size(), 2)
self.assertTrue(nodes.exists("localhost1", "13807"))
self.assertTrue(nodes.exists("localhost2", "13807"))
def establishOutBoundConnections():
conf = ServerConf()
log.log(cpc.util.log.TRACE, "Starting to establish outgoing connections")
for node in conf.getNodes().nodes.itervalues():
establishOutboundConnection(node)
log.log(cpc.util.log.TRACE, "Finished establishing outgoing "
"connections")
def run(self, serverState, request, response):
priority = int(request.getParam("priority"))
nodeId = request.getParam("nodeId")
conf = ServerConf()
nodes = conf.getNodes()
nodes.changePriority(nodeId, priority)
conf.write()
response.add("", nodes.getNodesByPriority())
log.info("Changed %s node priority to %d" % (nodeId, priority))
def run(self, serverState, request, response):
priority = int(request.getParam("priority"))
nodeId = request.getParam("nodeId")
conf = ServerConf()
nodes = conf.getNodes()
nodes.changePriority(nodeId, priority)
conf.write()
response.add("", nodes.getNodesByPriority())
log.info("Changed %s node priority to %d" % (nodeId, priority))
def establishOutBoundConnections():
conf = ServerConf()
log.log(cpc.util.log.TRACE,"Starting to establish outgoing connections")
for node in conf.getNodes().nodes.itervalues():
establishOutboundConnection(node)
log.log(cpc.util.log.TRACE,"Finished establishing outgoing "
"connections")
def _handleSession(self, request):
handler_base._handleSession(self, request)
if 'user' not in request.session \
and not ServerConf().getServerVerification() \
and self.headers.has_key('originating-server-id'):
if ServerConf().getNodes().exists(
self.headers['originating-server-id']):
request.session['user'] = User(1, 'root', UserLevel.SUPERUSER)
def testAddNodes(self):
conf = ServerConf(confdir=self.confDir)
conf.addNode('localhost1')
conf.addNode('localhost2')
nodes = conf.getNodes()
self.assertEquals(nodes.size(),2)
self.assertTrue(nodes.exists("localhost1","13807"))
self.assertTrue(nodes.exists("localhost2","13807") )
def __init__(self, host=None, port=None):
self.conf = ServerConf()
self.host = host
self.port = port
if self.host == None:
self.host = self.conf.getServerHost()
if self.port == None:
self.port = self.conf.getServerSecurePort()
self.privateKey = self.conf.getPrivateKey()
self.keychain = self.conf.getCaChainFile()
def run(self, serverState, request, response):
nodeConnectRequest = json.loads(request.getParam('nodeConnectRequest'),
object_hook=json_serializer.fromJson)
conf = ServerConf()
conf.addNodeConnectRequest(nodeConnectRequest)
result =dict()
result['serverId'] = conf.getServerId()
result['fqdn'] = conf.getFqdn()
response.add("", result)
log.info("Handled add node request")
def establishInboundConnections(serverState):
"""
for each node that is not connected
try to establish an inbound connection
"""
conf = ServerConf()
log.log(cpc.util.log.TRACE, "Starting to establish incoming connections")
for node in conf.getNodes().nodes.itervalues():
establishInboundConnection(node, serverState)
log.log(cpc.util.log.TRACE, "Finished establishing incoming "
"connections")
def initTracker():
conf = ServerConf()
dirs= Asset.getDirs()
try:
os.makedirs(conf.getLocalAssetsDir())
except:
pass
for dir in dirs:
try:
os.makedirs(os.path.join(conf.getLocalAssetsDir(), dir))
except:
pass
def establishInboundConnections(serverState):
"""
for each node that is not connected
try to establish an inbound connection
"""
conf = ServerConf()
log.log(cpc.util.log.TRACE,"Starting to establish incoming connections")
for node in conf.getNodes().nodes.itervalues():
establishInboundConnection(node, serverState)
log.log(cpc.util.log.TRACE,"Finished establishing incoming "
"connections")
def sendAddNodeRequest(self, host):
"""
"""
conf = ServerConf()
cmdstring = 'connect-server-request'
fields = []
input = Input('cmd', cmdstring)
inf = open(conf.getCACertFile(), "r")
key = inf.read()
nodeConnectRequest = NodeConnectRequest(conf.getServerId(),
conf.getClientSecurePort(),
conf.getServerSecurePort(),
key, conf.getFqdn(),
conf.getHostName())
input2 = Input(
'nodeConnectRequest',
json.dumps(nodeConnectRequest,
default=json_serializer.toJson,
indent=4))
input3 = Input('unqalifiedDomainName', host)
fields.append(input)
fields.append(input2)
fields.append(input3)
fields.append(Input('version', "1"))
# this goes over the client Secure Port, and we don't want the server to use
# cookies
response = self.postRequest(ServerRequest.prepareRequest(fields),
require_certificate_authentication=False,
disable_cookies=True)
return response
def readDir(self, bindir, platforms):
"""Read a directory (usually in the search path) for all executables.
bindir = the directory name.
platforms = the list of availble platforms."""
reader = ExecutableReader(bindir)
try:
files = os.listdir(bindir)
except OSError:
files = []
for file in files:
try:
basedir = os.path.join(bindir, file)
log.debug("basedir is %s" % basedir)
pfile = os.path.join(basedir, "plugin")
log.debug("pfile is %s" % pfile)
nfile = os.path.join(basedir, "executable.xml")
log.debug("exec xml is %s" % nfile)
pl = None
# check whether this is in fact a plugin
if (not os.path.isdir(basedir)) and os.access(
basedir, os.X_OK):
plf = basedir
pl = cpc.util.plugin.ExecutablePlugin(basedir,
conf=ServerConf())
# or it contains a plugin
elif (not os.path.isdir(pfile)) and os.access(pfile, os.X_OK):
plf = basedir
pl = cpc.util.plugin.ExecutablePlugin(pfile,
conf=ServerConf())
if pl is not None:
# and run the plugin if it is one
for platform in platforms:
(retcode, retst) = pl.run(plf, platform.getName())
log.debug("returned: %s" % retst)
if retcode == 0:
reader.readString(retst,
"executable plugin output",
basedir)
elif os.path.exists(nfile):
try:
# otherwise just read the executable xml
reader.read(nfile, basedir)
except IOError:
pass
except IOError:
pass
self.executables.extend(reader.getExecutables())
def do_GET(self):
self.log.log(cpc.util.log.TRACE, '%s %s' % (self.command, self.path))
# if the path starts with application root + / or ? we have a message to process
#otherwise we should just strip any request params and keep the resource reference
if (self.isApplicationRoot()):
request = HttpMethodParser.parseGET(self.headers.dict, self.path)
self.processMessage(request)
#take the input and put it into a request object
else:
if self.path == "/":
self.path += "index.html"
else:
self.path = self.path.split('?', 1)[0] #strip trailing '?...'
webDir = ServerConf().getWebRootPath()
resourcePath = webDir + self.path
if not os.path.isfile(resourcePath):
self.responseCode = 404
resourcePath = webDir + '/404.html'
response = ServerResponse()
file = open(resourcePath, 'rb')
response.setFile(file, mimetypes.guess_type(resourcePath))
self._sendResponse(response)
class DBHandler(object):
"""
Provides access to the server database. Can be used from several threads.
example usage is
handler = DBHandler()
with handler.getCursor() as c:
c.execute("query")
c.execute("query")
This provides a transactional cursor which will rollback any changes if an
exception is throw at any time during the 'with' clause. The changes are
committed once the with-clause goes out of scope.
Note that transaction only cover DML statements and will implicitly commit
before any non-dml, such as a CREATE TABLE
"""
def __init__(self):
from cpc.util.conf.server_conf import ServerConf
self.conf = ServerConf()
self.dbpath = os.path.join(self.conf.getConfDir(),'users.db')
def getCursor(self):
return DBCursor(self.dbpath)
def allocateDatabase(self):
if os.path.isfile(self.dbpath):
raise DataBaseError("Database already exist at %s"%self.dbpath)
sqlite3.connect(self.dbpath)
def testBroadCastMessage(self):
self.create5NodeNetwork()
#this way we initialize the configs for a specific server which lets us test method calls as as specific server
clientConf = ConnectionBundle(confdir=self.serverConfs[0], reload=True)
serverConf = ServerConf(confdir=self.serverConfs[0], reload=True)
broadCastMessage = BroadcastMessage()
#logs should mention that we are setting elements to cache
broadCastMessage.updateNetworkTopology()
clientConf = ConnectionBundle(confdir=self.serverConfs[1], reload=True)
serverConf = ServerConf(confdir=self.serverConfs[1], reload=True)
#this should be brought from the cache(check in the logs)
ServerToServerMessage.getNetworkTopology()
def run(self, serverState, request, response):
projectName = request.getParam("project")
user = self.getUser(request)
if (request.haveFile("projectFile")):
projectBundle = request.getFile('projectFile')
try:
serverState.getProjectList().add(projectName)
UserHandler().addUserToProject(user, projectName)
extractPath = "%s/%s" % (ServerConf().getRunDir(), projectName)
tar = tarfile.open(fileobj=projectBundle, mode="r")
tar.extractall(path=extractPath)
tar.close()
del (tar)
serverState.readProjectState(projectName)
gc.collect()
except:
response.add("No project file provided", status="ERROR")
return
response.add("Project restored as %s" % projectName)
log.info("Project load %s" % (projectName))
else:
response.add("No project file provided", status="ERROR")
log.info("Project load %s failed" % (projectName))
def testInit(self):
#init the server conf
conf = ServerConf(confdir=self.confDir)
self.assertEquals(conf.get('conf_dir'),self.confDir)
self.assertEquals(self.confFile,conf.get('conf_file'))
conf.getServerKeyDir() #just to make sure it has been initiated the proper way
conf.setServerHost("testhost")
conf2 = ServerConf(confdir = self.confDir) #creating a new instance reinitiates the config and reads parameters from config file
self.assertEquals('testhost',conf2.get('server_host'))
def serverLoop(conf, serverState):
"""The main loop of the server process."""
cpc.util.log.initServerLog(conf, log_mode=ServerConf().getMode())
th2 = Thread(target=serveHTTPSWithCertAuthentication, args=[serverState])
th2.daemon = True
th2.start()
serveHTTPSWithNoCertReq(serverState)
def run(self, serverState, request, response):
#get the connection params for this node
newParams = json.loads(request.getParam("connectionParams"))
conf = ServerConf()
nodes = conf.getNodes()
if nodes.exists(newParams['serverId']):
node = nodes.get(newParams['serverId'])
node.setHostname(newParams['hostname'])
node.setServerSecurePort(newParams['server_secure_port'])
node.setClientSecurePort(newParams['client_secure_port'])
node.setQualifiedName(newParams['fqdn'])
#Needed so that we write changes to conf file
conf.removeNode(node.server_id)
conf.addNode(node)
#update the network topology
ServerToServerMessage.getNetworkTopology(resetCache=True)
response.add("Updated connection parameters")
log.info("Updated connection params for %s" % node.toString())
else:
response.add("Requested update for node %s but this node " \
"is not a neigbouring node "%newParams[
'serverId'],
status="ERROR")
log.error("Failed updating connection params for %s" %
newParams['serverId'])
def run(self, serverState, request, response):
conf = ServerConf()
sentConnectRequests = conf.getSentNodeConnectRequests()
node = json.loads(request.getParam('connectRequest'),
object_hook=json_serializer.fromJson)
if (sentConnectRequests.exists(node.getId())):
nodeToAdd = sentConnectRequests.get(node.getId())
conf.addNode(
Node(node.server_id, node.getClientSecurePort(),
node.getServerSecurePort(), node.getQualifiedName(),
nodeToAdd.getHostname()))
#conf.addNode(nodeToAdd)
openssl = OpenSSL(conf)
openssl.addCa(node.key)
sentConnectRequests.removeNode(node.getId())
conf.set('sent_node_connect_requests', sentConnectRequests)
# need to send back a status in the data notifying ok
response.add('Connection to node %s established' % node.toString())
log.info("Node connection accepted")
#add it to the node list
else:
response.add('No previous node request sent for host %s' %
node.toString())
log.info("Node connection not accepted")
def grant(key): #key is server-id
conf = ServerConf()
nodeConnectRequests = conf.getNodeConnectRequests()
if nodeConnectRequests.exists(key):
nodeToAdd = nodeConnectRequests.get(
key) #this returns a nodeConnectRequest object
serv = RawServerMessage(nodeToAdd.getHostname(),
nodeToAdd.getClientSecurePort())
#letting the requesting node know that it is accepted
#also sending this servers connection parameters
resp = serv.addNodeAccepted()
conf.addNode(
Node(nodeToAdd.server_id, nodeToAdd.getClientSecurePort(),
nodeToAdd.getServerSecurePort(),
nodeToAdd.getQualifiedName(), nodeToAdd.getHostname()))
#trust the key
openssl = OpenSSL(conf)
openssl.addCa(nodeToAdd.key)
nodeConnectRequests.removeNode(nodeToAdd.getId())
conf.set('node_connect_requests', nodeConnectRequests)
return True
else:
return False
def addNodeAccepted(self):
conf = ServerConf()
inf = open(conf.getCACertFile(), "r")
key = inf.read()
#only sending fqdn the requesting should already know the unqualified
# hostname
node = NodeConnectRequest(conf.getServerId(),
conf.getClientSecurePort(),
conf.getServerSecurePort(), key,
conf.getFqdn(), None)
cmdstring = 'node-connection-accepted'
fields = []
input = Input('cmd', cmdstring)
input2 = Input(
'connectRequest',
json.dumps(node, default=json_serializer.toJson, indent=4))
fields.append(input)
fields.append(input2)
fields.append(Input('version', "1"))
# this goes over client secure port , and we don't want the server to use
# cookies
response = self.postRequest(ServerRequest.prepareRequest(fields),
require_certificate_authentication=False,
disable_cookies=True)
return response
def serveHTTPSWithNoCertReq(serverState):
try:
httpd = HTTPSServerNoCertAuthentication(
request_handler.handlerForRequestWithNoCertReq, ServerConf(),
serverState)
sa = httpd.socket.getsockname()
log.info("Serving HTTPS for client communication on %s port %s..." %
(sa[0], sa[1]))
httpd.serve_forever()
except KeyboardInterrupt:
print "Interrupted"
serverState.doQuit()
except Exception:
#TODO better error handling of server errors during startup
print "HTTPS port %s already taken" % ServerConf().getClientSecurePort(
)
serverState.doQuit()
def run(self, serverState, request, response):
conf = ServerConf()
nodes = conf.getNodes()
resp = dict()
nodes = nodes.getNodesByPriority()
log.debug(nodes)
resp['connections'] = []
resp['broken_connections']=[]
for node in nodes:
if node.isConnected():
resp['connections'].append(node)
else:
resp['broken_connections'].append(node)
resp['sent_connect_requests'] = conf.getSentNodeConnectRequests()
resp['received_connect_requests']= conf.getNodeConnectRequests()
response.add("",resp)
log.info("Listed nodes")
def testStart2Servers(self):
numServers = 2
self.createConfFolders(numServers)
hostname = gethostname()
node0HttpsPort = 13807
node1HttpsPort = 13808
node0HttpPort = 14807
node1HttpPort = 14808
for i in range(numServers):
args = ['../../../../cpc-server','-c',self.serverConfs[i],'start'] #doing cpc.server.server.forkAndRun(cf, debug ) directly here will will for some strange reason mess up things when shutting down, the process wont shutdown
subprocess.call(args)
time.sleep(2)
#connect node 0 to node 1
args = ['../../../../cpc-server','-c',self.serverConfs[0],'connnect-server',hostname,str(node1HttpPort)] #doing cpc.server.server.forkAndRun(cf, debug ) directly here will will for some strange reason mess up things when shutting down, the process wont shutdown
subprocess.call(args)
args = ['../../../../cpc-server','-c',self.serverConfs[1],'trust',hostname,str(node0HttpsPort)] #doing cpc.server.server.forkAndRun(cf, debug ) directly here will will for some strange reason mess up things when shutting down, the process wont shutdown
subprocess.call(args)
#verify existense of of nodes in each conf file
conf1 = ServerConf(confdir=self.serverConfs[0], reload=True)
node0Nodes = conf1.getNodes()
self.assertTrue(node0Nodes.exists(hostname,node1HttpsPort))
conf2 = ServerConf(confdir=self.serverConfs[1],reload=True)
node1Nodes = conf2.getNodes()
self.assertTrue(node1Nodes.exists(hostname,node0HttpsPort))
#do a network topology call
conf = ConnectionBundle(confdir=self.serverConfs[0],reload=True)
client = ClientMessage()
topology =ProcessedResponse(client.networkTopology()).getData()
self.assertEquals(topology.size() ,2)
def run(self, serverState, request, response):
conf = ServerConf()
nodes = conf.getNodes()
resp = dict()
nodes = nodes.getNodesByPriority()
log.debug(nodes)
resp['connections'] = []
resp['broken_connections'] = []
for node in nodes:
if node.isConnected():
resp['connections'].append(node)
else:
resp['broken_connections'].append(node)
resp['sent_connect_requests'] = conf.getSentNodeConnectRequests()
resp['received_connect_requests'] = conf.getNodeConnectRequests()
response.add("", resp)
log.info("Listed nodes")
def __init__(self,host=None,port=None):
self.conf = ServerConf()
self.host = host
self.port = port
if self.host == None:
self.host = self.conf.getServerHost()
if self.port == None:
self.port = self.conf.getServerSecurePort()
self.privateKey = self.conf.getPrivateKey()
self.keychain = self.conf.getCaChainFile()
def requestNetworkTopology(topology,serverState=None):
"""
Asks each neigbouring node for their network topology
inputs:
topology:Nodes The list of the topology generated so far
serverState:ServerState
if provided worker states are fetched.
since this method is called by getNetworkTopology() which in turn
is called from places where we do not pass (and don't want) the serverState
we provide this option. Also it is not needed as the calling server always
knows the most up to date state of its own workers.
"""
conf = ServerConf()
thisNode = Node.getSelfNode(conf)
thisNode.setNodes(conf.getNodes())
topology.addNode(thisNode)
if serverState:
thisNode.workerStates = WorkerStateHandler.getConnectedWorkers(serverState.getWorkerStates())
for node in thisNode.getNodes().nodes.itervalues():
if topology.exists(node.getId()) == False:
#connect to correct node
if node.isConnected():
try:
clnt = DirectServerMessage(node,conf=conf)
#send along the current topology
rawresp = clnt.networkTopology(topology)
processedResponse = ProcessedResponse(rawresp)
topology = processedResponse.getData()
except ServerConnectionError as e:
#we cannot connect to the node,
# and its marked as unreachable
#we must still add it to the topology
log.error("node %s unreachable when asking for network "
"topology: error was %s"%(node.getId(),e.__str__()))
topology.addNode(node)
#todo notify in topology that this node is not connected?
return topology
def run(self, serverState, request, response):
conf = ServerConf()
#nodeReqs = copy.deepcopy(conf.getNodeConnectRequests())
nodeReqs = conf.getNodeConnectRequests()
connected = []
N = 0
for nodeConnectRequest in nodeReqs.nodes.values():
N += 1
ScGrantNodeConnection.grant(nodeConnectRequest.getId())
ret = {}
connectedNodes = conf.getNodes()
for node in connectedNodes.nodes.itervalues():
connected.append(node)
ret['connected'] = connected
ScGrantNodeConnection.establishConnectionsAndBroadcastTopology(
serverState,connected)
response.add("", ret)
log.info("Granted node connection for %d nodes" % (N))
def run(self, serverState, request, response):
conf = ServerConf()
#nodeReqs = copy.deepcopy(conf.getNodeConnectRequests())
nodeReqs = conf.getNodeConnectRequests()
connected = []
N = 0
for nodeConnectRequest in nodeReqs.nodes.values():
N += 1
ScGrantNodeConnection.grant(nodeConnectRequest.getId())
ret = {}
connectedNodes = conf.getNodes()
for node in connectedNodes.nodes.itervalues():
connected.append(node)
ret['connected'] = connected
ScGrantNodeConnection.establishConnectionsAndBroadcastTopology(
serverState, connected)
response.add("", ret)
log.info("Granted node connection for %d nodes" % (N))
def runServer(logLevel=None, doFork=True):
"""
Starts the server process
logLevel prod|debug|trace Determines the log level
doFork Forks and detaches the process and runs\
it as a daemon
"""
conf = ServerConf()
conf.setMode(logLevel)
# initialize the server state before forking.
serverState = ServerState(conf)
serverState.read()
pid = 0
try:
# First fork so the parent can exit.
if doFork:
pid = os.fork()
except OSError, e:
raise Error, "%s [%d]" % (e.strerror, e.errno)
def reestablishConnections(serverState):
'''
Tries to periodically check for nodes that have gone down and reestablish
connections to them
'''
log.log(cpc.util.log.TRACE,"Starting reestablish connection thread")
conf = ServerConf()
while True:
for node in conf.getNodes().nodes.itervalues():
if not node.isConnected():
establishInboundConnection(node,serverState)
establishOutboundConnection(node)
if not node.isConnected():
log.log(cpc.util.log.TRACE,("Tried to reestablish a "
"connection"
" to %s but failed "%node.toString()))
reconnectInterval = conf.getReconnectInterval()
time.sleep(reconnectInterval)
def addNodeAccepted(self):
conf = ServerConf()
inf = open(conf.getCACertFile(), "r")
key = inf.read()
#only sending fqdn the requesting should already know the unqualified
# hostname
node = NodeConnectRequest(conf.getServerId(),
conf.getClientSecurePort(),
conf.getServerSecurePort(),
key,
conf.getFqdn(),
None)
cmdstring ='node-connection-accepted'
fields = []
input=Input('cmd',cmdstring)
input2=Input('connectRequest',
json.dumps(node,default=json_serializer.toJson,indent=4))
fields.append(input)
fields.append(input2)
fields.append(Input('version', "1"))
# this goes over client secure port , and we don't want the server to use
# cookies
response= self.postRequest(ServerRequest.prepareRequest(fields),
require_certificate_authentication=False,
disable_cookies=True)
return response
def setupClient(self):
'''
Creates a connection bundle for the Client and worker
@returns ConnectionBundle
'''
connectionBundle = ConnectionBundle(create=True, fqdn=self.cn)
serverConf = ServerConf()
#generate random ascii string
randstring = ''.join(
random.choice(string.ascii_uppercase + string.digits) for x in
range(6))
tempDir = "%s/tmp/%s" % (self.conf.getConfDir(), randstring)
privKeyFile = "%s/priv.pem" % tempDir
pubKeyFile = "%s/pub.pem" % tempDir
certReqConfigFile = "%s/cert_req.txt" % tempDir
certFile = "%s/cert.pem" % tempDir
os.makedirs(tempDir) #we create a temp dir for intermediate files
self._generateKeyPair(privKeyFile=privKeyFile, pubKeyFile=pubKeyFile)
self._generateCertReqConf(
distinguished_cn="%s_%s" % (connectionBundle.CN_ID, self.cn),
certReqConfigFile=certReqConfigFile)
self._generateCert(privKeyFile, certFile, certReqConfigFile)
#now we need to read everything in to the connection bundle
connectionBundle.setPrivateKey(open(privKeyFile, 'r').read())
connectionBundle.setPublicKey(open(pubKeyFile, 'r').read())
connectionBundle.setCert(open(certFile, 'r').read())
connectionBundle.setCaCert(open(self.conf.getCACertFile(), "r").read())
shutil.rmtree(tempDir)
connectionBundle.setClientSecurePort(
serverConf.getClientSecurePort())
connectionBundle.setServerSecurePort(
serverConf.getServerSecurePort())
connectionBundle.setHostname(ServerConf().getHostName())
return connectionBundle
def run(self, serverState, request, response):
#get the connection params for this node
newParams =json.loads(request.getParam("connectionParams"))
conf = ServerConf()
nodes = conf.getNodes()
if nodes.exists(newParams['serverId']):
node = nodes.get(newParams['serverId'])
node.setHostname(newParams['hostname'])
node.setServerSecurePort(newParams['server_secure_port'])
node.setClientSecurePort(newParams['client_secure_port'])
node.setQualifiedName(newParams['fqdn'])
#Needed so that we write changes to conf file
conf.removeNode(node.server_id)
conf.addNode(node)
#update the network topology
ServerToServerMessage.getNetworkTopology(resetCache=True)
response.add("Updated connection parameters")
log.info("Updated connection params for %s"%node.toString())
else:
response.add("Requested update for node %s but this node " \
"is not a neigbouring node "%newParams[
'serverId'],
status="ERROR")
log.error("Failed updating connection params for %s"%newParams['serverId'])
def grant(key): #key is server-id
conf = ServerConf()
nodeConnectRequests = conf.getNodeConnectRequests()
if nodeConnectRequests.exists(key):
nodeToAdd = nodeConnectRequests.get(key) #this returns a nodeConnectRequest object
serv = RawServerMessage(nodeToAdd.getHostname(),
nodeToAdd.getClientSecurePort())
#letting the requesting node know that it is accepted
#also sending this servers connection parameters
resp = serv.addNodeAccepted()
conf.addNode(Node(nodeToAdd.server_id,
nodeToAdd.getClientSecurePort(),
nodeToAdd.getServerSecurePort(),
nodeToAdd.getQualifiedName(),nodeToAdd.getHostname()))
#trust the key
openssl = OpenSSL(conf)
openssl.addCa(nodeToAdd.key)
nodeConnectRequests.removeNode(nodeToAdd.getId())
conf.set('node_connect_requests', nodeConnectRequests)
return True
else:
return False
def run(self, serverState, request, response):
conf = ServerConf()
sentConnectRequests = conf.getSentNodeConnectRequests()
node = json.loads(request.getParam('connectRequest'),
object_hook=json_serializer.fromJson)
if(sentConnectRequests.exists(node.getId())):
nodeToAdd = sentConnectRequests.get(node.getId())
conf.addNode(Node(node.server_id,
node.getClientSecurePort(),
node.getServerSecurePort(),
node .getQualifiedName(),nodeToAdd.getHostname()))
#conf.addNode(nodeToAdd)
openssl = OpenSSL(conf)
openssl.addCa(node.key)
sentConnectRequests.removeNode(node.getId())
conf.set('sent_node_connect_requests', sentConnectRequests)
# need to send back a status in the data notifying ok
response.add('Connection to node %s established'%node.toString())
log.info("Node connection accepted")
#add it to the node list
else:
response.add('No previous node request sent for host %s' %node
.toString())
log.info("Node connection not accepted")
def _sendResponse(self,retmsg,closeConnection=True,revertSocket=False):
conf = ServerConf()
rets = retmsg.render()
self.log.log(cpc.util.log.TRACE,"Done. Reply message is: '%s'\n"%rets)
self.send_response(self.responseCode)
self.send_header("content-length", len(rets))
if 'originating-server-id' not in retmsg.headers:
self.send_header("originating-server-id", conf.getServerId())
if self.set_cookie is not None:
self.send_header('Set-Cookie', self.set_cookie)
for key,value in retmsg.headers.iteritems():
self.send_header(key, value)
#if the server should be reverted we need to return a keep-alive but
# also get out of the request handling loop
if(revertSocket):
self.send_header("Connection", "keep-alive")
#this takes us out of the request handling loop
self.close_connection = 1
elif closeConnection:
self.send_header("Connection", "close")
#this is for keeping inbound connections alive
else:
self.send_header("Connection", "keep-alive")
self.end_headers()
if isinstance(rets, mmap.mmap):
# we need this because strings don't support reads, and
# write expects this.
str = rets.read(len(rets))
#print str
self.wfile.write(str)
else:
self.wfile.write(rets)
retmsg.close()
return True
def createConfFolders(self,num):
server_secure_port = 13807
client_secure_port = 14807
for i in range(num):
self.createConfFolder(i)
server_conf = ServerConf(confdir=self.serverConfs[i],reload=True)
server_conf.set('server_https_port',str(server_secure_port))
server_conf.set('client_secure_port',str(client_secure_port))
server_conf.set('mode',"debug")
client_conf = ConnectionBundle(confdir=self.serverConfs[i],reload=True)
client_conf.set('client_secure_port',str(client_secure_port))
client_conf.set('client_https_port',str(server_secure_port))
client_secure_port +=1
server_secure_port +=1
def run(self, serverState, request, response):
conf = ServerConf()
info = dict()
info['fqdn'] = conf.getFqdn()
info['version'] = __version__
try:
conf.getServerId()
info['serverId'] = conf.getServerId()
info['server_secure_port'] = conf.getServerSecurePort()
info['client_secure_port'] = conf.getClientSecurePort()
except ServerIdNotFoundException as e:
info['serverId'] = "ERROR: %s"%e.str
response.add("",info)