def _loop(self):
self.elapsedTime = time.time() - self.startTime - self.pausedTime
if self.bytesTransferred >= self.transferLimit:
self.stop()
return
if self.elapsedTime >= self.duration:
self.stop()
return
if self.jobState == JobState.RUNNING:
numRequests = min(int(config.parameter("network", "clients.max")), abs(int(math.ceil(self.clientFunction(time.time())))))
try:
timeBetween = 1.0/numRequests
except ZeroDivisionError:
reactor.callLater(0, self._loop) # avoid recursing
return
for i in range(0, numRequests):
# queue may be empty if there are no URLs in the job spec
try:
request = self.httpClientRequestQueue.get(False)
except Empty:
self.stop()
return
self.httpClientRequestQueue.put(request)
reactor.callLater(timeBetween, self._request,
request[0], request[1], request[2],
request[3], request[4], request[5])
reactor.callLater(1, self.iterator)
def _loop(self):
if self.jobState == JobState.RUNNING:
# calculates client function at time t
clientFunctionResult = min(int(config.parameter("network", "clients.max")), abs(int(math.ceil(self.clientFunction(time.time())))))
# only add clients if clientFunc(t) calls for more clients than are
# currently in the system. this may not always be the case -- if a
# traffic slowdown is being simulated, clients in the system need to
# die off. an example of this is a sine wave; at t=1 there will be more
# clients than t=1.5
if clientFunctionResult > self.clients:
for i in range(self.clients, clientFunctionResult):
# take a request item off the queue and then move it right up top.
# this seems nonsensical but it makes the clients fetch all the URLs
# in the order of the queue, and easily allows n clients to handle n+1
# URLs
#
# queue may be empty if there are no URLs in the job spec
try:
request = self.httpClientRequestQueue.get(False)
except Empty:
self.stop()
return
self.httpClientRequestQueue.put(request)
self.clients = self.clients + 1
reactor.callLater(0, self._request,
request[0], request[1], request[2],
request[3], request[4], request[5])
def startServer(port):
logging.basicConfig(level=eval("logging.%s" %
config.parameter("log", "level")))
log = logging.getLogger("main")
twistedLog.startLogging(sys.stderr)
# since master servers will ping back to the slave upon connection,
# start listening for HTTP requests before trying to connect up to the master
log.debug("Listening on port %s, starting reactor" %
config.parameter("network", "port"))
reactor.listenTCP(config.parameter("network", "port", type=int),
createRestApi())
if config.parameter("misc", "standalone", type=bool) != True:
# connect to master server in INI file
scheme = config.parameter("master", "scheme")
host = config.parameter("master", "host")
port = config.parameter("master", "port", type=int)
path = config.parameter("master", "path")
slaveSpec = SlaveSpec()
slaveSpec.host = socket.gethostname()
slaveSpec.port = config.parameter("network", "port", type=int)
slaveSpec.path = ""
slaveSpec.maxRequestsPerSec = config.parameter("network",
"clients.max",
type=int)
masterUrl = "%s://%s:%d/%s/slave" % (scheme, host, port, path)
log.info("Connecting to master: %s" % masterUrl)
try:
request = RestApiClient.POST(str(masterUrl),
slaveSpec.toJson(),
timeout=10,
credentials=("slave", "slave"))
yield request
except:
log.error("Could not connect to master. Exiting")
reactor.stop()
return
slaveId = int(request.result)
log.info("Connected to master. This server is slave ID %d" % slaveId)
def createRestApi():
try:
if config.parameter("network", "authentication", type=bool) == False:
log.warn("HTTP Authentication disabled")
return server.Site(siteRoot)
else:
raise
except:
rootWrapper = guard.HTTPAuthSessionWrapper(Portal(RootRealm(), [RootDBChecker(db)]), [guard.BasicCredentialFactory("thundercloud root management")])
return server.Site(rootWrapper)
def startServer(port):
logging.basicConfig(level=eval("logging.%s" % config.parameter("log", "level")))
log = logging.getLogger("main")
twistedLog.startLogging(sys.stderr)
# since master servers will ping back to the slave upon connection,
# start listening for HTTP requests before trying to connect up to the master
log.debug("Listening on port %s, starting reactor" % config.parameter("network", "port"))
reactor.listenTCP(config.parameter("network", "port", type=int), createRestApi())
if config.parameter("misc", "standalone", type=bool) != True:
# connect to master server in INI file
scheme = config.parameter("master", "scheme")
host = config.parameter("master", "host")
port = config.parameter("master", "port", type=int)
path = config.parameter("master", "path")
slaveSpec = SlaveSpec()
slaveSpec.host = socket.gethostname()
slaveSpec.port = config.parameter("network", "port", type=int)
slaveSpec.path = ""
slaveSpec.maxRequestsPerSec = config.parameter("network", "clients.max", type=int)
masterUrl = "%s://%s:%d/%s/slave" % (scheme, host, port, path)
log.info("Connecting to master: %s" % masterUrl)
try:
request = RestApiClient.POST(str(masterUrl), slaveSpec.toJson(), timeout=10, credentials=("slave", "slave"))
yield request
except:
log.error("Could not connect to master. Exiting")
reactor.stop()
return
slaveId = int(request.result)
log.info("Connected to master. This server is slave ID %d" % slaveId)
def startServer(port):
logging.basicConfig(level=eval("logging.%s" %
config.parameter("log", "level")))
log = logging.getLogger("main")
twistedLog.startLogging(sys.stderr)
# add slaves in the INI file if they're around and add-able
slaves = {}
for (key, val) in config.section("slave"):
slaveNo = key[:key.find(".")]
attr = key[key.find(".") + 1:]
try:
setattr(slaves[slaveNo], attr, val)
except KeyError:
slaves[slaveNo] = SlaveSpec()
setattr(slaves[slaveNo], attr, val)
deferreds = []
for slaveSpec in slaves.itervalues():
log.info("Connecting slave: %s" % slaveSpec)
deferreds.append(Orchestrator.registerSlave(slaveSpec))
deferredList = DeferredList(deferreds)
yield deferredList
result = deferredList.result
if result is not None:
for (success, slaveId) in result:
if success == False:
log.error("Error adding slave")
else:
log.info("Slave %s connected" % slaveId)
log.debug("Listening on port %s", port)
reactor.listenTCP(port, createRestApi())
# XXX
userSpec = UserSpec()
userSpec.username = "******"
userSpec.password = "******"
yield UserManager.create(userSpec)
userSpec2 = UserSpec()
userSpec2.username = "******"
userSpec2.password = "******"
yield UserManager.create(userSpec2)
def startServer(port):
logging.basicConfig(level=eval("logging.%s" % config.parameter("log", "level")))
log = logging.getLogger("main")
twistedLog.startLogging(sys.stderr)
# add slaves in the INI file if they're around and add-able
slaves = {}
for (key, val) in config.section("slave"):
slaveNo = key[:key.find(".")]
attr = key[key.find(".")+1:]
try:
setattr(slaves[slaveNo], attr, val)
except KeyError:
slaves[slaveNo] = SlaveSpec()
setattr(slaves[slaveNo], attr, val)
deferreds = []
for slaveSpec in slaves.itervalues():
log.info("Connecting slave: %s" % slaveSpec)
deferreds.append(Orchestrator.registerSlave(slaveSpec))
deferredList = DeferredList(deferreds)
yield deferredList
result = deferredList.result
if result is not None:
for (success, slaveId) in result:
if success == False:
log.error("Error adding slave")
else:
log.info("Slave %s connected" % slaveId)
log.debug("Listening on port %s", port)
reactor.listenTCP(port, createRestApi())
# XXX
userSpec = UserSpec()
userSpec.username = "******"
userSpec.password = "******"
yield UserManager.create(userSpec)
userSpec2 = UserSpec()
userSpec2.username = "******"
userSpec2.password = "******"
yield UserManager.create(userSpec2)
masterUrl = "%s://%s:%d/%s/slave" % (scheme, host, port, path)
log.info("Connecting to master: %s" % masterUrl)
try:
request = RestApiClient.POST(str(masterUrl),
slaveSpec.toJson(),
timeout=10,
credentials=("slave", "slave"))
yield request
except:
log.error("Could not connect to master. Exiting")
reactor.stop()
return
slaveId = int(request.result)
log.info("Connected to master. This server is slave ID %d" % slaveId)
if __name__ == "__main__":
try:
config.readConfig(sys.argv[1])
except:
print "No config file specified, exiting"
sys.exit(1)
reactor.callWhenRunning(startServer,
config.parameter("network", "port", type=int))
reactor.run()
from twisted.internet import reactor
from thundercloud import config
import thunderserver
import sys
if __name__ == "__main__":
try:
config.readConfig(sys.argv[1])
except:
print "No config file specified, exiting"
sys.exit(1)
reactor.callWhenRunning(thunderserver.startServer, config.parameter("network", "port", type=int))
reactor.run()
