def _loop(self): self.elapsedTime = time.time() - self.startTime - self.pausedTime if self.bytesTransferred >= self.transferLimit: self.stop() return if self.elapsedTime >= self.duration: self.stop() return if self.jobState == JobState.RUNNING: numRequests = min(int(config.parameter("network", "clients.max")), abs(int(math.ceil(self.clientFunction(time.time()))))) try: timeBetween = 1.0/numRequests except ZeroDivisionError: reactor.callLater(0, self._loop) # avoid recursing return for i in range(0, numRequests): # queue may be empty if there are no URLs in the job spec try: request = self.httpClientRequestQueue.get(False) except Empty: self.stop() return self.httpClientRequestQueue.put(request) reactor.callLater(timeBetween, self._request, request[0], request[1], request[2], request[3], request[4], request[5]) reactor.callLater(1, self.iterator)
def _loop(self): if self.jobState == JobState.RUNNING: # calculates client function at time t clientFunctionResult = min(int(config.parameter("network", "clients.max")), abs(int(math.ceil(self.clientFunction(time.time()))))) # only add clients if clientFunc(t) calls for more clients than are # currently in the system. this may not always be the case -- if a # traffic slowdown is being simulated, clients in the system need to # die off. an example of this is a sine wave; at t=1 there will be more # clients than t=1.5 if clientFunctionResult > self.clients: for i in range(self.clients, clientFunctionResult): # take a request item off the queue and then move it right up top. # this seems nonsensical but it makes the clients fetch all the URLs # in the order of the queue, and easily allows n clients to handle n+1 # URLs # # queue may be empty if there are no URLs in the job spec try: request = self.httpClientRequestQueue.get(False) except Empty: self.stop() return self.httpClientRequestQueue.put(request) self.clients = self.clients + 1 reactor.callLater(0, self._request, request[0], request[1], request[2], request[3], request[4], request[5])
def startServer(port): logging.basicConfig(level=eval("logging.%s" % config.parameter("log", "level"))) log = logging.getLogger("main") twistedLog.startLogging(sys.stderr) # since master servers will ping back to the slave upon connection, # start listening for HTTP requests before trying to connect up to the master log.debug("Listening on port %s, starting reactor" % config.parameter("network", "port")) reactor.listenTCP(config.parameter("network", "port", type=int), createRestApi()) if config.parameter("misc", "standalone", type=bool) != True: # connect to master server in INI file scheme = config.parameter("master", "scheme") host = config.parameter("master", "host") port = config.parameter("master", "port", type=int) path = config.parameter("master", "path") slaveSpec = SlaveSpec() slaveSpec.host = socket.gethostname() slaveSpec.port = config.parameter("network", "port", type=int) slaveSpec.path = "" slaveSpec.maxRequestsPerSec = config.parameter("network", "clients.max", type=int) masterUrl = "%s://%s:%d/%s/slave" % (scheme, host, port, path) log.info("Connecting to master: %s" % masterUrl) try: request = RestApiClient.POST(str(masterUrl), slaveSpec.toJson(), timeout=10, credentials=("slave", "slave")) yield request except: log.error("Could not connect to master. Exiting") reactor.stop() return slaveId = int(request.result) log.info("Connected to master. This server is slave ID %d" % slaveId)
def createRestApi(): try: if config.parameter("network", "authentication", type=bool) == False: log.warn("HTTP Authentication disabled") return server.Site(siteRoot) else: raise except: rootWrapper = guard.HTTPAuthSessionWrapper(Portal(RootRealm(), [RootDBChecker(db)]), [guard.BasicCredentialFactory("thundercloud root management")]) return server.Site(rootWrapper)
def startServer(port): logging.basicConfig(level=eval("logging.%s" % config.parameter("log", "level"))) log = logging.getLogger("main") twistedLog.startLogging(sys.stderr) # add slaves in the INI file if they're around and add-able slaves = {} for (key, val) in config.section("slave"): slaveNo = key[:key.find(".")] attr = key[key.find(".") + 1:] try: setattr(slaves[slaveNo], attr, val) except KeyError: slaves[slaveNo] = SlaveSpec() setattr(slaves[slaveNo], attr, val) deferreds = [] for slaveSpec in slaves.itervalues(): log.info("Connecting slave: %s" % slaveSpec) deferreds.append(Orchestrator.registerSlave(slaveSpec)) deferredList = DeferredList(deferreds) yield deferredList result = deferredList.result if result is not None: for (success, slaveId) in result: if success == False: log.error("Error adding slave") else: log.info("Slave %s connected" % slaveId) log.debug("Listening on port %s", port) reactor.listenTCP(port, createRestApi()) # XXX userSpec = UserSpec() userSpec.username = "******" userSpec.password = "******" yield UserManager.create(userSpec) userSpec2 = UserSpec() userSpec2.username = "******" userSpec2.password = "******" yield UserManager.create(userSpec2)
def startServer(port): logging.basicConfig(level=eval("logging.%s" % config.parameter("log", "level"))) log = logging.getLogger("main") twistedLog.startLogging(sys.stderr) # add slaves in the INI file if they're around and add-able slaves = {} for (key, val) in config.section("slave"): slaveNo = key[:key.find(".")] attr = key[key.find(".")+1:] try: setattr(slaves[slaveNo], attr, val) except KeyError: slaves[slaveNo] = SlaveSpec() setattr(slaves[slaveNo], attr, val) deferreds = [] for slaveSpec in slaves.itervalues(): log.info("Connecting slave: %s" % slaveSpec) deferreds.append(Orchestrator.registerSlave(slaveSpec)) deferredList = DeferredList(deferreds) yield deferredList result = deferredList.result if result is not None: for (success, slaveId) in result: if success == False: log.error("Error adding slave") else: log.info("Slave %s connected" % slaveId) log.debug("Listening on port %s", port) reactor.listenTCP(port, createRestApi()) # XXX userSpec = UserSpec() userSpec.username = "******" userSpec.password = "******" yield UserManager.create(userSpec) userSpec2 = UserSpec() userSpec2.username = "******" userSpec2.password = "******" yield UserManager.create(userSpec2)
masterUrl = "%s://%s:%d/%s/slave" % (scheme, host, port, path) log.info("Connecting to master: %s" % masterUrl) try: request = RestApiClient.POST(str(masterUrl), slaveSpec.toJson(), timeout=10, credentials=("slave", "slave")) yield request except: log.error("Could not connect to master. Exiting") reactor.stop() return slaveId = int(request.result) log.info("Connected to master. This server is slave ID %d" % slaveId) if __name__ == "__main__": try: config.readConfig(sys.argv[1]) except: print "No config file specified, exiting" sys.exit(1) reactor.callWhenRunning(startServer, config.parameter("network", "port", type=int)) reactor.run()
from twisted.internet import reactor from thundercloud import config import thunderserver import sys if __name__ == "__main__": try: config.readConfig(sys.argv[1]) except: print "No config file specified, exiting" sys.exit(1) reactor.callWhenRunning(thunderserver.startServer, config.parameter("network", "port", type=int)) reactor.run()