class RequestQueuerTestCase(unittest.TestCase): def setUp(self): self.deferred = Deferred() self.mini_web_server = MiniWebServer() self.rq = RequestQueuer(max_requests_per_host_per_second=3, max_simultaneous_requests_per_host=5) def tearDown(self): return self.mini_web_server.shutdown() def testRequestQueuerOnSuccess(self): d = self.rq.getPage("http://127.0.0.1:8080/helloworld", timeout=5) return d def testRequestQueuerOnFailure(self): d = self.rq.getPage("http://0.0.0.0:99", timeout=5) d.addErrback(self._getPageErrback) return d def testHostMaxRequestsPerSecond(self,): self.failUnlessEqual( self.rq.getHostMaxRequestsPerSecond("example.com"), 3) self.rq.setHostMaxRequestsPerSecond("example2.com", 7) self.failUnlessEqual( self.rq.getHostMaxRequestsPerSecond("example2.com"), 7) def testHostMaxSimultaneousRequests(self,): self.failUnlessEqual( self.rq.getHostMaxSimultaneousRequests("example.com"), 5) self.rq.setHostMaxSimultaneousRequests("example2.com", 11) self.failUnlessEqual( self.rq.getHostMaxSimultaneousRequests("example2.com"), 11) def testActive(self): self.failUnlessEqual(isinstance(self.rq.getActive(), int), True) def testPending(self): self.failUnlessEqual(isinstance(self.rq.getPending(), int), True) def testActiveRequestsByHost(self): self.failUnlessEqual(isinstance(self.rq.getActiveRequestsByHost(), dict), True) def testPendingRequestsByHost(self): self.failUnlessEqual(isinstance(self.rq.getPendingRequestsByHost(), dict), True) def _getPageErrback(self, error): return True
class BaseServer(object): exposed_functions = [] exposed_function_resources = {} logging_handler = None shutdown_trigger_id = None uuid = uuid4().hex start_time = time.time() active_jobs = {} reserved_arguments = [ "reservation_function_name", "reservation_created", "reservation_next_request", "reservation_error"] functions = {} delta_functions = {} categories = {} fast_cache = {} function_resource = None def __init__(self, config, pg=None): # Resource Mappings self.service_mapping = config["service_mapping"] self.service_args_mapping = config["service_args_mapping"] self.inverted_args_mapping = dict([(s[0], invert(s[1])) for s in self.service_args_mapping.items()]) # configure stats self.stats = config.get('stats', stats.stats) stats.stats = self.stats # Request Queuer self.rq = RequestQueuer( max_simultaneous_requests=config["max_simultaneous_requests"], max_requests_per_host_per_second=config["max_requests_per_host_per_second"], max_simultaneous_requests_per_host=config["max_simultaneous_requests_per_host"]) self.rq.setHostMaxRequestsPerSecond("127.0.0.1", 0) self.rq.setHostMaxSimultaneousRequests("127.0.0.1", 0) if pg is None: self.pg = PageGetter(rq=self.rq) else: self.pg = pg def start(self): start_deferred = Deferred() reactor.callWhenRunning(self._baseStart, start_deferred) return start_deferred def _baseStart(self, start_deferred): logger.debug("Starting Base components.") self.shutdown_trigger_id = reactor.addSystemEventTrigger( 'before', 'shutdown', self.shutdown) start_deferred.callback(True) @inlineCallbacks def shutdown(self): while self.rq.getPending() > 0 or self.rq.getActive() > 0: logger.debug("%s requests active, %s requests pending." % ( self.rq.getPending(), self.rq.getActive() )) shutdown_deferred = Deferred() # Call the Deferred after a second to continue the loop. reactor.callLater(1, shutdown_deferred.callback) yield shutdown_deferred self.shutdown_trigger_id = None logger.critical("Server shut down.") logger.removeHandler(self.logging_handler) returnValue(True) def getManholeFactory(self, namespace, **passwords): realm = manhole_ssh.TerminalRealm() def getManhole(_): return manhole.Manhole(namespace) realm.chainedProtocolFactory.protocolFactory = getManhole p = portal.Portal(realm) p.registerChecker( checkers.InMemoryUsernamePasswordDatabaseDontUse(**passwords)) f = manhole_ssh.ConchFactory(p) return f def delta(self, func, handler): self.delta_functions[id(func)] = handler def expose(self, *args, **kwargs): return self.makeCallable(expose=True, *args, **kwargs) @inlineCallbacks def executeJob(self, job): dotted_function = '.'.join(job.function_name.split('/')) timer = 'job.%s.duration' % (dotted_function) self.stats.timer.start(timer, 0.5) self.stats.timer.start('job.time', 0.1) if not job.mapped: job = self.mapJob(job) f = self.functions[job.function_name] if job.uuid is not None: self.active_jobs[job.uuid] = True if f["get_job_uuid"]: job.kwargs["job_uuid"] = job.uuid if f["check_fast_cache"]: job.kwargs["fast_cache"] = job.fast_cache try: data = yield self.executeFunction(job.function_name, **job.kwargs) except NegativeCacheException: self.stats.timer.stop(timer) self.stats.timer.stop('job.time') raise except QueueTimeoutException: self.stats.timer.stop(timer) self.stats.timer.stop('job.time') raise except Exception, e: self.stats.increment('job.%s.failure' % dotted_function) self.stats.timer.stop(timer) self.stats.timer.stop('job.time') raise finally: