class RequestQueuerTestCase(unittest.TestCase):
def setUp(self):
self.deferred = Deferred()
self.mini_web_server = MiniWebServer()
self.rq = RequestQueuer(max_requests_per_host_per_second=3, max_simultaneous_requests_per_host=5)
def tearDown(self):
return self.mini_web_server.shutdown()
def testRequestQueuerOnSuccess(self):
d = self.rq.getPage("http://127.0.0.1:8080/helloworld", timeout=5)
return d
def testRequestQueuerOnFailure(self):
d = self.rq.getPage("http://0.0.0.0:99", timeout=5)
d.addErrback(self._getPageErrback)
return d
def testHostMaxRequestsPerSecond(self,):
self.failUnlessEqual(
self.rq.getHostMaxRequestsPerSecond("example.com"), 3)
self.rq.setHostMaxRequestsPerSecond("example2.com", 7)
self.failUnlessEqual(
self.rq.getHostMaxRequestsPerSecond("example2.com"), 7)
def testHostMaxSimultaneousRequests(self,):
self.failUnlessEqual(
self.rq.getHostMaxSimultaneousRequests("example.com"), 5)
self.rq.setHostMaxSimultaneousRequests("example2.com", 11)
self.failUnlessEqual(
self.rq.getHostMaxSimultaneousRequests("example2.com"),
11)
def testActive(self):
self.failUnlessEqual(isinstance(self.rq.getActive(), int), True)
def testPending(self):
self.failUnlessEqual(isinstance(self.rq.getPending(), int), True)
def testActiveRequestsByHost(self):
self.failUnlessEqual(isinstance(self.rq.getActiveRequestsByHost(), dict), True)
def testPendingRequestsByHost(self):
self.failUnlessEqual(isinstance(self.rq.getPendingRequestsByHost(), dict), True)
def _getPageErrback(self, error):
return True
class BaseServer(object):
exposed_functions = []
exposed_function_resources = {}
logging_handler = None
shutdown_trigger_id = None
uuid = uuid4().hex
start_time = time.time()
active_jobs = {}
reserved_arguments = [
"reservation_function_name",
"reservation_created",
"reservation_next_request",
"reservation_error"]
functions = {}
delta_functions = {}
categories = {}
fast_cache = {}
function_resource = None
def __init__(self, config, pg=None):
# Resource Mappings
self.service_mapping = config["service_mapping"]
self.service_args_mapping = config["service_args_mapping"]
self.inverted_args_mapping = dict([(s[0], invert(s[1]))
for s in self.service_args_mapping.items()])
# configure stats
self.stats = config.get('stats', stats.stats)
stats.stats = self.stats
# Request Queuer
self.rq = RequestQueuer(
max_simultaneous_requests=config["max_simultaneous_requests"],
max_requests_per_host_per_second=config["max_requests_per_host_per_second"],
max_simultaneous_requests_per_host=config["max_simultaneous_requests_per_host"])
self.rq.setHostMaxRequestsPerSecond("127.0.0.1", 0)
self.rq.setHostMaxSimultaneousRequests("127.0.0.1", 0)
if pg is None:
self.pg = PageGetter(rq=self.rq)
else:
self.pg = pg
def start(self):
start_deferred = Deferred()
reactor.callWhenRunning(self._baseStart, start_deferred)
return start_deferred
def _baseStart(self, start_deferred):
logger.debug("Starting Base components.")
self.shutdown_trigger_id = reactor.addSystemEventTrigger(
'before',
'shutdown',
self.shutdown)
start_deferred.callback(True)
@inlineCallbacks
def shutdown(self):
while self.rq.getPending() > 0 or self.rq.getActive() > 0:
logger.debug("%s requests active, %s requests pending." % (
self.rq.getPending(),
self.rq.getActive()
))
shutdown_deferred = Deferred()
# Call the Deferred after a second to continue the loop.
reactor.callLater(1, shutdown_deferred.callback)
yield shutdown_deferred
self.shutdown_trigger_id = None
logger.critical("Server shut down.")
logger.removeHandler(self.logging_handler)
returnValue(True)
def getManholeFactory(self, namespace, **passwords):
realm = manhole_ssh.TerminalRealm()
def getManhole(_):
return manhole.Manhole(namespace)
realm.chainedProtocolFactory.protocolFactory = getManhole
p = portal.Portal(realm)
p.registerChecker(
checkers.InMemoryUsernamePasswordDatabaseDontUse(**passwords))
f = manhole_ssh.ConchFactory(p)
return f
def delta(self, func, handler):
self.delta_functions[id(func)] = handler
def expose(self, *args, **kwargs):
return self.makeCallable(expose=True, *args, **kwargs)
@inlineCallbacks
def executeJob(self, job):
dotted_function = '.'.join(job.function_name.split('/'))
timer = 'job.%s.duration' % (dotted_function)
self.stats.timer.start(timer, 0.5)
self.stats.timer.start('job.time', 0.1)
if not job.mapped:
job = self.mapJob(job)
f = self.functions[job.function_name]
if job.uuid is not None:
self.active_jobs[job.uuid] = True
if f["get_job_uuid"]:
job.kwargs["job_uuid"] = job.uuid
if f["check_fast_cache"]:
job.kwargs["fast_cache"] = job.fast_cache
try:
data = yield self.executeFunction(job.function_name, **job.kwargs)
except NegativeCacheException:
self.stats.timer.stop(timer)
self.stats.timer.stop('job.time')
raise
except QueueTimeoutException:
self.stats.timer.stop(timer)
self.stats.timer.stop('job.time')
raise
except Exception, e:
self.stats.increment('job.%s.failure' % dotted_function)
self.stats.timer.stop(timer)
self.stats.timer.stop('job.time')
raise
finally:
