class WebClientAgentsPool:
def __init__(self, persistent=True):
self.persistent = persistent
self.agents = SortedCollection(key=lambda x: x.url.netloc)
self.pool = HTTPConnectionPool(reactor)
self.pool.maxPersistentPerHost = getattr(
settings, 'DTX_WEB_DEFER_MAX_PERSISTENT_PER_HOST', 8)
self.pool.cachedConnectionTimeout = getattr(
settings, 'DTX_WEB_DEFER_CONNECT_TIMEOUT', 10)
def createAgent(self, uri):
url = urlparse(uri)
try:
agent = self.agents.find(url.netloc)
log.msg(u'Using existing agent for {}'.format(url.netloc))
agent.url = url
self.agents.remove(agent)
return agent
except:
log.msg(u'Creating new agent for {}'.format(url.netloc))
agent = WebClientAgent(self, url)
return agent
def returnAgent(self, agent):
if (self.persistent):
log.msg(u'Returning agent to the pool'.format(self.url.netloc))
self.agents.insert(agent)
class WebClientAgentsPool:
def __init__(self, persistent=True):
self.persistent = persistent
self.agents = SortedCollection(key=lambda x: x.url.netloc)
self.pool = HTTPConnectionPool(reactor)
self.pool.maxPersistentPerHost = getattr(settings, "DTX_WEB_DEFER_MAX_PERSISTENT_PER_HOST", 8)
self.pool.cachedConnectionTimeout = getattr(settings, "DTX_WEB_DEFER_CONNECT_TIMEOUT", 10)
def createAgent(self, uri):
url = urlparse(uri)
try:
agent = self.agents.find(url.netloc)
log.msg(u"Using existing agent for {}".format(url.netloc))
agent.url = url
self.agents.remove(agent)
return agent
except:
log.msg(u"Creating new agent for {}".format(url.netloc))
agent = WebClientAgent(self, url)
return agent
def returnAgent(self, agent):
if self.persistent:
log.msg(u"Returning agent to the pool".format(self.url.netloc))
self.agents.insert(agent)
class DtxTwistedWebResource(Resource):
def __init__(self, pattern, urlconf):
with log.enter(obj=self) as tm:
Resource.__init__(self)
self.pattern = pattern
self.urlconf = urlconf
self.callbacks = SortedCollection(key=lambda x: x.name)
self._search_callbacks(urlconf)
for item in self.callbacks:
tm.msg(item)
def _add_callback(self, name):
try:
cb = self.callbacks.find(name)
cb.count += 1
except:
path = name.split('.')
mod = import_module('.'.join(path[:-1]))
fun = mod.__dict__[path[-1]]
if (not isinstance(fun, DtxTwistedWebCallbackDecorator)):
log.msg('Decorating {} ({})'.format(name, fun))
mod.__dict__[path[-1]] = DtxTwistedWebCallbackDecorator(fun)
self.callbacks.insert(DtxCallbackInfo(name))
def _search_callbacks(self, urlconf):
urlpatterns = urlconf.__dict__.get('urlpatterns', [])
for item in urlpatterns:
if (issubclass(item.__class__, urlresolvers.RegexURLPattern)):
callback_str = item.__dict__.get('_callback_str', None)
if (callback_str):
self._add_callback(callback_str)
else:
callback_str = '.'.join((item._callback.__module__, item._callback.__name__))
self._add_callback(callback_str)
item._callback = None
item._callback_str = callback_str
elif (issubclass(item.__class__, urlresolvers.RegexURLResolver)):
self._search_callbacks(item.urlconf_name)
def render_request(self, request, method):
with log.enter(obj=self) as tm:
request.method = method
request.url = urlparse.urlsplit(request.uri)
tm.msg(u'Resolving \'{}\' at {}'.format(unicode(request.url.path), self.urlconf))
try:
match = urlresolvers.resolve(unicode(request.url.path), urlconf=self.urlconf)
tm.msg(u'Matched {}'.format(match))
request.urlconf = self.urlconf
task.deferLater(reactor, 0, invokeResolverMatch, request, match)
return NOT_DONE_YET
except urlresolvers.Resolver404, ex:
tm.err(u'Not found')
request.setResponseCode(404)
return '404'
except BaseException, ex:
tm.err(traceback.format_exc())
request.setResponseCode(500)
return '500'
class RemoteConfig:
def __init__(self, urlconf):
self.urlconf = urlconf
self.workers = SortedCollection(key=lambda x: x.address)
self.index = 0
def __unicode__(self):
return u'{}: {}'.format(unicode(self.urlconf), len(self.workers))
def addWorker(self, address):
try:
self.workers.find(address)
except:
log.msg(u'Registering {} for {}'.format(address, self.urlconf))
self.workers.insert(RemoteWorker(address))
def delWorker(self, address):
try:
worker = self.workers.find(address)
log.msg(u'Unregistering {} for {}'.format(address, self.urlconf))
self.workers.remove(worker)
except:
pass
return len(self.workers)
def nextWorker(self):
with log.enter(obj=self) as tm:
if (not self.workers):
log.err(u'No registered workers for {}'.format(self.urlconf))
return None
start_index = self.index
while (42):
if (self.index > len(self.workers) - 1):
self.index = 0
worker = self.workers[self.index]
self.index += 1
if (worker.enabled()):
tm.msg(u'Returning {}/{}: {}'.format(
self.index, len(self.workers), worker))
return worker
if (start_index == self.index):
log.err(u'No free workers for {}'.format(self.urlconf))
return None
class RemoteConfig:
def __init__(self, urlconf):
self.urlconf = urlconf
self.workers = SortedCollection(key=lambda x: x.address)
self.index = 0
def __unicode__(self):
return u"{}: {}".format(unicode(self.urlconf), len(self.workers))
def addWorker(self, address):
try:
self.workers.find(address)
except:
log.msg(u"Registering {} for {}".format(address, self.urlconf))
self.workers.insert(RemoteWorker(address))
def delWorker(self, address):
try:
worker = self.workers.find(address)
log.msg(u"Unregistering {} for {}".format(address, self.urlconf))
self.workers.remove(worker)
except:
pass
return len(self.workers)
def nextWorker(self):
with log.enter(obj=self) as tm:
if not self.workers:
log.err(u"No registered workers for {}".format(self.urlconf))
return None
start_index = self.index
while 42:
if self.index > len(self.workers) - 1:
self.index = 0
worker = self.workers[self.index]
self.index += 1
if worker.enabled():
tm.msg(u"Returning {}/{}: {}".format(self.index, len(self.workers), worker))
return worker
if start_index == self.index:
log.err(u"No free workers for {}".format(self.urlconf))
return None
class RemoteNodes:
def __init__(self):
self.configs = SortedCollection(key=lambda x: x.urlconf)
log.msg(u"Registering static web sites")
for address, urlconf in getattr(settings, "DTX_WEB_REMOTE_SITES", []):
log.msg(u"Registering {} for {}".format(address, urlconf))
self.addWorker(address, urlconf)
def __unicode__(self):
return u", ".join([unicode(v) for v in self.configs])
def addWorker(self, address, urlconf):
conf = None
try:
conf = self.configs.find(urlconf)
except:
log.msg(u"Registering {}".format(urlconf))
conf = RemoteConfig(urlconf)
self.configs.insert(conf)
conf.addWorker(address)
def delWorker(self, address, urlconf):
try:
conf = self.configs.find(urlconf)
if not conf.delWorker(address):
log.msg(u"Unregistering {}".format(urlconf))
self.configs.remove(conf)
except:
pass
def nextWorker(self, urlconf):
with log.enter(obj=self) as tm:
try:
conf = self.configs.find(urlconf.__name__)
return conf.nextWorker()
except:
log.err(traceback.format_exc())
for cfg in self.configs:
log.err(unicode(cfg))
return None
class RemoteNodes:
def __init__(self):
self.configs = SortedCollection(key=lambda x: x.urlconf)
log.msg(u'Registering static web sites')
for address, urlconf in getattr(settings, 'DTX_WEB_REMOTE_SITES', []):
log.msg(u'Registering {} for {}'.format(address, urlconf))
self.addWorker(address, urlconf)
def __unicode__(self):
return u', '.join([unicode(v) for v in self.configs])
def addWorker(self, address, urlconf):
conf = None
try:
conf = self.configs.find(urlconf)
except:
log.msg(u'Registering {}'.format(urlconf))
conf = RemoteConfig(urlconf)
self.configs.insert(conf)
conf.addWorker(address)
def delWorker(self, address, urlconf):
try:
conf = self.configs.find(urlconf)
if not conf.delWorker(address):
log.msg(u'Unregistering {}'.format(urlconf))
self.configs.remove(conf)
except:
pass
def nextWorker(self, urlconf):
with log.enter(obj=self) as tm:
try:
conf = self.configs.find(urlconf.__name__)
return conf.nextWorker()
except:
log.err(traceback.format_exc())
for cfg in self.configs:
log.err(unicode(cfg))
return None
