class WebClientAgentsPool: def __init__(self, persistent=True): self.persistent = persistent self.agents = SortedCollection(key=lambda x: x.url.netloc) self.pool = HTTPConnectionPool(reactor) self.pool.maxPersistentPerHost = getattr( settings, 'DTX_WEB_DEFER_MAX_PERSISTENT_PER_HOST', 8) self.pool.cachedConnectionTimeout = getattr( settings, 'DTX_WEB_DEFER_CONNECT_TIMEOUT', 10) def createAgent(self, uri): url = urlparse(uri) try: agent = self.agents.find(url.netloc) log.msg(u'Using existing agent for {}'.format(url.netloc)) agent.url = url self.agents.remove(agent) return agent except: log.msg(u'Creating new agent for {}'.format(url.netloc)) agent = WebClientAgent(self, url) return agent def returnAgent(self, agent): if (self.persistent): log.msg(u'Returning agent to the pool'.format(self.url.netloc)) self.agents.insert(agent)
class WebClientAgentsPool: def __init__(self, persistent=True): self.persistent = persistent self.agents = SortedCollection(key=lambda x: x.url.netloc) self.pool = HTTPConnectionPool(reactor) self.pool.maxPersistentPerHost = getattr(settings, "DTX_WEB_DEFER_MAX_PERSISTENT_PER_HOST", 8) self.pool.cachedConnectionTimeout = getattr(settings, "DTX_WEB_DEFER_CONNECT_TIMEOUT", 10) def createAgent(self, uri): url = urlparse(uri) try: agent = self.agents.find(url.netloc) log.msg(u"Using existing agent for {}".format(url.netloc)) agent.url = url self.agents.remove(agent) return agent except: log.msg(u"Creating new agent for {}".format(url.netloc)) agent = WebClientAgent(self, url) return agent def returnAgent(self, agent): if self.persistent: log.msg(u"Returning agent to the pool".format(self.url.netloc)) self.agents.insert(agent)
class DtxTwistedWebResource(Resource): def __init__(self, pattern, urlconf): with log.enter(obj=self) as tm: Resource.__init__(self) self.pattern = pattern self.urlconf = urlconf self.callbacks = SortedCollection(key=lambda x: x.name) self._search_callbacks(urlconf) for item in self.callbacks: tm.msg(item) def _add_callback(self, name): try: cb = self.callbacks.find(name) cb.count += 1 except: path = name.split('.') mod = import_module('.'.join(path[:-1])) fun = mod.__dict__[path[-1]] if (not isinstance(fun, DtxTwistedWebCallbackDecorator)): log.msg('Decorating {} ({})'.format(name, fun)) mod.__dict__[path[-1]] = DtxTwistedWebCallbackDecorator(fun) self.callbacks.insert(DtxCallbackInfo(name)) def _search_callbacks(self, urlconf): urlpatterns = urlconf.__dict__.get('urlpatterns', []) for item in urlpatterns: if (issubclass(item.__class__, urlresolvers.RegexURLPattern)): callback_str = item.__dict__.get('_callback_str', None) if (callback_str): self._add_callback(callback_str) else: callback_str = '.'.join((item._callback.__module__, item._callback.__name__)) self._add_callback(callback_str) item._callback = None item._callback_str = callback_str elif (issubclass(item.__class__, urlresolvers.RegexURLResolver)): self._search_callbacks(item.urlconf_name) def render_request(self, request, method): with log.enter(obj=self) as tm: request.method = method request.url = urlparse.urlsplit(request.uri) tm.msg(u'Resolving \'{}\' at {}'.format(unicode(request.url.path), self.urlconf)) try: match = urlresolvers.resolve(unicode(request.url.path), urlconf=self.urlconf) tm.msg(u'Matched {}'.format(match)) request.urlconf = self.urlconf task.deferLater(reactor, 0, invokeResolverMatch, request, match) return NOT_DONE_YET except urlresolvers.Resolver404, ex: tm.err(u'Not found') request.setResponseCode(404) return '404' except BaseException, ex: tm.err(traceback.format_exc()) request.setResponseCode(500) return '500'
class RemoteConfig: def __init__(self, urlconf): self.urlconf = urlconf self.workers = SortedCollection(key=lambda x: x.address) self.index = 0 def __unicode__(self): return u'{}: {}'.format(unicode(self.urlconf), len(self.workers)) def addWorker(self, address): try: self.workers.find(address) except: log.msg(u'Registering {} for {}'.format(address, self.urlconf)) self.workers.insert(RemoteWorker(address)) def delWorker(self, address): try: worker = self.workers.find(address) log.msg(u'Unregistering {} for {}'.format(address, self.urlconf)) self.workers.remove(worker) except: pass return len(self.workers) def nextWorker(self): with log.enter(obj=self) as tm: if (not self.workers): log.err(u'No registered workers for {}'.format(self.urlconf)) return None start_index = self.index while (42): if (self.index > len(self.workers) - 1): self.index = 0 worker = self.workers[self.index] self.index += 1 if (worker.enabled()): tm.msg(u'Returning {}/{}: {}'.format( self.index, len(self.workers), worker)) return worker if (start_index == self.index): log.err(u'No free workers for {}'.format(self.urlconf)) return None
class RemoteConfig: def __init__(self, urlconf): self.urlconf = urlconf self.workers = SortedCollection(key=lambda x: x.address) self.index = 0 def __unicode__(self): return u"{}: {}".format(unicode(self.urlconf), len(self.workers)) def addWorker(self, address): try: self.workers.find(address) except: log.msg(u"Registering {} for {}".format(address, self.urlconf)) self.workers.insert(RemoteWorker(address)) def delWorker(self, address): try: worker = self.workers.find(address) log.msg(u"Unregistering {} for {}".format(address, self.urlconf)) self.workers.remove(worker) except: pass return len(self.workers) def nextWorker(self): with log.enter(obj=self) as tm: if not self.workers: log.err(u"No registered workers for {}".format(self.urlconf)) return None start_index = self.index while 42: if self.index > len(self.workers) - 1: self.index = 0 worker = self.workers[self.index] self.index += 1 if worker.enabled(): tm.msg(u"Returning {}/{}: {}".format(self.index, len(self.workers), worker)) return worker if start_index == self.index: log.err(u"No free workers for {}".format(self.urlconf)) return None
class RemoteNodes: def __init__(self): self.configs = SortedCollection(key=lambda x: x.urlconf) log.msg(u"Registering static web sites") for address, urlconf in getattr(settings, "DTX_WEB_REMOTE_SITES", []): log.msg(u"Registering {} for {}".format(address, urlconf)) self.addWorker(address, urlconf) def __unicode__(self): return u", ".join([unicode(v) for v in self.configs]) def addWorker(self, address, urlconf): conf = None try: conf = self.configs.find(urlconf) except: log.msg(u"Registering {}".format(urlconf)) conf = RemoteConfig(urlconf) self.configs.insert(conf) conf.addWorker(address) def delWorker(self, address, urlconf): try: conf = self.configs.find(urlconf) if not conf.delWorker(address): log.msg(u"Unregistering {}".format(urlconf)) self.configs.remove(conf) except: pass def nextWorker(self, urlconf): with log.enter(obj=self) as tm: try: conf = self.configs.find(urlconf.__name__) return conf.nextWorker() except: log.err(traceback.format_exc()) for cfg in self.configs: log.err(unicode(cfg)) return None
class RemoteNodes: def __init__(self): self.configs = SortedCollection(key=lambda x: x.urlconf) log.msg(u'Registering static web sites') for address, urlconf in getattr(settings, 'DTX_WEB_REMOTE_SITES', []): log.msg(u'Registering {} for {}'.format(address, urlconf)) self.addWorker(address, urlconf) def __unicode__(self): return u', '.join([unicode(v) for v in self.configs]) def addWorker(self, address, urlconf): conf = None try: conf = self.configs.find(urlconf) except: log.msg(u'Registering {}'.format(urlconf)) conf = RemoteConfig(urlconf) self.configs.insert(conf) conf.addWorker(address) def delWorker(self, address, urlconf): try: conf = self.configs.find(urlconf) if not conf.delWorker(address): log.msg(u'Unregistering {}'.format(urlconf)) self.configs.remove(conf) except: pass def nextWorker(self, urlconf): with log.enter(obj=self) as tm: try: conf = self.configs.find(urlconf.__name__) return conf.nextWorker() except: log.err(traceback.format_exc()) for cfg in self.configs: log.err(unicode(cfg)) return None