class WebClientAgentsPool:
def __init__(self, persistent=True):
self.persistent = persistent
self.agents = SortedCollection(key=lambda x: x.url.netloc)
self.pool = HTTPConnectionPool(reactor)
self.pool.maxPersistentPerHost = getattr(settings, "DTX_WEB_DEFER_MAX_PERSISTENT_PER_HOST", 8)
self.pool.cachedConnectionTimeout = getattr(settings, "DTX_WEB_DEFER_CONNECT_TIMEOUT", 10)
def createAgent(self, uri):
url = urlparse(uri)
try:
agent = self.agents.find(url.netloc)
log.msg(u"Using existing agent for {}".format(url.netloc))
agent.url = url
self.agents.remove(agent)
return agent
except:
log.msg(u"Creating new agent for {}".format(url.netloc))
agent = WebClientAgent(self, url)
return agent
def returnAgent(self, agent):
if self.persistent:
log.msg(u"Returning agent to the pool".format(self.url.netloc))
self.agents.insert(agent)
class WebClientAgentsPool:
def __init__(self, persistent=True):
self.persistent = persistent
self.agents = SortedCollection(key=lambda x: x.url.netloc)
self.pool = HTTPConnectionPool(reactor)
self.pool.maxPersistentPerHost = getattr(
settings, 'DTX_WEB_DEFER_MAX_PERSISTENT_PER_HOST', 8)
self.pool.cachedConnectionTimeout = getattr(
settings, 'DTX_WEB_DEFER_CONNECT_TIMEOUT', 10)
def createAgent(self, uri):
url = urlparse(uri)
try:
agent = self.agents.find(url.netloc)
log.msg(u'Using existing agent for {}'.format(url.netloc))
agent.url = url
self.agents.remove(agent)
return agent
except:
log.msg(u'Creating new agent for {}'.format(url.netloc))
agent = WebClientAgent(self, url)
return agent
def returnAgent(self, agent):
if (self.persistent):
log.msg(u'Returning agent to the pool'.format(self.url.netloc))
self.agents.insert(agent)
class Cache:
def __init__(self, get_item_defer_callable, seconds=120):
self.get_item_defer_callable = get_item_defer_callable
self.seconds = seconds
self.cache = {}
self.keys = SortedCollection([], key=lambda item: item['ts'])
self.update_requests = {}
@IC
def get(self, key):
item = None
if key in self.cache:
item = self.cache[key]['item']
if self.cache[key]['ts'] < timezone.now() - datetime.timedelta(
seconds=self.seconds):
reactor.callLater(0, self.update_cache, key)
else:
yield self.update_cache(key)
item = self.cache.get(key, None)
if item:
item = item['item']
Return(item)
@IC
def update_cache(self, key):
try:
if not key in self.update_requests:
self.update_requests[key] = maybeDeferred(
self.get_item_defer_callable, key)
item = None
try:
item = yield self.update_requests[key]
except Exception, ex:
print "ERROR IN DEFERRED UPDATING CACHE:", ex
import traceback
traceback.print_exc()
del self.update_requests[key]
ts = timezone.now()
item_d = {
'item': item,
'ts': ts,
'key': key,
}
if key in self.cache:
self.keys.remove(self.cache[key])
self.cache[key] = item_d
self.keys.insert(item_d)
while 42:
try:
old = self.keys.find_le(ts - datetime.timedelta(
seconds=self.seconds * 2))
del self.cache[old['key']]
self.keys.remove(old)
except ValueError, ex:
break
else:
class Cache:
def __init__(self,get_item_defer_callable,seconds=120):
self.get_item_defer_callable = get_item_defer_callable
self.seconds = seconds
self.cache = {}
self.keys = SortedCollection([],key=lambda item:item['ts'])
self.update_requests = {}
@IC
def get(self,key):
item = None
if key in self.cache:
item = self.cache[key]['item']
if self.cache[key]['ts'] < timezone.now() - datetime.timedelta(seconds=self.seconds):
reactor.callLater(0,self.update_cache,key)
else:
yield self.update_cache(key)
item = self.cache.get(key,None)
if item:
item = item['item']
Return(item)
@IC
def update_cache(self,key):
try:
if not key in self.update_requests:
self.update_requests[key] = maybeDeferred(self.get_item_defer_callable,key)
item = None
try:
item = yield self.update_requests[key]
except Exception,ex:
print "ERROR IN DEFERRED UPDATING CACHE:",ex
import traceback
traceback.print_exc()
del self.update_requests[key]
ts = timezone.now()
item_d = {
'item':item,
'ts':ts,
'key':key,
}
if key in self.cache:
self.keys.remove(self.cache[key])
self.cache[key] = item_d
self.keys.insert(item_d)
while 42:
try:
old = self.keys.find_le(ts-datetime.timedelta(seconds=self.seconds*2))
del self.cache[old['key']]
self.keys.remove(old)
except ValueError,ex:
break
else:
class RemoteConfig:
def __init__(self, urlconf):
self.urlconf = urlconf
self.workers = SortedCollection(key=lambda x: x.address)
self.index = 0
def __unicode__(self):
return u'{}: {}'.format(unicode(self.urlconf), len(self.workers))
def addWorker(self, address):
try:
self.workers.find(address)
except:
log.msg(u'Registering {} for {}'.format(address, self.urlconf))
self.workers.insert(RemoteWorker(address))
def delWorker(self, address):
try:
worker = self.workers.find(address)
log.msg(u'Unregistering {} for {}'.format(address, self.urlconf))
self.workers.remove(worker)
except:
pass
return len(self.workers)
def nextWorker(self):
with log.enter(obj=self) as tm:
if (not self.workers):
log.err(u'No registered workers for {}'.format(self.urlconf))
return None
start_index = self.index
while (42):
if (self.index > len(self.workers) - 1):
self.index = 0
worker = self.workers[self.index]
self.index += 1
if (worker.enabled()):
tm.msg(u'Returning {}/{}: {}'.format(
self.index, len(self.workers), worker))
return worker
if (start_index == self.index):
log.err(u'No free workers for {}'.format(self.urlconf))
return None
class RemoteConfig:
def __init__(self, urlconf):
self.urlconf = urlconf
self.workers = SortedCollection(key=lambda x: x.address)
self.index = 0
def __unicode__(self):
return u"{}: {}".format(unicode(self.urlconf), len(self.workers))
def addWorker(self, address):
try:
self.workers.find(address)
except:
log.msg(u"Registering {} for {}".format(address, self.urlconf))
self.workers.insert(RemoteWorker(address))
def delWorker(self, address):
try:
worker = self.workers.find(address)
log.msg(u"Unregistering {} for {}".format(address, self.urlconf))
self.workers.remove(worker)
except:
pass
return len(self.workers)
def nextWorker(self):
with log.enter(obj=self) as tm:
if not self.workers:
log.err(u"No registered workers for {}".format(self.urlconf))
return None
start_index = self.index
while 42:
if self.index > len(self.workers) - 1:
self.index = 0
worker = self.workers[self.index]
self.index += 1
if worker.enabled():
tm.msg(u"Returning {}/{}: {}".format(self.index, len(self.workers), worker))
return worker
if start_index == self.index:
log.err(u"No free workers for {}".format(self.urlconf))
return None
class RemoteNodes:
def __init__(self):
self.configs = SortedCollection(key=lambda x: x.urlconf)
log.msg(u"Registering static web sites")
for address, urlconf in getattr(settings, "DTX_WEB_REMOTE_SITES", []):
log.msg(u"Registering {} for {}".format(address, urlconf))
self.addWorker(address, urlconf)
def __unicode__(self):
return u", ".join([unicode(v) for v in self.configs])
def addWorker(self, address, urlconf):
conf = None
try:
conf = self.configs.find(urlconf)
except:
log.msg(u"Registering {}".format(urlconf))
conf = RemoteConfig(urlconf)
self.configs.insert(conf)
conf.addWorker(address)
def delWorker(self, address, urlconf):
try:
conf = self.configs.find(urlconf)
if not conf.delWorker(address):
log.msg(u"Unregistering {}".format(urlconf))
self.configs.remove(conf)
except:
pass
def nextWorker(self, urlconf):
with log.enter(obj=self) as tm:
try:
conf = self.configs.find(urlconf.__name__)
return conf.nextWorker()
except:
log.err(traceback.format_exc())
for cfg in self.configs:
log.err(unicode(cfg))
return None
class RemoteNodes:
def __init__(self):
self.configs = SortedCollection(key=lambda x: x.urlconf)
log.msg(u'Registering static web sites')
for address, urlconf in getattr(settings, 'DTX_WEB_REMOTE_SITES', []):
log.msg(u'Registering {} for {}'.format(address, urlconf))
self.addWorker(address, urlconf)
def __unicode__(self):
return u', '.join([unicode(v) for v in self.configs])
def addWorker(self, address, urlconf):
conf = None
try:
conf = self.configs.find(urlconf)
except:
log.msg(u'Registering {}'.format(urlconf))
conf = RemoteConfig(urlconf)
self.configs.insert(conf)
conf.addWorker(address)
def delWorker(self, address, urlconf):
try:
conf = self.configs.find(urlconf)
if not conf.delWorker(address):
log.msg(u'Unregistering {}'.format(urlconf))
self.configs.remove(conf)
except:
pass
def nextWorker(self, urlconf):
with log.enter(obj=self) as tm:
try:
conf = self.configs.find(urlconf.__name__)
return conf.nextWorker()
except:
log.err(traceback.format_exc())
for cfg in self.configs:
log.err(unicode(cfg))
return None
