class WebClientAgentsPool:
def __init__(self, persistent=True):
self.persistent = persistent
self.agents = SortedCollection(key=lambda x: x.url.netloc)
self.pool = HTTPConnectionPool(reactor)
self.pool.maxPersistentPerHost = getattr(settings, "DTX_WEB_DEFER_MAX_PERSISTENT_PER_HOST", 8)
self.pool.cachedConnectionTimeout = getattr(settings, "DTX_WEB_DEFER_CONNECT_TIMEOUT", 10)
def createAgent(self, uri):
url = urlparse(uri)
try:
agent = self.agents.find(url.netloc)
log.msg(u"Using existing agent for {}".format(url.netloc))
agent.url = url
self.agents.remove(agent)
return agent
except:
log.msg(u"Creating new agent for {}".format(url.netloc))
agent = WebClientAgent(self, url)
return agent
def returnAgent(self, agent):
if self.persistent:
log.msg(u"Returning agent to the pool".format(self.url.netloc))
self.agents.insert(agent)
class WebClientAgentsPool:
def __init__(self, persistent=True):
self.persistent = persistent
self.agents = SortedCollection(key=lambda x: x.url.netloc)
self.pool = HTTPConnectionPool(reactor)
self.pool.maxPersistentPerHost = getattr(
settings, 'DTX_WEB_DEFER_MAX_PERSISTENT_PER_HOST', 8)
self.pool.cachedConnectionTimeout = getattr(
settings, 'DTX_WEB_DEFER_CONNECT_TIMEOUT', 10)
def createAgent(self, uri):
url = urlparse(uri)
try:
agent = self.agents.find(url.netloc)
log.msg(u'Using existing agent for {}'.format(url.netloc))
agent.url = url
self.agents.remove(agent)
return agent
except:
log.msg(u'Creating new agent for {}'.format(url.netloc))
agent = WebClientAgent(self, url)
return agent
def returnAgent(self, agent):
if (self.persistent):
log.msg(u'Returning agent to the pool'.format(self.url.netloc))
self.agents.insert(agent)
class DtxTwistedWebResource(Resource):
def __init__(self, pattern, urlconf):
with log.enter(obj=self) as tm:
Resource.__init__(self)
self.pattern = pattern
self.urlconf = urlconf
self.callbacks = SortedCollection(key=lambda x: x.name)
self._search_callbacks(urlconf)
for item in self.callbacks:
tm.msg(item)
def _add_callback(self, name):
try:
cb = self.callbacks.find(name)
cb.count += 1
except:
path = name.split('.')
mod = import_module('.'.join(path[:-1]))
fun = mod.__dict__[path[-1]]
if (not isinstance(fun, DtxTwistedWebCallbackDecorator)):
log.msg('Decorating {} ({})'.format(name, fun))
mod.__dict__[path[-1]] = DtxTwistedWebCallbackDecorator(fun)
self.callbacks.insert(DtxCallbackInfo(name))
def _search_callbacks(self, urlconf):
urlpatterns = urlconf.__dict__.get('urlpatterns', [])
for item in urlpatterns:
if (issubclass(item.__class__, urlresolvers.RegexURLPattern)):
callback_str = item.__dict__.get('_callback_str', None)
if (callback_str):
self._add_callback(callback_str)
else:
callback_str = '.'.join((item._callback.__module__, item._callback.__name__))
self._add_callback(callback_str)
item._callback = None
item._callback_str = callback_str
elif (issubclass(item.__class__, urlresolvers.RegexURLResolver)):
self._search_callbacks(item.urlconf_name)
def render_request(self, request, method):
with log.enter(obj=self) as tm:
request.method = method
request.url = urlparse.urlsplit(request.uri)
tm.msg(u'Resolving \'{}\' at {}'.format(unicode(request.url.path), self.urlconf))
try:
match = urlresolvers.resolve(unicode(request.url.path), urlconf=self.urlconf)
tm.msg(u'Matched {}'.format(match))
request.urlconf = self.urlconf
task.deferLater(reactor, 0, invokeResolverMatch, request, match)
return NOT_DONE_YET
except urlresolvers.Resolver404, ex:
tm.err(u'Not found')
request.setResponseCode(404)
return '404'
except BaseException, ex:
tm.err(traceback.format_exc())
request.setResponseCode(500)
return '500'
def __init__(self, persistent=True):
self.persistent = persistent
self.agents = SortedCollection(key=lambda x: x.url.netloc)
self.pool = HTTPConnectionPool(reactor)
self.pool.maxPersistentPerHost = getattr(
settings, 'DTX_WEB_DEFER_MAX_PERSISTENT_PER_HOST', 8)
self.pool.cachedConnectionTimeout = getattr(
settings, 'DTX_WEB_DEFER_CONNECT_TIMEOUT', 10)
class ClientProtocol(WampClientProtocol):
def __init__(self):
#WampClientProtocol.__init__(self)
self.evalSessions = SortedCollection(key=lambda x: x.id)
@classmethod
def connect(cls, uri, debug=False):
global client_factory
client_factory = ClientFactory(uri)
client_factory.uri = uri
client_factory.protocol = cls
connectWS(client_factory)
return client_factory
def done(self, *args):
self.sendClose()
reactor.stop()
def onConnected(self):
pass
def onSessionOpen(self):
log.msg(u"Connected")
self.onConnected()
@inlineCallbacks
def eval(self, algo, data):
df = Deferred()
df.callback(None)
@inlineCallbacks
def execjs(self, command, broadcast=False):
with log.enter(obj=self) as tm:
uri = yaa_api + 'session#execjs'
log.msg('Calling {} {}'.format(uri, {'exec': command}))
yield self.call(uri, {'exec': command}, broadcast)
@inlineCallbacks
def evaljs(self, js, params, role=0, timeout=None):
with log.enter(obj=self) as tm:
try:
uri = yaa_api + 'session#evaljs'
log.msg(u'Calling {} {}'.format(
uri, {
'js': js,
'params': type(params),
'timeout': timeout if (timeout) else 0
}))
id = yield self.call(uri, js, params, role, timeout if
(timeout) else 0)
df = Deferred()
self.evalSessions.insert(EvalControlBlock(id, df))
yield df
except Exception, exc:
log.msg(traceback.format_exc(), level=logging.ERROR)
raise
class ClientProtocol(WampClientProtocol):
def __init__(self):
#WampClientProtocol.__init__(self)
self.evalSessions = SortedCollection(key=lambda x:x.id)
@classmethod
def connect(cls, uri, debug=False):
global client_factory
client_factory = ClientFactory(uri)
client_factory.uri = uri
client_factory.protocol = cls
connectWS(client_factory)
return client_factory
def done(self, *args):
self.sendClose()
reactor.stop()
def onConnected(self):
pass
def onSessionOpen(self):
log.msg(u"Connected")
self.onConnected()
@inlineCallbacks
def eval(self, algo, data):
df = Deferred()
df.callback(None)
@inlineCallbacks
def execjs(self, command, broadcast=False):
with log.enter(obj=self) as tm:
uri = yaa_api + 'session#execjs'
log.msg('Calling {} {}'.format(uri, {'exec': command}))
yield self.call(uri, {'exec': command}, broadcast)
@inlineCallbacks
def evaljs(self, js, params, role=0, timeout=None):
with log.enter(obj=self) as tm:
try:
uri = yaa_api + 'session#evaljs'
log.msg(u'Calling {} {}'.format(uri, {'js': js, 'params': type(params), 'timeout': timeout if (timeout) else 0}))
id = yield self.call(uri, js, params, role, timeout if (timeout) else 0)
df = Deferred()
self.evalSessions.insert(EvalControlBlock(id, df))
yield df
except Exception, exc:
log.msg(traceback.format_exc(), level=logging.ERROR)
raise
def __init__(self, pattern, urlconf):
with log.enter(obj=self) as tm:
Resource.__init__(self)
self.pattern = pattern
self.urlconf = urlconf
self.callbacks = SortedCollection(key=lambda x: x.name)
self._search_callbacks(urlconf)
for item in self.callbacks:
tm.msg(item)
class Cache:
def __init__(self, get_item_defer_callable, seconds=120):
self.get_item_defer_callable = get_item_defer_callable
self.seconds = seconds
self.cache = {}
self.keys = SortedCollection([], key=lambda item: item['ts'])
self.update_requests = {}
@IC
def get(self, key):
item = None
if key in self.cache:
item = self.cache[key]['item']
if self.cache[key]['ts'] < timezone.now() - datetime.timedelta(
seconds=self.seconds):
reactor.callLater(0, self.update_cache, key)
else:
yield self.update_cache(key)
item = self.cache.get(key, None)
if item:
item = item['item']
Return(item)
@IC
def update_cache(self, key):
try:
if not key in self.update_requests:
self.update_requests[key] = maybeDeferred(
self.get_item_defer_callable, key)
item = None
try:
item = yield self.update_requests[key]
except Exception, ex:
print "ERROR IN DEFERRED UPDATING CACHE:", ex
import traceback
traceback.print_exc()
del self.update_requests[key]
ts = timezone.now()
item_d = {
'item': item,
'ts': ts,
'key': key,
}
if key in self.cache:
self.keys.remove(self.cache[key])
self.cache[key] = item_d
self.keys.insert(item_d)
while 42:
try:
old = self.keys.find_le(ts - datetime.timedelta(
seconds=self.seconds * 2))
del self.cache[old['key']]
self.keys.remove(old)
except ValueError, ex:
break
else:
class Cache:
def __init__(self,get_item_defer_callable,seconds=120):
self.get_item_defer_callable = get_item_defer_callable
self.seconds = seconds
self.cache = {}
self.keys = SortedCollection([],key=lambda item:item['ts'])
self.update_requests = {}
@IC
def get(self,key):
item = None
if key in self.cache:
item = self.cache[key]['item']
if self.cache[key]['ts'] < timezone.now() - datetime.timedelta(seconds=self.seconds):
reactor.callLater(0,self.update_cache,key)
else:
yield self.update_cache(key)
item = self.cache.get(key,None)
if item:
item = item['item']
Return(item)
@IC
def update_cache(self,key):
try:
if not key in self.update_requests:
self.update_requests[key] = maybeDeferred(self.get_item_defer_callable,key)
item = None
try:
item = yield self.update_requests[key]
except Exception,ex:
print "ERROR IN DEFERRED UPDATING CACHE:",ex
import traceback
traceback.print_exc()
del self.update_requests[key]
ts = timezone.now()
item_d = {
'item':item,
'ts':ts,
'key':key,
}
if key in self.cache:
self.keys.remove(self.cache[key])
self.cache[key] = item_d
self.keys.insert(item_d)
while 42:
try:
old = self.keys.find_le(ts-datetime.timedelta(seconds=self.seconds*2))
del self.cache[old['key']]
self.keys.remove(old)
except ValueError,ex:
break
else:
class RemoteConfig:
def __init__(self, urlconf):
self.urlconf = urlconf
self.workers = SortedCollection(key=lambda x: x.address)
self.index = 0
def __unicode__(self):
return u'{}: {}'.format(unicode(self.urlconf), len(self.workers))
def addWorker(self, address):
try:
self.workers.find(address)
except:
log.msg(u'Registering {} for {}'.format(address, self.urlconf))
self.workers.insert(RemoteWorker(address))
def delWorker(self, address):
try:
worker = self.workers.find(address)
log.msg(u'Unregistering {} for {}'.format(address, self.urlconf))
self.workers.remove(worker)
except:
pass
return len(self.workers)
def nextWorker(self):
with log.enter(obj=self) as tm:
if (not self.workers):
log.err(u'No registered workers for {}'.format(self.urlconf))
return None
start_index = self.index
while (42):
if (self.index > len(self.workers) - 1):
self.index = 0
worker = self.workers[self.index]
self.index += 1
if (worker.enabled()):
tm.msg(u'Returning {}/{}: {}'.format(
self.index, len(self.workers), worker))
return worker
if (start_index == self.index):
log.err(u'No free workers for {}'.format(self.urlconf))
return None
class RemoteConfig:
def __init__(self, urlconf):
self.urlconf = urlconf
self.workers = SortedCollection(key=lambda x: x.address)
self.index = 0
def __unicode__(self):
return u"{}: {}".format(unicode(self.urlconf), len(self.workers))
def addWorker(self, address):
try:
self.workers.find(address)
except:
log.msg(u"Registering {} for {}".format(address, self.urlconf))
self.workers.insert(RemoteWorker(address))
def delWorker(self, address):
try:
worker = self.workers.find(address)
log.msg(u"Unregistering {} for {}".format(address, self.urlconf))
self.workers.remove(worker)
except:
pass
return len(self.workers)
def nextWorker(self):
with log.enter(obj=self) as tm:
if not self.workers:
log.err(u"No registered workers for {}".format(self.urlconf))
return None
start_index = self.index
while 42:
if self.index > len(self.workers) - 1:
self.index = 0
worker = self.workers[self.index]
self.index += 1
if worker.enabled():
tm.msg(u"Returning {}/{}: {}".format(self.index, len(self.workers), worker))
return worker
if start_index == self.index:
log.err(u"No free workers for {}".format(self.urlconf))
return None
class RemoteNodes:
def __init__(self):
self.configs = SortedCollection(key=lambda x: x.urlconf)
log.msg(u"Registering static web sites")
for address, urlconf in getattr(settings, "DTX_WEB_REMOTE_SITES", []):
log.msg(u"Registering {} for {}".format(address, urlconf))
self.addWorker(address, urlconf)
def __unicode__(self):
return u", ".join([unicode(v) for v in self.configs])
def addWorker(self, address, urlconf):
conf = None
try:
conf = self.configs.find(urlconf)
except:
log.msg(u"Registering {}".format(urlconf))
conf = RemoteConfig(urlconf)
self.configs.insert(conf)
conf.addWorker(address)
def delWorker(self, address, urlconf):
try:
conf = self.configs.find(urlconf)
if not conf.delWorker(address):
log.msg(u"Unregistering {}".format(urlconf))
self.configs.remove(conf)
except:
pass
def nextWorker(self, urlconf):
with log.enter(obj=self) as tm:
try:
conf = self.configs.find(urlconf.__name__)
return conf.nextWorker()
except:
log.err(traceback.format_exc())
for cfg in self.configs:
log.err(unicode(cfg))
return None
class RemoteNodes:
def __init__(self):
self.configs = SortedCollection(key=lambda x: x.urlconf)
log.msg(u'Registering static web sites')
for address, urlconf in getattr(settings, 'DTX_WEB_REMOTE_SITES', []):
log.msg(u'Registering {} for {}'.format(address, urlconf))
self.addWorker(address, urlconf)
def __unicode__(self):
return u', '.join([unicode(v) for v in self.configs])
def addWorker(self, address, urlconf):
conf = None
try:
conf = self.configs.find(urlconf)
except:
log.msg(u'Registering {}'.format(urlconf))
conf = RemoteConfig(urlconf)
self.configs.insert(conf)
conf.addWorker(address)
def delWorker(self, address, urlconf):
try:
conf = self.configs.find(urlconf)
if not conf.delWorker(address):
log.msg(u'Unregistering {}'.format(urlconf))
self.configs.remove(conf)
except:
pass
def nextWorker(self, urlconf):
with log.enter(obj=self) as tm:
try:
conf = self.configs.find(urlconf.__name__)
return conf.nextWorker()
except:
log.err(traceback.format_exc())
for cfg in self.configs:
log.err(unicode(cfg))
return None
def __init__(self, persistent=True):
self.persistent = persistent
self.agents = SortedCollection(key=lambda x: x.url.netloc)
self.pool = HTTPConnectionPool(reactor)
self.pool.maxPersistentPerHost = getattr(settings, "DTX_WEB_DEFER_MAX_PERSISTENT_PER_HOST", 8)
self.pool.cachedConnectionTimeout = getattr(settings, "DTX_WEB_DEFER_CONNECT_TIMEOUT", 10)
def __init__(self):
#WampClientProtocol.__init__(self)
self.evalSessions = SortedCollection(key=lambda x:x.id)
def __init__(self):
self.configs = SortedCollection(key=lambda x: x.urlconf)
log.msg(u"Registering static web sites")
for address, urlconf in getattr(settings, "DTX_WEB_REMOTE_SITES", []):
log.msg(u"Registering {} for {}".format(address, urlconf))
self.addWorker(address, urlconf)
def __init__(self, urlconf):
self.urlconf = urlconf
self.workers = SortedCollection(key=lambda x: x.address)
self.index = 0
def __init__(self, get_item_defer_callable, seconds=120):
self.get_item_defer_callable = get_item_defer_callable
self.seconds = seconds
self.cache = {}
self.keys = SortedCollection([], key=lambda item: item['ts'])
self.update_requests = {}
def __init__(self):
#WampClientProtocol.__init__(self)
self.evalSessions = SortedCollection(key=lambda x: x.id)
def __init__(self, urlconf):
self.urlconf = urlconf
self.workers = SortedCollection(key=lambda x: x.address)
self.index = 0
def __init__(self,get_item_defer_callable,seconds=120):
self.get_item_defer_callable = get_item_defer_callable
self.seconds = seconds
self.cache = {}
self.keys = SortedCollection([],key=lambda item:item['ts'])
self.update_requests = {}
from django.utils import translation
from django.utils.translation import ugettext_lazy as _
from django.utils.translation import ugettext
from django.utils.translation import ugettext_noop
from django.utils.translation import get_language
from django.contrib.staticfiles import finders
from dtx.utils.snippets.sorted_collection import SortedCollection
from dtx.memcache import client as dtx_memcache
from dtx.core import logger
log = logger.log(__name__)
static_files_by_path = SortedCollection(key=lambda x: x.path)
class StaticFileInfo:
def __init__(self, path, body_digest, mime_type, mime_charset):
self.path = path
self.path_digest = hashlib.md5(path).hexdigest()
self.body_digest = body_digest
self.mime_type = mime_type
self.mime_charset = mime_charset
self.mtime = os.stat(path).st_mtime
@classmethod
def create(cls, path, body_digest, mime_type, mime_charset):
global static_files_by_path
info = cls(path, body_digest, mime_type, mime_charset)
from twisted.internet.defer import inlineCallbacks, returnValue, Deferred
from twisted.internet import task
from twisted.internet import reactor
from django.conf import settings
from dtx.utils.snippets.sorted_collection import SortedCollection
from dtx.core import logger
log = logger.log(__name__)
##############################################################################################################################
#
##############################################################################################################################
wamp_connections = SortedCollection(key=lambda x: x.uuid)
wamp_connection_uuid = 1
##############################################################################################################################
#
##############################################################################################################################
class ServerFactory(WampServerFactory):
def __init__(self, uri, apps, debug=False):
WampServerFactory.__init__(self, uri, debug)
self.apps = []
for app in apps:
if (isinstance(app, (str))):
mod = import_module(app)
else:
def __init__(self):
self.configs = SortedCollection(key=lambda x: x.urlconf)
log.msg(u'Registering static web sites')
for address, urlconf in getattr(settings, 'DTX_WEB_REMOTE_SITES', []):
log.msg(u'Registering {} for {}'.format(address, urlconf))
self.addWorker(address, urlconf)