class WebClientAgentsPool: def __init__(self, persistent=True): self.persistent = persistent self.agents = SortedCollection(key=lambda x: x.url.netloc) self.pool = HTTPConnectionPool(reactor) self.pool.maxPersistentPerHost = getattr(settings, "DTX_WEB_DEFER_MAX_PERSISTENT_PER_HOST", 8) self.pool.cachedConnectionTimeout = getattr(settings, "DTX_WEB_DEFER_CONNECT_TIMEOUT", 10) def createAgent(self, uri): url = urlparse(uri) try: agent = self.agents.find(url.netloc) log.msg(u"Using existing agent for {}".format(url.netloc)) agent.url = url self.agents.remove(agent) return agent except: log.msg(u"Creating new agent for {}".format(url.netloc)) agent = WebClientAgent(self, url) return agent def returnAgent(self, agent): if self.persistent: log.msg(u"Returning agent to the pool".format(self.url.netloc)) self.agents.insert(agent)
class WebClientAgentsPool: def __init__(self, persistent=True): self.persistent = persistent self.agents = SortedCollection(key=lambda x: x.url.netloc) self.pool = HTTPConnectionPool(reactor) self.pool.maxPersistentPerHost = getattr( settings, 'DTX_WEB_DEFER_MAX_PERSISTENT_PER_HOST', 8) self.pool.cachedConnectionTimeout = getattr( settings, 'DTX_WEB_DEFER_CONNECT_TIMEOUT', 10) def createAgent(self, uri): url = urlparse(uri) try: agent = self.agents.find(url.netloc) log.msg(u'Using existing agent for {}'.format(url.netloc)) agent.url = url self.agents.remove(agent) return agent except: log.msg(u'Creating new agent for {}'.format(url.netloc)) agent = WebClientAgent(self, url) return agent def returnAgent(self, agent): if (self.persistent): log.msg(u'Returning agent to the pool'.format(self.url.netloc)) self.agents.insert(agent)
class DtxTwistedWebResource(Resource): def __init__(self, pattern, urlconf): with log.enter(obj=self) as tm: Resource.__init__(self) self.pattern = pattern self.urlconf = urlconf self.callbacks = SortedCollection(key=lambda x: x.name) self._search_callbacks(urlconf) for item in self.callbacks: tm.msg(item) def _add_callback(self, name): try: cb = self.callbacks.find(name) cb.count += 1 except: path = name.split('.') mod = import_module('.'.join(path[:-1])) fun = mod.__dict__[path[-1]] if (not isinstance(fun, DtxTwistedWebCallbackDecorator)): log.msg('Decorating {} ({})'.format(name, fun)) mod.__dict__[path[-1]] = DtxTwistedWebCallbackDecorator(fun) self.callbacks.insert(DtxCallbackInfo(name)) def _search_callbacks(self, urlconf): urlpatterns = urlconf.__dict__.get('urlpatterns', []) for item in urlpatterns: if (issubclass(item.__class__, urlresolvers.RegexURLPattern)): callback_str = item.__dict__.get('_callback_str', None) if (callback_str): self._add_callback(callback_str) else: callback_str = '.'.join((item._callback.__module__, item._callback.__name__)) self._add_callback(callback_str) item._callback = None item._callback_str = callback_str elif (issubclass(item.__class__, urlresolvers.RegexURLResolver)): self._search_callbacks(item.urlconf_name) def render_request(self, request, method): with log.enter(obj=self) as tm: request.method = method request.url = urlparse.urlsplit(request.uri) tm.msg(u'Resolving \'{}\' at {}'.format(unicode(request.url.path), self.urlconf)) try: match = urlresolvers.resolve(unicode(request.url.path), urlconf=self.urlconf) tm.msg(u'Matched {}'.format(match)) request.urlconf = self.urlconf task.deferLater(reactor, 0, invokeResolverMatch, request, match) return NOT_DONE_YET except urlresolvers.Resolver404, ex: tm.err(u'Not found') request.setResponseCode(404) return '404' except BaseException, ex: tm.err(traceback.format_exc()) request.setResponseCode(500) return '500'
def __init__(self, persistent=True): self.persistent = persistent self.agents = SortedCollection(key=lambda x: x.url.netloc) self.pool = HTTPConnectionPool(reactor) self.pool.maxPersistentPerHost = getattr( settings, 'DTX_WEB_DEFER_MAX_PERSISTENT_PER_HOST', 8) self.pool.cachedConnectionTimeout = getattr( settings, 'DTX_WEB_DEFER_CONNECT_TIMEOUT', 10)
class ClientProtocol(WampClientProtocol): def __init__(self): #WampClientProtocol.__init__(self) self.evalSessions = SortedCollection(key=lambda x: x.id) @classmethod def connect(cls, uri, debug=False): global client_factory client_factory = ClientFactory(uri) client_factory.uri = uri client_factory.protocol = cls connectWS(client_factory) return client_factory def done(self, *args): self.sendClose() reactor.stop() def onConnected(self): pass def onSessionOpen(self): log.msg(u"Connected") self.onConnected() @inlineCallbacks def eval(self, algo, data): df = Deferred() df.callback(None) @inlineCallbacks def execjs(self, command, broadcast=False): with log.enter(obj=self) as tm: uri = yaa_api + 'session#execjs' log.msg('Calling {} {}'.format(uri, {'exec': command})) yield self.call(uri, {'exec': command}, broadcast) @inlineCallbacks def evaljs(self, js, params, role=0, timeout=None): with log.enter(obj=self) as tm: try: uri = yaa_api + 'session#evaljs' log.msg(u'Calling {} {}'.format( uri, { 'js': js, 'params': type(params), 'timeout': timeout if (timeout) else 0 })) id = yield self.call(uri, js, params, role, timeout if (timeout) else 0) df = Deferred() self.evalSessions.insert(EvalControlBlock(id, df)) yield df except Exception, exc: log.msg(traceback.format_exc(), level=logging.ERROR) raise
class ClientProtocol(WampClientProtocol): def __init__(self): #WampClientProtocol.__init__(self) self.evalSessions = SortedCollection(key=lambda x:x.id) @classmethod def connect(cls, uri, debug=False): global client_factory client_factory = ClientFactory(uri) client_factory.uri = uri client_factory.protocol = cls connectWS(client_factory) return client_factory def done(self, *args): self.sendClose() reactor.stop() def onConnected(self): pass def onSessionOpen(self): log.msg(u"Connected") self.onConnected() @inlineCallbacks def eval(self, algo, data): df = Deferred() df.callback(None) @inlineCallbacks def execjs(self, command, broadcast=False): with log.enter(obj=self) as tm: uri = yaa_api + 'session#execjs' log.msg('Calling {} {}'.format(uri, {'exec': command})) yield self.call(uri, {'exec': command}, broadcast) @inlineCallbacks def evaljs(self, js, params, role=0, timeout=None): with log.enter(obj=self) as tm: try: uri = yaa_api + 'session#evaljs' log.msg(u'Calling {} {}'.format(uri, {'js': js, 'params': type(params), 'timeout': timeout if (timeout) else 0})) id = yield self.call(uri, js, params, role, timeout if (timeout) else 0) df = Deferred() self.evalSessions.insert(EvalControlBlock(id, df)) yield df except Exception, exc: log.msg(traceback.format_exc(), level=logging.ERROR) raise
def __init__(self, pattern, urlconf): with log.enter(obj=self) as tm: Resource.__init__(self) self.pattern = pattern self.urlconf = urlconf self.callbacks = SortedCollection(key=lambda x: x.name) self._search_callbacks(urlconf) for item in self.callbacks: tm.msg(item)
class Cache: def __init__(self, get_item_defer_callable, seconds=120): self.get_item_defer_callable = get_item_defer_callable self.seconds = seconds self.cache = {} self.keys = SortedCollection([], key=lambda item: item['ts']) self.update_requests = {} @IC def get(self, key): item = None if key in self.cache: item = self.cache[key]['item'] if self.cache[key]['ts'] < timezone.now() - datetime.timedelta( seconds=self.seconds): reactor.callLater(0, self.update_cache, key) else: yield self.update_cache(key) item = self.cache.get(key, None) if item: item = item['item'] Return(item) @IC def update_cache(self, key): try: if not key in self.update_requests: self.update_requests[key] = maybeDeferred( self.get_item_defer_callable, key) item = None try: item = yield self.update_requests[key] except Exception, ex: print "ERROR IN DEFERRED UPDATING CACHE:", ex import traceback traceback.print_exc() del self.update_requests[key] ts = timezone.now() item_d = { 'item': item, 'ts': ts, 'key': key, } if key in self.cache: self.keys.remove(self.cache[key]) self.cache[key] = item_d self.keys.insert(item_d) while 42: try: old = self.keys.find_le(ts - datetime.timedelta( seconds=self.seconds * 2)) del self.cache[old['key']] self.keys.remove(old) except ValueError, ex: break else:
class Cache: def __init__(self,get_item_defer_callable,seconds=120): self.get_item_defer_callable = get_item_defer_callable self.seconds = seconds self.cache = {} self.keys = SortedCollection([],key=lambda item:item['ts']) self.update_requests = {} @IC def get(self,key): item = None if key in self.cache: item = self.cache[key]['item'] if self.cache[key]['ts'] < timezone.now() - datetime.timedelta(seconds=self.seconds): reactor.callLater(0,self.update_cache,key) else: yield self.update_cache(key) item = self.cache.get(key,None) if item: item = item['item'] Return(item) @IC def update_cache(self,key): try: if not key in self.update_requests: self.update_requests[key] = maybeDeferred(self.get_item_defer_callable,key) item = None try: item = yield self.update_requests[key] except Exception,ex: print "ERROR IN DEFERRED UPDATING CACHE:",ex import traceback traceback.print_exc() del self.update_requests[key] ts = timezone.now() item_d = { 'item':item, 'ts':ts, 'key':key, } if key in self.cache: self.keys.remove(self.cache[key]) self.cache[key] = item_d self.keys.insert(item_d) while 42: try: old = self.keys.find_le(ts-datetime.timedelta(seconds=self.seconds*2)) del self.cache[old['key']] self.keys.remove(old) except ValueError,ex: break else:
class RemoteConfig: def __init__(self, urlconf): self.urlconf = urlconf self.workers = SortedCollection(key=lambda x: x.address) self.index = 0 def __unicode__(self): return u'{}: {}'.format(unicode(self.urlconf), len(self.workers)) def addWorker(self, address): try: self.workers.find(address) except: log.msg(u'Registering {} for {}'.format(address, self.urlconf)) self.workers.insert(RemoteWorker(address)) def delWorker(self, address): try: worker = self.workers.find(address) log.msg(u'Unregistering {} for {}'.format(address, self.urlconf)) self.workers.remove(worker) except: pass return len(self.workers) def nextWorker(self): with log.enter(obj=self) as tm: if (not self.workers): log.err(u'No registered workers for {}'.format(self.urlconf)) return None start_index = self.index while (42): if (self.index > len(self.workers) - 1): self.index = 0 worker = self.workers[self.index] self.index += 1 if (worker.enabled()): tm.msg(u'Returning {}/{}: {}'.format( self.index, len(self.workers), worker)) return worker if (start_index == self.index): log.err(u'No free workers for {}'.format(self.urlconf)) return None
class RemoteConfig: def __init__(self, urlconf): self.urlconf = urlconf self.workers = SortedCollection(key=lambda x: x.address) self.index = 0 def __unicode__(self): return u"{}: {}".format(unicode(self.urlconf), len(self.workers)) def addWorker(self, address): try: self.workers.find(address) except: log.msg(u"Registering {} for {}".format(address, self.urlconf)) self.workers.insert(RemoteWorker(address)) def delWorker(self, address): try: worker = self.workers.find(address) log.msg(u"Unregistering {} for {}".format(address, self.urlconf)) self.workers.remove(worker) except: pass return len(self.workers) def nextWorker(self): with log.enter(obj=self) as tm: if not self.workers: log.err(u"No registered workers for {}".format(self.urlconf)) return None start_index = self.index while 42: if self.index > len(self.workers) - 1: self.index = 0 worker = self.workers[self.index] self.index += 1 if worker.enabled(): tm.msg(u"Returning {}/{}: {}".format(self.index, len(self.workers), worker)) return worker if start_index == self.index: log.err(u"No free workers for {}".format(self.urlconf)) return None
class RemoteNodes: def __init__(self): self.configs = SortedCollection(key=lambda x: x.urlconf) log.msg(u"Registering static web sites") for address, urlconf in getattr(settings, "DTX_WEB_REMOTE_SITES", []): log.msg(u"Registering {} for {}".format(address, urlconf)) self.addWorker(address, urlconf) def __unicode__(self): return u", ".join([unicode(v) for v in self.configs]) def addWorker(self, address, urlconf): conf = None try: conf = self.configs.find(urlconf) except: log.msg(u"Registering {}".format(urlconf)) conf = RemoteConfig(urlconf) self.configs.insert(conf) conf.addWorker(address) def delWorker(self, address, urlconf): try: conf = self.configs.find(urlconf) if not conf.delWorker(address): log.msg(u"Unregistering {}".format(urlconf)) self.configs.remove(conf) except: pass def nextWorker(self, urlconf): with log.enter(obj=self) as tm: try: conf = self.configs.find(urlconf.__name__) return conf.nextWorker() except: log.err(traceback.format_exc()) for cfg in self.configs: log.err(unicode(cfg)) return None
class RemoteNodes: def __init__(self): self.configs = SortedCollection(key=lambda x: x.urlconf) log.msg(u'Registering static web sites') for address, urlconf in getattr(settings, 'DTX_WEB_REMOTE_SITES', []): log.msg(u'Registering {} for {}'.format(address, urlconf)) self.addWorker(address, urlconf) def __unicode__(self): return u', '.join([unicode(v) for v in self.configs]) def addWorker(self, address, urlconf): conf = None try: conf = self.configs.find(urlconf) except: log.msg(u'Registering {}'.format(urlconf)) conf = RemoteConfig(urlconf) self.configs.insert(conf) conf.addWorker(address) def delWorker(self, address, urlconf): try: conf = self.configs.find(urlconf) if not conf.delWorker(address): log.msg(u'Unregistering {}'.format(urlconf)) self.configs.remove(conf) except: pass def nextWorker(self, urlconf): with log.enter(obj=self) as tm: try: conf = self.configs.find(urlconf.__name__) return conf.nextWorker() except: log.err(traceback.format_exc()) for cfg in self.configs: log.err(unicode(cfg)) return None
def __init__(self, persistent=True): self.persistent = persistent self.agents = SortedCollection(key=lambda x: x.url.netloc) self.pool = HTTPConnectionPool(reactor) self.pool.maxPersistentPerHost = getattr(settings, "DTX_WEB_DEFER_MAX_PERSISTENT_PER_HOST", 8) self.pool.cachedConnectionTimeout = getattr(settings, "DTX_WEB_DEFER_CONNECT_TIMEOUT", 10)
def __init__(self): #WampClientProtocol.__init__(self) self.evalSessions = SortedCollection(key=lambda x:x.id)
def __init__(self): self.configs = SortedCollection(key=lambda x: x.urlconf) log.msg(u"Registering static web sites") for address, urlconf in getattr(settings, "DTX_WEB_REMOTE_SITES", []): log.msg(u"Registering {} for {}".format(address, urlconf)) self.addWorker(address, urlconf)
def __init__(self, urlconf): self.urlconf = urlconf self.workers = SortedCollection(key=lambda x: x.address) self.index = 0
def __init__(self, get_item_defer_callable, seconds=120): self.get_item_defer_callable = get_item_defer_callable self.seconds = seconds self.cache = {} self.keys = SortedCollection([], key=lambda item: item['ts']) self.update_requests = {}
def __init__(self): #WampClientProtocol.__init__(self) self.evalSessions = SortedCollection(key=lambda x: x.id)
def __init__(self, urlconf): self.urlconf = urlconf self.workers = SortedCollection(key=lambda x: x.address) self.index = 0
def __init__(self,get_item_defer_callable,seconds=120): self.get_item_defer_callable = get_item_defer_callable self.seconds = seconds self.cache = {} self.keys = SortedCollection([],key=lambda item:item['ts']) self.update_requests = {}
from django.utils import translation from django.utils.translation import ugettext_lazy as _ from django.utils.translation import ugettext from django.utils.translation import ugettext_noop from django.utils.translation import get_language from django.contrib.staticfiles import finders from dtx.utils.snippets.sorted_collection import SortedCollection from dtx.memcache import client as dtx_memcache from dtx.core import logger log = logger.log(__name__) static_files_by_path = SortedCollection(key=lambda x: x.path) class StaticFileInfo: def __init__(self, path, body_digest, mime_type, mime_charset): self.path = path self.path_digest = hashlib.md5(path).hexdigest() self.body_digest = body_digest self.mime_type = mime_type self.mime_charset = mime_charset self.mtime = os.stat(path).st_mtime @classmethod def create(cls, path, body_digest, mime_type, mime_charset): global static_files_by_path info = cls(path, body_digest, mime_type, mime_charset)
from twisted.internet.defer import inlineCallbacks, returnValue, Deferred from twisted.internet import task from twisted.internet import reactor from django.conf import settings from dtx.utils.snippets.sorted_collection import SortedCollection from dtx.core import logger log = logger.log(__name__) ############################################################################################################################## # ############################################################################################################################## wamp_connections = SortedCollection(key=lambda x: x.uuid) wamp_connection_uuid = 1 ############################################################################################################################## # ############################################################################################################################## class ServerFactory(WampServerFactory): def __init__(self, uri, apps, debug=False): WampServerFactory.__init__(self, uri, debug) self.apps = [] for app in apps: if (isinstance(app, (str))): mod = import_module(app) else:
def __init__(self): self.configs = SortedCollection(key=lambda x: x.urlconf) log.msg(u'Registering static web sites') for address, urlconf in getattr(settings, 'DTX_WEB_REMOTE_SITES', []): log.msg(u'Registering {} for {}'.format(address, urlconf)) self.addWorker(address, urlconf)