class SynchronizedContainer:
def __init__(self, obj):
self._o = obj
self._lock = RLock()
def __enter__(self):
if self._lock.__enter__():
return self._o
def __exit__(self, t, v, tb):
self._lock.__exit__(t, v, tb)
def __call__(self, *a, **kw):
with self as sfnc:
return sfnc(*a, **kw)
class LockBox(object):
"Simple abstraction for mutable value locked by a recursive Lock"
def __init__(self, value):
self.lock = RLock()
self.box = value
def __enter__(self):
self.lock.__enter__()
return self.box
def __exit__(self, *a):
self.lock.__exit__(*a)
class PickleableLock(object):
def __init__(self):
self.lock = RLock()
def __getstate__(self):
return ''
def __setstate__(self, value):
return self.__init__()
def __getattr__(self, item):
return self.lock.__getattr__(item)
def __enter__(self):
self.lock.__enter__()
def __exit__(self, exc_type, exc_val, exc_tb):
self.lock.__exit__(exc_type, exc_val, exc_tb)
class LockBox(object):
"Simple abstraction for mutable value locked by a recursive Lock"
def __init__(self, value):
self.lock = RLock()
self.box = value
def set(self, value):
"""
Safely set the box to a new value.
Useful for immutable value types
"""
# we're recursive so this should work if we already hold it
with self as old_value:
self.box = value
return old_value
def get(self):
"""
Safely get the current value
This is only useful for immutable value types
"""
with self as value:
return value
def cas(self, before, after, compare=lambda x, y: x is y):
with self as current:
if compare(before, current):
self.box = after
return True
return False
def __enter__(self):
self.lock.__enter__()
return self.box
def __exit__(self, *a):
self.lock.__exit__(*a)
class BoolLock(
object
): # Easily activate/deactivate locking without changes to the code
def __init__(self, use_lock=False):
self.use_lock = use_lock
self.lock = RLock()
def release(self):
if self.use_lock:
self.lock.release()
def acquire(self, blocking=True, timeout=-1):
if self.use_lock:
self.lock.acquire(blocking, timeout)
def __enter__(self, blocking=True, timeout=-1):
return self if not self.use_lock else self.lock.__enter__(
blocking, timeout)
def __exit__(self, exc_type, exc_val, exc_tb):
if self.use_lock:
self.lock.__exit__(exc_type, exc_val, exc_tb)
class Lockable:
def __init__(self):
self._Lock = RLock()
def __enter__(self):
return self._Lock.__enter__()
def __exit__(self, exc_type, exc_value, traceback):
return self._Lock.__exit__(exc_type, exc_value, traceback)
def acquire(self, *params, **args):
return self._Lock.acquire(*params, **args)
def release(self):
return self._Lock.release()
class Lockable:
def __init__(self):
self._Lock = RLock()
self._WakeUp = Condition(self._Lock)
def __enter__(self):
return self._Lock.__enter__()
def __exit__(self, exc_type, exc_value, traceback):
return self._Lock.__exit__(exc_type, exc_value, traceback)
def wait(self, timeout = None):
with self._Lock:
self._WakeUp.wait(timeout)
def notify(self, n=1):
with self._Lock:
self._WakeUp.notify(n)
class AutodiscoverCache:
"""Stores the translation from (email domain, credentials) -> AutodiscoverProtocol object so we can re-use TCP
connections to an autodiscover server within the same process. Also persists the email domain -> (autodiscover
endpoint URL, auth_type) translation to the filesystem so the cache can be shared between multiple processes.
According to Microsoft, we may forever cache the (email domain -> autodiscover endpoint URL) mapping, or until
it stops responding. My previous experience with Exchange products in mind, I'm not sure if I should trust that
advice. But it could save some valuable seconds every time we start a new connection to a known server. In any
case, the persistent storage must not contain any sensitive information since the cache could be readable by
unprivileged users. Domain, endpoint and auth_type are OK to cache since this info is make publicly available on
HTTP and DNS servers via the autodiscover protocol. Just don't persist any credentials info.
If an autodiscover lookup fails for any reason, the corresponding cache entry must be purged.
'shelve' is supposedly thread-safe and process-safe, which suits our needs.
"""
def __init__(self):
self._protocols = {
} # Mapping from (domain, credentials) to AutodiscoverProtocol
self._lock = RLock()
@property
def _storage_file(self):
return AUTODISCOVER_PERSISTENT_STORAGE
def clear(self):
# Wipe the entire cache
with shelve_open_with_failover(self._storage_file) as db:
db.clear()
self._protocols.clear()
def __len__(self):
return len(self._protocols)
def __contains__(self, key):
domain = key[0]
with shelve_open_with_failover(self._storage_file) as db:
return str(domain) in db
def __getitem__(self, key):
protocol = self._protocols.get(key)
if protocol:
return protocol
domain, credentials = key
with shelve_open_with_failover(self._storage_file) as db:
endpoint, auth_type, retry_policy = db[str(
domain)] # It's OK to fail with KeyError here
protocol = AutodiscoverProtocol(
config=Configuration(service_endpoint=endpoint,
credentials=credentials,
auth_type=auth_type,
retry_policy=retry_policy))
self._protocols[key] = protocol
return protocol
def __setitem__(self, key, protocol):
# Populate both local and persistent cache
domain = key[0]
with shelve_open_with_failover(self._storage_file) as db:
# Don't change this payload without bumping the cache file version in shelve_filename()
db[str(domain)] = (protocol.service_endpoint, protocol.auth_type,
protocol.retry_policy)
self._protocols[key] = protocol
def __delitem__(self, key):
# Empty both local and persistent cache. Don't fail on non-existing entries because we could end here
# multiple times due to race conditions.
domain = key[0]
with shelve_open_with_failover(self._storage_file) as db:
try:
del db[str(domain)]
except KeyError:
pass
try:
del self._protocols[key]
except KeyError:
pass
def close(self):
# Close all open connections
for (domain, _), protocol in self._protocols.items():
log.debug('Domain %s: Closing sessions', domain)
protocol.close()
del protocol
self._protocols.clear()
def __enter__(self):
self._lock.__enter__()
def __exit__(self, *args, **kwargs):
self._lock.__exit__(*args, **kwargs)
def __del__(self):
# pylint: disable=bare-except
try:
self.close()
except Exception: # nosec
# __del__ should never fail
pass
def __str__(self):
return str(self._protocols)
class WPApp(object):
Version = "Undefined"
MIME_TYPES_BASE = {
"gif": "image/gif",
"jpg": "image/jpeg",
"jpeg": "image/jpeg",
"js": "text/javascript",
"html": "text/html",
"txt": "text/plain",
"css": "text/css"
}
def __init__(self,
root_class,
strict=False,
static_path="/static",
static_location="static",
enable_static=False,
prefix=None,
replace_prefix=None,
disable_robots=True):
assert issubclass(root_class, WPHandler)
self.RootClass = root_class
self.JEnv = None
self._AppLock = RLock()
self._Strict = strict
self.ScriptHome = None
self.StaticPath = static_path
self.StaticLocation = static_location
self.StaticEnabled = enable_static and static_location
self.Initialized = False
self.DisableRobots = disable_robots
self.Prefix = prefix
self.ReplacePrefix = replace_prefix
def _app_lock(self):
return self._AppLock
def __enter__(self):
return self._AppLock.__enter__()
def __exit__(self, *params):
return self._AppLock.__exit__(*params)
# override
@app_synchronized
def acceptIncomingTransfer(self, method, uri, headers):
return True
@app_synchronized
def initJinjaEnvironment(self, tempdirs=[], filters={}, globals={}):
# to be called by subclass
#print "initJinja2(%s)" % (tempdirs,)
from jinja2 import Environment, FileSystemLoader
if not isinstance(tempdirs, list):
tempdirs = [tempdirs]
self.JEnv = Environment(loader=FileSystemLoader(tempdirs))
for n, f in filters.items():
self.JEnv.filters[n] = f
self.JGlobals = {}
self.JGlobals.update(globals)
@app_synchronized
def setJinjaFilters(self, filters):
for n, f in filters.items():
self.JEnv.filters[n] = f
@app_synchronized
def setJinjaGlobals(self, globals):
self.JGlobals = {}
self.JGlobals.update(globals)
def applicationErrorResponse(self, headline, exc_info):
typ, val, tb = exc_info
exc_text = traceback.format_exception(typ, val, tb)
exc_text = ''.join(exc_text)
text = """<html><body><h2>Application error</h2>
<h3>%s</h3>
<pre>%s</pre>
</body>
</html>""" % (headline, exc_text)
#print exc_text
return Response(text, status='500 Application Error')
def static(self, relpath):
while ".." in relpath:
relpath = relpath.replace("..", ".")
home = self.StaticLocation
path = os.path.join(home, relpath)
#print "static: path=", path
try:
st_mode = os.stat(path).st_mode
if not stat.S_ISREG(st_mode):
#print "not a regular file"
raise ValueError("Not regular file")
except:
#raise
return Response("Not found", status=404)
ext = path.rsplit('.', 1)[-1]
mime_type = self.MIME_TYPES_BASE.get(ext, "text/html")
def read_iter(f):
while True:
data = f.read(100000)
if not data: break
yield data
#print "returning response..."
return Response(app_iter=read_iter(open(path, "rb")),
content_type=mime_type)
def convertPath(self, path):
if self.Prefix is not None:
matched = ""
ok = False
if path == self.Prefix:
matched = path
ok = True
elif path.startswith(self.Prefix + '/'):
matched = self.Prefix
ok = True
if not ok:
return None
#print("convertPath: %s:%s -> %s %s" % (path, self.Prefix, matched, ok))
if self.ReplacePrefix is not None:
path = self.ReplacePrefix + (path[len(matched):] or "/")
#print("convertPath: -> %s" % (path,))
return path
def __call__(self, environ, start_response):
#print 'app call ...'
path = environ.get('PATH_INFO', '')
environ["WebPie.original_path"] = path
#print 'path:', path_down
path = self.convertPath(path)
if path is None:
return HTTPNotFound()(environ, start_response)
environ["PATH_INFO"] = path
#print("__call__: path=%s" % (path,))
req = Request(environ)
if not self.Initialized:
self.ScriptName = environ.get('SCRIPT_NAME', '')
self.Script = environ.get('SCRIPT_FILENAME',
os.environ.get('UWSGI_SCRIPT_FILENAME'))
self.ScriptHome = os.path.dirname(self.Script
or sys.argv[0]) or "."
if self.StaticEnabled:
if not self.StaticLocation[0] in ('.', '/'):
self.StaticLocation = self.ScriptHome + "/" + self.StaticLocation
#print "static location:", self.StaticLocation
self.Initialized = True
if self.StaticEnabled and path.startswith(self.StaticPath + "/"):
resp = self.static(path[len(self.StaticPath) + 1:])
elif self.DisableRobots and path.endswith("/robots.txt"):
resp = Response("User-agent: *\nDisallow: /\n",
content_type="text/plain")
else:
root = self.RootClass(req, self)
try:
return root.wsgi_call(environ, start_response)
except:
resp = self.applicationErrorResponse("Uncaught exception",
sys.exc_info())
return resp(environ, start_response)
def JinjaGlobals(self):
# override me
return {}
def addEnvironment(self, d):
params = {}
params.update(self.JGlobals)
params.update(self.JinjaGlobals())
params.update(d)
return params
def render_to_string(self, temp, **kv):
t = self.JEnv.get_template(temp)
return t.render(self.addEnvironment(kv))
def render_to_iterator(self, temp, **kv):
t = self.JEnv.get_template(temp)
return t.generate(self.addEnvironment(kv))
def run_server(self, port, **args):
from .HTTPServer import HTTPServer
srv = HTTPServer(port, self, **args)
srv.start()
srv.join()
class WPApp(object):
Version = "Undefined"
def __init__(self,
root_class_or_handler,
strict=False,
static_path="/static",
static_location=None,
enable_static=False,
prefix=None,
replace_prefix="",
environ={}):
self.RootHandler = self.RootClass = None
if inspect.isclass(root_class_or_handler):
self.RootClass = root_class_or_handler
else:
self.RootHandler = root_class_or_handler
#print("WPApp.__init__: self.RootClass=", self.RootClass, " self.RootHandler=", self.RootHandler)
self.JEnv = None
self._AppLock = RLock()
self.ScriptHome = None
self.Initialized = False
self.Prefix = prefix
self.ReplacePrefix = replace_prefix
self.HandlerParams = []
self.HandlerArgs = {}
self.Environ = {}
self.Environ.update(environ)
def _app_lock(self):
return self._AppLock
def __enter__(self):
return self._AppLock.__enter__()
def __exit__(self, *params):
return self._AppLock.__exit__(*params)
# override
@app_synchronized
def acceptIncomingTransfer(self, method, uri, headers):
return True
def init(self):
pass
@app_synchronized
def initJinjaEnvironment(self, tempdirs=[], filters={}, globals={}):
# to be called by subclass
#print "initJinja2(%s)" % (tempdirs,)
from jinja2 import Environment, FileSystemLoader
if not isinstance(tempdirs, list):
tempdirs = [tempdirs]
self.JEnv = Environment(loader=FileSystemLoader(tempdirs))
for n, f in filters.items():
self.JEnv.filters[n] = f
self.JGlobals = {}
self.JGlobals.update(globals)
@app_synchronized
def setJinjaFilters(self, filters):
for n, f in filters.items():
self.JEnv.filters[n] = f
@app_synchronized
def setJinjaGlobals(self, globals):
self.JGlobals = {}
self.JGlobals.update(globals)
def applicationErrorResponse(self, headline, exc_info):
typ, val, tb = exc_info
exc_text = traceback.format_exception(typ, val, tb)
exc_text = ''.join(exc_text)
text = """<html><body><h2>Application error</h2>
<h3>%s</h3>
<pre>%s</pre>
</body>
</html>""" % (headline, exc_text)
#print exc_text
return Response(text, status='500 Application Error')
def convertPath(self, path):
if self.Prefix is not None:
matched = None
if path == self.Prefix:
matched = path
elif path.startswith(self.Prefix + '/'):
matched = self.Prefix
if matched is None:
return None
if self.ReplacePrefix is not None:
path = self.ReplacePrefix + path[len(matched):]
path = path or "/"
#print(f"converted to: [{path}]")
return path
def handler_options(self, *params, **args):
self.HandlerParams = params
self.HandlerArgs = args
return self
def find_web_method(self, handler, request, path, path_down, args):
#
# walks down the tree of handler finds the web method and calls it
# returs the Response
#
path = path or "/"
method = None
while path_down and not path_down[0]:
path_down.pop(0)
#is_wp_handler = isinstance(handler, WPHandler)
#print(f"find_web_method({handler}, WPHandler:{is_wp_handler}, path={path}, path_down={path_down})")
if isinstance(handler, WPHandler):
handler.Path = path
if path_down:
name = path_down[0]
attr = handler.step_down(name)
if attr is not None:
if not path.endswith("/"): path += "/"
return self.find_web_method(attr, request, path + name,
path_down[1:], args)
if callable(handler):
method = handler
elif not path_down:
prefix = (path.split("/")[-1] or ".") + "/"
redirect = prefix + handler.DefaultMethod
raise HTTPFound(location=redirect)
elif callable(handler):
method = handler
relpath = "/".join(path_down)
return method, relpath
def parseQuery(self, query):
out = {}
for w in (query or "").split("&"):
if w:
words = w.split("=", 1)
k = words[0]
if k:
v = None
if len(words) > 1: v = words[1]
if k in out:
old = out[k]
if type(old) != type([]):
old = [old]
out[k] = old
out[k].append(v)
else:
out[k] = v
return out
def wsgi_call(self, root_handler, environ, start_response):
# path_to = '/'
path = environ.get('PATH_INFO', '')
#while "//" in path:
# path.replace("//", "/")
path_down = path.split("/")
#while '' in path_down:
# path_down.remove('')
args = self.parseQuery(environ.get("QUERY_STRING", ""))
request = Request(environ)
try:
method, relpath = self.find_web_method(root_handler, request, "",
path_down, args)
#print("WPApp.wsgi_call: method:", method, " relpath:", relpath)
if method is None:
response = HTTPNotFound("Invalid path %s" % (path, ))
else:
#print("method:", method)
response = method(request, relpath, **args)
#print("response:", response)
except HTTPFound as val:
# redirect
response = val
except HTTPException as val:
#print 'caught:', type(val), val
response = val
except HTTPResponseException as val:
#print 'caught:', type(val), val
response = val
except:
response = self.applicationErrorResponse("Uncaught exception",
sys.exc_info())
try:
response = makeResponse(response)
except ValueError as e:
response = self.applicationErrorResponse(str(e), sys.exc_info())
out = response(environ, start_response)
if isinstance(root_handler, WPHandler):
root_handler.destroy()
root_handler._destroy()
return out
def __call__(self, environ, start_response):
path = environ.get('PATH_INFO', '')
#print('app call: path:', path)
if not "WebPie.original_path" in environ:
environ["WebPie.original_path"] = path
environ.update(self.Environ)
#print 'path:', path_down
path = self.convertPath(path)
if path is None:
return HTTPNotFound()(environ, start_response)
#if (not path or path=="/") and self.DefaultPath is not None:
# #print ("redirecting to", self.DefaultPath)
# return HTTPFound(location=self.DefaultPath)(environ, start_response)
environ["PATH_INFO"] = path
req = Request(environ)
if not self.Initialized:
self.ScriptName = environ.get('SCRIPT_NAME') or ''
self.Script = environ.get('SCRIPT_FILENAME') or \
os.environ.get('UWSGI_SCRIPT_FILENAME')
self.ScriptHome = os.path.dirname(self.Script
or sys.argv[0]) or "."
self.init()
self.Initialized = True
self.init()
root_handler = self.RootHandler or self.RootClass(
req, self, *self.HandlerParams, **self.HandlerArgs)
#print("root_handler:", root_handler)
try:
return self.wsgi_call(root_handler, environ, start_response)
except:
resp = self.applicationErrorResponse("Uncaught exception",
sys.exc_info())
return resp(environ, start_response)
def init(self):
# overraidable. will be called once after self.ScriptName, self.ScriptHome, self.Script are initialized
# it is good idea to init Jinja environment here
pass
def jinja_globals(self):
# override me
return {}
def add_globals(self, d):
params = {}
params.update(self.JGlobals)
params.update(self.jinja_globals())
params.update(d)
return params
def render_to_string(self, temp, **kv):
t = self.JEnv.get_template(temp)
return t.render(self.add_globals(kv))
def render_to_iterator(self, temp, **kv):
t = self.JEnv.get_template(temp)
return t.generate(self.add_globals(kv))
def run_server(self, port, **args):
from .HTTPServer import HTTPServer
srv = HTTPServer(port, self, **args)
srv.start()
srv.join()
class MemoryQueue(Generic[T]):
"""
Queue which remembers items that were put into it, until forced to forget.
Items can be added to the queue without limits.
Offering items, however, comes with limitations. Item cannot be offered twice, as long as it is in the queue or
memory.
Queue is thread-safe. In order to perform multiple operations in succession, use the queue as context. This will
acquire the internal lock used by the queue.
"""
def __init__(self):
self.__lock = RLock()
self.__queue: List[T] = []
self.__memory: List[T] = []
def __enter__(self):
self.__lock.__enter__()
def __exit__(self, *args):
self.__lock.__exit__(*args)
def __len__(self) -> int:
with self:
return len(self.__queue)
def __getitem__(self, key) -> T:
with self:
return self.__queue[key]
def __setitem__(self, key, value):
with self:
self.__queue[key] = value
def __delitem__(self, key):
with self:
del self.__queue[key]
def __iter__(self) -> Iterator[T]:
with self:
copy = self.__queue[:]
return iter(copy)
def __reversed__(self) -> Iterator[T]:
with self:
reverse_copy = self.__queue[::-1]
return iter(reverse_copy)
def __contains__(self, item: T) -> bool:
with self:
return item in self.__queue
def __str__(self) -> str:
with self:
return str(self.__queue)
def __eq__(self, o: object) -> bool:
try:
# noinspection PyTypeChecker
other = iter(o)
except TypeError:
return False
with self:
return all(a == b for a, b in zip_longest(self, other, fillvalue=NON_EXISTENT))
def next(self) -> Optional[T]:
"""
FIFO pop operation. If an item is returned, it is memorized so that it still cannot be offered again.
:returns next item in the queue, if any
"""
with self:
try:
item = self.__queue.pop(0)
except IndexError:
return None
self.__memory.append(item)
return item
def add(self, item: T) -> int:
"""
Adds item to the end of this queue. Uniqueness or memory is ignored.
:returns position the item was added to
"""
if item is not None:
with self:
self.__queue.append(item)
return len(self)
def add_all(self, *items: T) -> List[int]:
"""
Adds multiple items to the end of this queue. Uniqueness or memory is ignored.
:returns positions the item were added to
"""
with self:
return [self.add(item) for item in items]
def offer(self, item: T, match: Callable[[T], bool]) -> int:
"""
Adds an item to the queue, but only if this item has not been added yet. Does not allow to add items in memory.
If an item that matches given predicate is already in the queue, it is replaced instead. Only the last matching
item will be replaced.
:returns position the item was added to; 0 if it is in memory; -position if it is in queue
"""
with self:
if item in self.__memory:
return 0
return self.__already_in_queue(item) or self.__replace_or_add(item, match)
def find(self, match: Callable[[T], bool]) -> Optional[T]:
"""
:returns the last matching item, if any
"""
with self:
i = self.__find(match)
return self[i] if i >= 0 else None
def bump(self, match: Callable[[T], bool]) -> Optional[T]:
"""
Bumps the last matching item to the top of the queue.
:returns item that was bumped, None if it was not found
"""
with self:
i = self.__find(match)
if i >= 0:
item = self[i]
self.__queue.insert(0, item)
del self[i + 1]
return item
def last(self) -> Optional[T]:
"""
:returns last item in memory, if any
"""
with self:
return self.__memory[-1] if self.__memory else None
def mandela(self, item: T):
"""
Sets or replaces the last item in memory
"""
with self:
if self.__memory:
del self.__memory[-1]
self.__memory.append(item)
def memory(self) -> List[T]:
"""
:returns copy of memory
"""
with self:
return self.__memory[:]
def forget(self):
"""
Clears memory
"""
with self:
self.__memory.clear()
def __already_in_queue(self, item: T) -> int:
return -1 - self.__find(lambda t: t == item)
def __replace_or_add(self, item, match: Callable[[T], bool]) -> int:
i = self.__find(match)
if i >= 0:
self[i] = item
return i + 1
return self.add(item)
def __find(self, match: Callable[[T], bool]) -> int:
i = len(self)
for in_queue in reversed(self):
i -= 1
if match(in_queue):
return i
return -1
