def format(self, *args, **kwargs):
args = [unicode(
s, 'utf-8') if isinstance(s, str) else s for s in args]
kwargs = dict((unicode(k, 'utf-8') if isinstance(k, str) else k,
unicode(v, 'utf-8') if isinstance(v, str) else v)
for k, v in iteritems(kwargs))
return str.__new__(Utf8, unicode(self, 'utf-8').format(*args, **kwargs).encode('utf-8'))
def format(self, *args, **kwargs):
args = [unicode(
s, 'utf-8') if isinstance(s, str) else s for s in args]
kwargs = dict((unicode(k, 'utf-8') if isinstance(k, str) else k,
unicode(v, 'utf-8') if isinstance(v, str) else v)
for k, v in iteritems(kwargs))
return str.__new__(Utf8, unicode(self, 'utf-8').format(*args, **kwargs).encode('utf-8'))
def apply_filter(self, message, symbols={}, filter=None, ftag=None):
def get_tr(message, prefix, filter):
s = self.get_t(message, prefix)
return filter(s) if filter else self.filter(s)
if filter:
prefix = '@' + (ftag or 'userdef') + '\x01'
else:
prefix = '@' + self.ftag + '\x01'
message = get_from_cache(
self.cache, prefix + message,
lambda: get_tr(message, prefix, filter))
if symbols or symbols == 0 or symbols == "":
if isinstance(symbols, dict):
symbols.update(
(key, xmlescape(value).translate(ttab_in))
for key, value in iteritems(symbols)
if not isinstance(value, NUMBERS))
else:
if not isinstance(symbols, tuple):
symbols = (symbols,)
symbols = tuple(
value if isinstance(value, NUMBERS)
else to_native(xmlescape(value)).translate(ttab_in)
for value in symbols)
message = self.params_substitution(message, symbols)
return to_native(XML(message.translate(ttab_out)).xml())
def to(self, responder, env=None):
env = env or {}
status = self.status
headers = self.headers
if status in defined_status:
status = '%d %s' % (status, defined_status[status])
elif isinstance(status, int):
status = '%d UNKNOWN ERROR' % status
else:
status = str(status)
if not regex_status.match(status):
status = '500 %s' % (defined_status[500])
headers.setdefault('Content-Type', 'text/html; charset=UTF-8')
body = self.body
if status[:1] == '4':
if not body:
body = status
if isinstance(body, (str, bytes, bytearray)):
headers['Content-Length'] = len(body)
rheaders = []
for k, v in iteritems(headers):
if isinstance(v, list):
rheaders += [(k, str(item)) for item in v]
elif v is not None:
rheaders.append((k, str(v)))
responder(status, rheaders)
if env.get('request_method', '') == 'HEAD':
return ['']
elif isinstance(body, (str, bytes, bytearray)):
return [body]
elif hasattr(body, '__iter__'):
return body
else:
return [str(body)]
def to(self, responder, env=None):
env = env or {}
status = self.status
headers = self.headers
if status in defined_status:
status = '%d %s' % (status, defined_status[status])
elif isinstance(status, int):
status = '%d UNKNOWN ERROR' % status
else:
status = str(status)
if not regex_status.match(status):
status = '500 %s' % (defined_status[500])
headers.setdefault('Content-Type', 'text/html; charset=UTF-8')
body = self.body
if status[:1] == '4':
if not body:
body = status
if isinstance(body, (str, bytes, bytearray)):
headers['Content-Length'] = len(body)
rheaders = []
for k, v in iteritems(headers):
if isinstance(v, list):
rheaders += [(k, str(item)) for item in v]
elif not v is None:
rheaders.append((k, str(v)))
responder(status, rheaders)
if env.get('request_method', '') == 'HEAD':
return ['']
elif isinstance(body, (str, bytes, bytearray)):
return [body]
elif hasattr(body, '__iter__'):
return body
else:
return [str(body)]
def parse_get_vars(self):
"""Takes the QUERY_STRING and unpacks it to get_vars
"""
query_string = self.env.get('query_string', '')
dget = urlparse.parse_qs(query_string, keep_blank_values=1) # Ref: https://docs.python.org/2/library/cgi.html#cgi.parse_qs
get_vars = self._get_vars = Storage(dget)
for (key, value) in iteritems(get_vars):
if isinstance(value, list) and len(value) == 1:
get_vars[key] = value[0]
def include_meta(self):
s = "\n"
for meta in iteritems((self.meta or {})):
k, v = meta
if isinstance(v, dict):
s += '<meta' + ''.join(' %s="%s"' % (xmlescape(key), to_native(xmlescape(v[key]))) for key in v) +' />\n'
else:
s += '<meta name="%s" content="%s" />\n' % (k, to_native(xmlescape(v)))
self.write(s, escape=False)
def include_meta(self):
s = "\n"
for meta in iteritems((self.meta or {})):
k, v = meta
if isinstance(v, dict):
s += '<meta' + ''.join(' %s="%s"' % (xmlescape(key),
to_native(xmlescape(v[key]))) for key in v) + ' />\n'
else:
s += '<meta name="%s" content="%s" />\n' % (k, to_native(xmlescape(v)))
self.write(s, escape=False)
def parse_get_vars(self):
"""Takes the QUERY_STRING and unpacks it to get_vars
"""
query_string = self.env.get('query_string', '')
dget = urlparse.parse_qs(query_string, keep_blank_values=1)
# Ref: https://docs.python.org/2/library/cgi.html#cgi.parse_qs
get_vars = self._get_vars = Storage(dget)
for (key, value) in iteritems(get_vars):
if isinstance(value, list) and len(value) == 1:
get_vars[key] = value[0]
def __mod__(self, right):
if isinstance(right, tuple):
right = tuple(unicode(v, 'utf-8') if isinstance(v, str) else v
for v in right)
elif isinstance(right, dict):
right = dict((unicode(k, 'utf-8') if isinstance(k, str) else k,
unicode(v, 'utf-8') if isinstance(v, str) else v)
for k, v in iteritems(right))
elif isinstance(right, str):
right = unicode(right, 'utf-8')
return str.__new__(Utf8, unicode(self, 'utf-8').__mod__(right).encode('utf-8'))
def parse_all_vars(self):
"""Merges get_vars and post_vars to vars
"""
self._vars = copy.copy(self.get_vars)
for key, value in iteritems(self.post_vars):
if key not in self._vars:
self._vars[key] = value
else:
if not isinstance(self._vars[key], list):
self._vars[key] = [self._vars[key]]
self._vars[key] += value if isinstance(value, list) else [value]
def __mod__(self, right):
if isinstance(right, tuple):
right = tuple(unicode(v, 'utf-8') if isinstance(v, str) else v
for v in right)
elif isinstance(right, dict):
right = dict((unicode(k, 'utf-8') if isinstance(k, str) else k,
unicode(v, 'utf-8') if isinstance(v, str) else v)
for k, v in iteritems(right))
elif isinstance(right, str):
right = unicode(right, 'utf-8')
return str.__new__(Utf8, unicode(self, 'utf-8').__mod__(right).encode('utf-8'))
def parse_all_vars(self):
"""Merges get_vars and post_vars to vars
"""
self._vars = copy.copy(self.get_vars)
for key, value in iteritems(self.post_vars):
if key not in self._vars:
self._vars[key] = value
else:
if not isinstance(self._vars[key], list):
self._vars[key] = [self._vars[key]]
self._vars[key] += value if isinstance(value, list) else [value]
def toolbar(self):
from gluon.html import DIV, SCRIPT, BEAUTIFY, TAG, A
BUTTON = TAG.button
admin = URL("admin", "default", "design", extension='html',
args=current.request.application)
from gluon.dal import DAL
dbstats = []
dbtables = {}
infos = DAL.get_instances()
for k, v in iteritems(infos):
dbstats.append(TABLE(*[TR(PRE(row[0]), '%.2fms' % (row[1]*1000))
for row in v['dbstats']]))
dbtables[k] = dict(defined=v['dbtables']['defined'] or '[no defined tables]',
lazy=v['dbtables']['lazy'] or '[no lazy tables]')
u = web2py_uuid()
backtotop = A('Back to top', _href="#totop-%s" % u)
# Convert lazy request.vars from property to Storage so they
# will be displayed in the toolbar.
request = copy.copy(current.request)
request.update(vars=current.request.vars,
get_vars=current.request.get_vars,
post_vars=current.request.post_vars)
return DIV(
BUTTON('design', _onclick="document.location='%s'" % admin),
BUTTON('request',
_onclick="jQuery('#request-%s').slideToggle()" % u),
BUTTON('response',
_onclick="jQuery('#response-%s').slideToggle()" % u),
BUTTON('session',
_onclick="jQuery('#session-%s').slideToggle()" % u),
BUTTON('db tables',
_onclick="jQuery('#db-tables-%s').slideToggle()" % u),
BUTTON('db stats',
_onclick="jQuery('#db-stats-%s').slideToggle()" % u),
DIV(BEAUTIFY(request), backtotop,
_class="w2p-toolbar-hidden", _id="request-%s" % u),
DIV(BEAUTIFY(current.session), backtotop,
_class="w2p-toolbar-hidden", _id="session-%s" % u),
DIV(BEAUTIFY(current.response), backtotop,
_class="w2p-toolbar-hidden", _id="response-%s" % u),
DIV(BEAUTIFY(dbtables), backtotop,
_class="w2p-toolbar-hidden", _id="db-tables-%s" % u),
DIV(BEAUTIFY(dbstats), backtotop,
_class="w2p-toolbar-hidden", _id="db-stats-%s" % u),
SCRIPT("jQuery('.w2p-toolbar-hidden').hide()"),
_id="totop-%s" % u
)
def toolbar(self):
from gluon.html import DIV, SCRIPT, BEAUTIFY, TAG, A
BUTTON = TAG.button
admin = URL("admin", "default", "design", extension='html',
args=current.request.application)
from gluon.dal import DAL
dbstats = []
dbtables = {}
infos = DAL.get_instances()
for k, v in iteritems(infos):
dbstats.append(TABLE(*[TR(PRE(row[0]), '%.2fms' % (row[1]*1000))
for row in v['dbstats']]))
dbtables[k] = dict(defined=v['dbtables']['defined'] or '[no defined tables]',
lazy=v['dbtables']['lazy'] or '[no lazy tables]')
u = web2py_uuid()
backtotop = A('Back to top', _href="#totop-%s" % u)
# Convert lazy request.vars from property to Storage so they
# will be displayed in the toolbar.
request = copy.copy(current.request)
request.update(vars=current.request.vars,
get_vars=current.request.get_vars,
post_vars=current.request.post_vars)
return DIV(
BUTTON('design', _onclick="document.location='%s'" % admin),
BUTTON('request',
_onclick="jQuery('#request-%s').slideToggle()" % u),
BUTTON('response',
_onclick="jQuery('#response-%s').slideToggle()" % u),
BUTTON('session',
_onclick="jQuery('#session-%s').slideToggle()" % u),
BUTTON('db tables',
_onclick="jQuery('#db-tables-%s').slideToggle()" % u),
BUTTON('db stats',
_onclick="jQuery('#db-stats-%s').slideToggle()" % u),
DIV(BEAUTIFY(request), backtotop,
_class="w2p-toolbar-hidden", _id="request-%s" % u),
DIV(BEAUTIFY(current.session), backtotop,
_class="w2p-toolbar-hidden", _id="session-%s" % u),
DIV(BEAUTIFY(current.response), backtotop,
_class="w2p-toolbar-hidden", _id="response-%s" % u),
DIV(BEAUTIFY(dbtables), backtotop,
_class="w2p-toolbar-hidden", _id="db-tables-%s" % u),
DIV(BEAUTIFY(dbstats), backtotop,
_class="w2p-toolbar-hidden", _id="db-stats-%s" % u),
SCRIPT("jQuery('.w2p-toolbar-hidden').hide()"),
_id="totop-%s" % u
)
def translate(self, message, symbols):
"""
Gets cached translated message with inserted parameters(symbols)
"""
message = get_from_cache(self.cache, message,
lambda: self.get_t(message))
if symbols or symbols == 0 or symbols == "":
if isinstance(symbols, dict):
symbols.update((key, str(value).translate(ttab_in))
for key, value in iteritems(symbols)
if not isinstance(value, NUMBERS))
else:
if not isinstance(symbols, tuple):
symbols = (symbols, )
symbols = tuple(
value if isinstance(value, NUMBERS
) else str(value).translate(ttab_in)
for value in symbols)
message = self.params_substitution(message, symbols)
return message.translate(ttab_out)
def translate(self, message, symbols):
"""
Gets cached translated message with inserted parameters(symbols)
"""
message = get_from_cache(self.cache, message,
lambda: self.get_t(message))
if symbols or symbols == 0 or symbols == "":
if isinstance(symbols, dict):
symbols.update(
(key, str(value).translate(ttab_in))
for key, value in iteritems(symbols)
if not isinstance(value, NUMBERS))
else:
if not isinstance(symbols, tuple):
symbols = (symbols,)
symbols = tuple(
value if isinstance(value, NUMBERS)
else str(value).translate(ttab_in)
for value in symbols)
message = self.params_substitution(message, symbols)
return message.translate(ttab_out)
def testTypeRecognition(self):
rtn = contenttype('.png')
self.assertEqual(rtn, 'image/png')
rtn = contenttype('.gif')
self.assertEqual(rtn, 'image/gif')
rtn = contenttype('.tar.bz2')
self.assertEqual(rtn, 'application/x-bzip-compressed-tar')
# test overrides and additions
mapping = {
'.load': 'text/html; charset=utf-8',
'.json': 'application/json',
'.jsonp': 'application/jsonp',
'.pickle': 'application/python-pickle',
'.w2p': 'application/w2p',
'.md': 'text/x-markdown; charset=utf-8'
}
for k, v in iteritems(mapping):
self.assertEqual(contenttype(k), v)
# test without dot extension
rtn = contenttype('png')
self.assertEqual(rtn, 'text/plain; charset=utf-8')
def testTypeRecognition(self):
rtn = contenttype('.png')
self.assertEqual(rtn, 'image/png')
rtn = contenttype('.gif')
self.assertEqual(rtn, 'image/gif')
rtn = contenttype('.tar.bz2')
self.assertEqual(rtn, 'application/x-bzip-compressed-tar')
# test overrides and additions
mapping = {
'.load': 'text/html; charset=utf-8',
'.json': 'application/json',
'.jsonp': 'application/jsonp',
'.pickle': 'application/python-pickle',
'.w2p': 'application/w2p',
'.md': 'text/x-markdown; charset=utf-8'
}
for k, v in iteritems(mapping):
self.assertEqual(contenttype(k), v)
# test without dot extension
rtn = contenttype('png')
self.assertEqual(rtn, 'text/plain; charset=utf-8')
def post(self,
url,
data=None,
cookies=None,
headers=None,
auth=None,
method='auto',
charset='utf-8'):
self.url = self.app + url
# if this POST form requires a postback do it
if data and '_formname' in data and self.postbacks and \
self.history and self.history[-1][1] != self.url:
# to bypass the web2py CSRF need to get formkey
# before submitting the form
self.get(url, cookies=cookies, headers=headers, auth=auth)
# unless cookies are specified, recycle cookies
if cookies is None:
cookies = self.cookies
cookies = cookies or {}
headers = headers or {}
args = [
urllib2.HTTPCookieProcessor(self.cookiejar),
urllib2.HTTPHandler(debuglevel=0)
]
# if required do basic auth
if auth:
auth_handler = urllib2.HTTPBasicAuthHandler()
auth_handler.add_password(**auth)
args.append(auth_handler)
opener = urllib2.build_opener(*args)
# copy headers from dict to list of key,value
headers_list = []
for key, value in iteritems(self.default_headers):
if not key in headers:
headers[key] = value
for key, value in iteritems(headers):
if isinstance(value, (list, tuple)):
for v in value:
headers_list.append((key, v))
else:
headers_list.append((key, value))
# move cookies to headers
for key, value in iteritems(cookies):
headers_list.append(('Cookie', '%s=%s' % (key, value)))
# add headers to request
for key, value in headers_list:
opener.addheaders.append((key, str(value)))
# assume everything is ok and make http request
error = None
try:
if isinstance(data, str):
self.method = 'POST' if method == 'auto' else method
elif isinstance(data, dict):
self.method = 'POST' if method == 'auto' else method
# if there is only one form, set _formname automatically
if not '_formname' in data and len(self.forms) == 1:
data['_formname'] = next(iter(
self.forms.keys())) # Use the first key
# if there is no formkey but it is known, set it
if '_formname' in data and not '_formkey' in data and \
data['_formname'] in self.forms:
data['_formkey'] = self.forms[data['_formname']]
# time the POST request
data = urlencode(data, doseq=True)
else:
self.method = 'GET' if method == 'auto' else method
data = None
t0 = time.time()
self.response = opener.open(self.url, to_bytes(data))
self.time = time.time() - t0
except urllib2.HTTPError as er:
error = er
# catch HTTP errors
self.time = time.time() - t0
self.response = er
if hasattr(self.response, 'getcode'):
self.status = self.response.getcode()
else: #python2.5
self.status = None
self.text = self.response.read()
if charset:
if charset == 'auto':
charset = self.response.headers.getparam('charset')
self.text = to_native(self.text, charset)
# In PY3 self.response.headers are case sensitive
self.headers = dict()
for h in self.response.headers:
self.headers[h.lower()] = self.response.headers[h]
# treat web2py tickets as special types of errors
if error is not None:
if 'web2py_error' in self.headers:
raise RuntimeError(self.headers['web2py_error'])
else:
raise error
self._parse_headers_in_cookies()
# check is a new session id has been issued, symptom of broken session
if self.session_regex is not None:
for cookie, value in iteritems(self.cookies):
match = self.session_regex.match(cookie)
if match:
name = match.group('name')
if name in self.sessions and self.sessions[name] != value:
print(RuntimeError('Changed session ID %s' % name))
self.sessions[name] = value
# find all forms and formkeys in page
if charset:
self.forms = {}
for match in FORM_REGEX.finditer(self.text):
self.forms[match.group('formname')] = match.group('formkey')
# log this request
self.history.append((self.method, self.url, self.status, self.time))
def post(self, url, data=None, cookies=None,
headers=None, auth=None, method='auto'):
self.url = self.app + url
# if this POST form requires a postback do it
if data and '_formname' in data and self.postbacks and \
self.history and self.history[-1][1] != self.url:
# to bypass the web2py CSRF need to get formkey
# before submitting the form
self.get(url, cookies=cookies, headers=headers, auth=auth)
# unless cookies are specified, recycle cookies
if cookies is None:
cookies = self.cookies
cookies = cookies or {}
headers = headers or {}
args = [
urllib2.HTTPCookieProcessor(self.cookiejar),
urllib2.HTTPHandler(debuglevel=0)
]
# if required do basic auth
if auth:
auth_handler = urllib2.HTTPBasicAuthHandler()
auth_handler.add_password(**auth)
args.append(auth_handler)
opener = urllib2.build_opener(*args)
# copy headers from dict to list of key,value
headers_list = []
for key, value in iteritems(self.default_headers):
if not key in headers:
headers[key] = value
for key, value in iteritems(headers):
if isinstance(value, (list, tuple)):
for v in value:
headers_list.append((key, v))
else:
headers_list.append((key, value))
# move cookies to headers
for key, value in iteritems(cookies):
headers_list.append(('Cookie', '%s=%s' % (key, value)))
# add headers to request
for key, value in headers_list:
opener.addheaders.append((key, str(value)))
# assume everything is ok and make http request
error = None
try:
if isinstance(data, str):
self.method = 'POST' if method=='auto' else method
elif isinstance(data, dict):
self.method = 'POST' if method=='auto' else method
# if there is only one form, set _formname automatically
if not '_formname' in data and len(self.forms) == 1:
data['_formname'] = self.forms.keys()[0]
# if there is no formkey but it is known, set it
if '_formname' in data and not '_formkey' in data and \
data['_formname'] in self.forms:
data['_formkey'] = self.forms[data['_formname']]
# time the POST request
data = urlencode(data, doseq=True)
else:
self.method = 'GET' if method=='auto' else method
data = None
t0 = time.time()
self.response = opener.open(self.url, to_bytes(data))
self.time = time.time() - t0
except urllib2.HTTPError as er:
error = er
# catch HTTP errors
self.time = time.time() - t0
self.response = er
if hasattr(self.response, 'getcode'):
self.status = self.response.getcode()
else:#python2.5
self.status = None
self.text = to_native(self.response.read())
# In PY3 self.response.headers are case sensitive
self.headers = dict()
for h in self.response.headers:
self.headers[h.lower()] = self.response.headers[h]
# treat web2py tickets as special types of errors
if error is not None:
if 'web2py_error' in self.headers:
raise RuntimeError(self.headers['web2py_error'])
else:
raise error
self._parse_headers_in_cookies()
# check is a new session id has been issued, symptom of broken session
if self.session_regex is not None:
for cookie, value in iteritems(self.cookies):
match = self.session_regex.match(cookie)
if match:
name = match.group('name')
if name in self.sessions and self.sessions[name] != value:
print(RuntimeError('Changed session ID %s' % name))
self.sessions[name] = value
# find all forms and formkeys in page
self.forms = {}
for match in FORM_REGEX.finditer(to_native(self.text)):
self.forms[match.group('formname')] = match.group('formkey')
# log this request
self.history.append((self.method, self.url, self.status, self.time))
def ccache():
if is_gae:
form = FORM(
P(TAG.BUTTON(T("Clear CACHE?"), _type="submit", _name="yes", _value="yes")))
else:
cache.ram.initialize()
cache.disk.initialize()
form = FORM(
P(TAG.BUTTON(
T("Clear CACHE?"), _type="submit", _name="yes", _value="yes")),
P(TAG.BUTTON(
T("Clear RAM"), _type="submit", _name="ram", _value="ram")),
P(TAG.BUTTON(
T("Clear DISK"), _type="submit", _name="disk", _value="disk")),
)
if form.accepts(request.vars, session):
session.flash = ""
if is_gae:
if request.vars.yes:
cache.ram.clear()
session.flash += T("Cache Cleared")
else:
clear_ram = False
clear_disk = False
if request.vars.yes:
clear_ram = clear_disk = True
if request.vars.ram:
clear_ram = True
if request.vars.disk:
clear_disk = True
if clear_ram:
cache.ram.clear()
session.flash += T("Ram Cleared")
if clear_disk:
cache.disk.clear()
session.flash += T("Disk Cleared")
redirect(URL(r=request))
try:
from pympler.asizeof import asizeof
except ImportError:
asizeof = False
import shelve
import os
import copy
import time
import math
from pydal.contrib import portalocker
ram = {
'entries': 0,
'bytes': 0,
'objects': 0,
'hits': 0,
'misses': 0,
'ratio': 0,
'oldest': time.time(),
'keys': []
}
disk = copy.copy(ram)
total = copy.copy(ram)
disk['keys'] = []
total['keys'] = []
def GetInHMS(seconds):
hours = math.floor(seconds / 3600)
seconds -= hours * 3600
minutes = math.floor(seconds / 60)
seconds -= minutes * 60
seconds = math.floor(seconds)
return (hours, minutes, seconds)
if is_gae:
gae_stats = cache.ram.client.get_stats()
try:
gae_stats['ratio'] = ((gae_stats['hits'] * 100) /
(gae_stats['hits'] + gae_stats['misses']))
except ZeroDivisionError:
gae_stats['ratio'] = T("?")
gae_stats['oldest'] = GetInHMS(time.time() - gae_stats['oldest_item_age'])
total.update(gae_stats)
else:
# get ram stats directly from the cache object
ram_stats = cache.ram.stats[request.application]
ram['hits'] = ram_stats['hit_total'] - ram_stats['misses']
ram['misses'] = ram_stats['misses']
try:
ram['ratio'] = ram['hits'] * 100 / ram_stats['hit_total']
except (KeyError, ZeroDivisionError):
ram['ratio'] = 0
for key, value in iteritems(cache.ram.storage):
if asizeof:
ram['bytes'] += asizeof(value[1])
ram['objects'] += 1
ram['entries'] += 1
if value[0] < ram['oldest']:
ram['oldest'] = value[0]
ram['keys'].append((key, GetInHMS(time.time() - value[0])))
for key in cache.disk.storage:
value = cache.disk.storage[key]
if key == 'web2py_cache_statistics' and isinstance(value[1], dict):
disk['hits'] = value[1]['hit_total'] - value[1]['misses']
disk['misses'] = value[1]['misses']
try:
disk['ratio'] = disk['hits'] * 100 / value[1]['hit_total']
except (KeyError, ZeroDivisionError):
disk['ratio'] = 0
else:
if asizeof:
disk['bytes'] += asizeof(value[1])
disk['objects'] += 1
disk['entries'] += 1
if value[0] < disk['oldest']:
disk['oldest'] = value[0]
disk['keys'].append((key, GetInHMS(time.time() - value[0])))
ram_keys = list(ram) # ['hits', 'objects', 'ratio', 'entries', 'keys', 'oldest', 'bytes', 'misses']
ram_keys.remove('ratio')
ram_keys.remove('oldest')
for key in ram_keys:
total[key] = ram[key] + disk[key]
try:
total['ratio'] = total['hits'] * 100 / (total['hits'] +
total['misses'])
except (KeyError, ZeroDivisionError):
total['ratio'] = 0
if disk['oldest'] < ram['oldest']:
total['oldest'] = disk['oldest']
else:
total['oldest'] = ram['oldest']
ram['oldest'] = GetInHMS(time.time() - ram['oldest'])
disk['oldest'] = GetInHMS(time.time() - disk['oldest'])
total['oldest'] = GetInHMS(time.time() - total['oldest'])
def key_table(keys):
return TABLE(
TR(TD(B(T('Key'))), TD(B(T('Time in Cache (h:m:s)')))),
*[TR(TD(k[0]), TD('%02d:%02d:%02d' % k[1])) for k in keys],
**dict(_class='cache-keys',
_style="border-collapse: separate; border-spacing: .5em;"))
if not is_gae:
ram['keys'] = key_table(ram['keys'])
disk['keys'] = key_table(disk['keys'])
total['keys'] = key_table(total['keys'])
return dict(form=form, total=total,
ram=ram, disk=disk, object_stats=asizeof != False)
def ccache():
if is_gae:
form = FORM(
P(
TAG.BUTTON(T("Clear CACHE?"),
_type="submit",
_name="yes",
_value="yes")))
else:
cache.ram.initialize()
cache.disk.initialize()
form = FORM(
P(
TAG.BUTTON(T("Clear CACHE?"),
_type="submit",
_name="yes",
_value="yes")),
P(
TAG.BUTTON(T("Clear RAM"),
_type="submit",
_name="ram",
_value="ram")),
P(
TAG.BUTTON(T("Clear DISK"),
_type="submit",
_name="disk",
_value="disk")),
)
if form.accepts(request.vars, session):
session.flash = ""
if is_gae:
if request.vars.yes:
cache.ram.clear()
session.flash += T("Cache Cleared")
else:
clear_ram = False
clear_disk = False
if request.vars.yes:
clear_ram = clear_disk = True
if request.vars.ram:
clear_ram = True
if request.vars.disk:
clear_disk = True
if clear_ram:
cache.ram.clear()
session.flash += T("Ram Cleared")
if clear_disk:
cache.disk.clear()
session.flash += T("Disk Cleared")
redirect(URL(r=request))
try:
from pympler.asizeof import asizeof
except ImportError:
asizeof = False
import shelve
import os
import copy
import time
import math
from pydal.contrib import portalocker
ram = {
'entries': 0,
'bytes': 0,
'objects': 0,
'hits': 0,
'misses': 0,
'ratio': 0,
'oldest': time.time(),
'keys': []
}
disk = copy.copy(ram)
total = copy.copy(ram)
disk['keys'] = []
total['keys'] = []
def GetInHMS(seconds):
hours = math.floor(seconds / 3600)
seconds -= hours * 3600
minutes = math.floor(seconds / 60)
seconds -= minutes * 60
seconds = math.floor(seconds)
return (hours, minutes, seconds)
if is_gae:
gae_stats = cache.ram.client.get_stats()
try:
gae_stats['ratio'] = ((gae_stats['hits'] * 100) /
(gae_stats['hits'] + gae_stats['misses']))
except ZeroDivisionError:
gae_stats['ratio'] = T("?")
gae_stats['oldest'] = GetInHMS(time.time() -
gae_stats['oldest_item_age'])
total.update(gae_stats)
else:
# get ram stats directly from the cache object
ram_stats = cache.ram.stats[request.application]
ram['hits'] = ram_stats['hit_total'] - ram_stats['misses']
ram['misses'] = ram_stats['misses']
try:
ram['ratio'] = ram['hits'] * 100 / ram_stats['hit_total']
except (KeyError, ZeroDivisionError):
ram['ratio'] = 0
for key, value in iteritems(cache.ram.storage):
if asizeof:
ram['bytes'] += asizeof(value[1])
ram['objects'] += 1
ram['entries'] += 1
if value[0] < ram['oldest']:
ram['oldest'] = value[0]
ram['keys'].append((key, GetInHMS(time.time() - value[0])))
for key in cache.disk.storage:
value = cache.disk.storage[key]
if key == 'web2py_cache_statistics' and isinstance(value[1], dict):
disk['hits'] = value[1]['hit_total'] - value[1]['misses']
disk['misses'] = value[1]['misses']
try:
disk['ratio'] = disk['hits'] * 100 / value[1]['hit_total']
except (KeyError, ZeroDivisionError):
disk['ratio'] = 0
else:
if asizeof:
disk['bytes'] += asizeof(value[1])
disk['objects'] += 1
disk['entries'] += 1
if value[0] < disk['oldest']:
disk['oldest'] = value[0]
disk['keys'].append((key, GetInHMS(time.time() - value[0])))
ram_keys = list(
ram
) # ['hits', 'objects', 'ratio', 'entries', 'keys', 'oldest', 'bytes', 'misses']
ram_keys.remove('ratio')
ram_keys.remove('oldest')
for key in ram_keys:
total[key] = ram[key] + disk[key]
try:
total['ratio'] = total['hits'] * 100 / (total['hits'] +
total['misses'])
except (KeyError, ZeroDivisionError):
total['ratio'] = 0
if disk['oldest'] < ram['oldest']:
total['oldest'] = disk['oldest']
else:
total['oldest'] = ram['oldest']
ram['oldest'] = GetInHMS(time.time() - ram['oldest'])
disk['oldest'] = GetInHMS(time.time() - disk['oldest'])
total['oldest'] = GetInHMS(time.time() - total['oldest'])
def key_table(keys):
return TABLE(
TR(TD(B(T('Key'))), TD(B(T('Time in Cache (h:m:s)')))),
*[TR(TD(k[0]), TD('%02d:%02d:%02d' % k[1])) for k in keys],
**dict(_class='cache-keys',
_style="border-collapse: separate; border-spacing: .5em;"))
if not is_gae:
ram['keys'] = key_table(ram['keys'])
disk['keys'] = key_table(disk['keys'])
total['keys'] = key_table(total['keys'])
return dict(form=form,
total=total,
ram=ram,
disk=disk,
object_stats=asizeof != False)
def ccache():
if is_gae:
form = FORM(P(TAG.BUTTON(T("Clear CACHE?"), _type="submit", _name="yes", _value="yes")))
else:
cache.ram.initialize()
cache.disk.initialize()
form = FORM(
P(TAG.BUTTON(T("Clear CACHE?"), _type="submit", _name="yes", _value="yes")),
P(TAG.BUTTON(T("Clear RAM"), _type="submit", _name="ram", _value="ram")),
P(TAG.BUTTON(T("Clear DISK"), _type="submit", _name="disk", _value="disk")),
)
if form.accepts(request.vars, session):
session.flash = ""
if is_gae:
if request.vars.yes:
cache.ram.clear()
session.flash += T("Cache Cleared")
else:
clear_ram = False
clear_disk = False
if request.vars.yes:
clear_ram = clear_disk = True
if request.vars.ram:
clear_ram = True
if request.vars.disk:
clear_disk = True
if clear_ram:
cache.ram.clear()
session.flash += T("Ram Cleared")
if clear_disk:
cache.disk.clear()
session.flash += T("Disk Cleared")
redirect(URL(r=request))
try:
from guppy import hpy
hp = hpy()
except ImportError:
hp = False
import shelve
import os
import copy
import time
import math
from gluon import portalocker
ram = {
"entries": 0,
"bytes": 0,
"objects": 0,
"hits": 0,
"misses": 0,
"ratio": 0,
"oldest": time.time(),
"keys": [],
}
disk = copy.copy(ram)
total = copy.copy(ram)
disk["keys"] = []
total["keys"] = []
def GetInHMS(seconds):
hours = math.floor(seconds / 3600)
seconds -= hours * 3600
minutes = math.floor(seconds / 60)
seconds -= minutes * 60
seconds = math.floor(seconds)
return (hours, minutes, seconds)
if is_gae:
gae_stats = cache.ram.client.get_stats()
try:
gae_stats["ratio"] = (gae_stats["hits"] * 100) / (gae_stats["hits"] + gae_stats["misses"])
except ZeroDivisionError:
gae_stats["ratio"] = T("?")
gae_stats["oldest"] = GetInHMS(time.time() - gae_stats["oldest_item_age"])
total.update(gae_stats)
else:
# get ram stats directly from the cache object
ram_stats = cache.ram.stats[request.application]
ram["hits"] = ram_stats["hit_total"] - ram_stats["misses"]
ram["misses"] = ram_stats["misses"]
try:
ram["ratio"] = ram["hits"] * 100 / ram_stats["hit_total"]
except (KeyError, ZeroDivisionError):
ram["ratio"] = 0
for key, value in iteritems(cache.ram.storage):
if hp:
ram["bytes"] += hp.iso(value[1]).size
ram["objects"] += hp.iso(value[1]).count
ram["entries"] += 1
if value[0] < ram["oldest"]:
ram["oldest"] = value[0]
ram["keys"].append((key, GetInHMS(time.time() - value[0])))
for key in cache.disk.storage:
value = cache.disk.storage[key]
if isinstance(value[1], dict):
disk["hits"] = value[1]["hit_total"] - value[1]["misses"]
disk["misses"] = value[1]["misses"]
try:
disk["ratio"] = disk["hits"] * 100 / value[1]["hit_total"]
except (KeyError, ZeroDivisionError):
disk["ratio"] = 0
else:
if hp:
disk["bytes"] += hp.iso(value[1]).size
disk["objects"] += hp.iso(value[1]).count
disk["entries"] += 1
if value[0] < disk["oldest"]:
disk["oldest"] = value[0]
disk["keys"].append((key, GetInHMS(time.time() - value[0])))
ram_keys = ram.keys() # ['hits', 'objects', 'ratio', 'entries', 'keys', 'oldest', 'bytes', 'misses']
ram_keys.remove("ratio")
ram_keys.remove("oldest")
for key in ram_keys:
total[key] = ram[key] + disk[key]
try:
total["ratio"] = total["hits"] * 100 / (total["hits"] + total["misses"])
except (KeyError, ZeroDivisionError):
total["ratio"] = 0
if disk["oldest"] < ram["oldest"]:
total["oldest"] = disk["oldest"]
else:
total["oldest"] = ram["oldest"]
ram["oldest"] = GetInHMS(time.time() - ram["oldest"])
disk["oldest"] = GetInHMS(time.time() - disk["oldest"])
total["oldest"] = GetInHMS(time.time() - total["oldest"])
def key_table(keys):
return TABLE(
TR(TD(B(T("Key"))), TD(B(T("Time in Cache (h:m:s)")))),
*[TR(TD(k[0]), TD("%02d:%02d:%02d" % k[1])) for k in keys],
**dict(_class="cache-keys", _style="border-collapse: separate; border-spacing: .5em;")
)
if not is_gae:
ram["keys"] = key_table(ram["keys"])
disk["keys"] = key_table(disk["keys"])
total["keys"] = key_table(total["keys"])
return dict(form=form, total=total, ram=ram, disk=disk, object_stats=hp != False)
def env(
a,
import_models=False,
c=None,
f=None,
dir='',
extra_request={},
):
"""
Returns web2py execution environment for application (a), controller (c),
function (f).
If import_models is True the exec all application models into the
environment.
extra_request allows you to pass along any extra variables to the request
object before your models get executed. This was mainly done to support
web2py_utils.test_runner, however you can use it with any wrapper scripts
that need access to the web2py environment.
"""
request = Request({})
response = Response()
session = Session()
request.application = a
# Populate the dummy environment with sensible defaults.
if not dir:
request.folder = os.path.join('applications', a)
else:
request.folder = dir
request.controller = c or 'default'
request.function = f or 'index'
response.view = '%s/%s.html' % (request.controller, request.function)
if global_settings.cmd_options:
ip = global_settings.cmd_options.ip
port = global_settings.cmd_options.port
request.is_shell = global_settings.cmd_options.shell is not None
request.is_scheduler = global_settings.cmd_options.scheduler is not None
else:
ip, port = '127.0.0.1', '8000'
request.env.http_host = '%s:%s' % (ip, port)
request.env.remote_addr = '127.0.0.1'
request.env.web2py_runtime_gae = global_settings.web2py_runtime_gae
for k, v in extra_request.items():
request[k] = v
path_info = '/%s/%s/%s' % (a, c, f)
if request.args:
path_info = '%s/%s' % (path_info, '/'.join(request.args))
if request.vars:
vars = [
'%s=%s' % (k, v) if v else '%s' % k
for (k, v) in iteritems(request.vars)
]
path_info = '%s?%s' % (path_info, '&'.join(vars))
request.env.path_info = path_info
# Monkey patch so credentials checks pass.
def check_credentials(request, other_application='admin'):
return True
fileutils.check_credentials = check_credentials
environment = build_environment(request, response, session)
if import_models:
try:
run_models_in(environment)
except RestrictedError as e:
sys.stderr.write(e.traceback + '\n')
sys.exit(1)
response._view_environment = copy.copy(environment)
environment['__name__'] = '__main__'
return environment
def env(
a,
import_models=False,
c=None,
f=None,
dir='',
extra_request={},
):
"""
Returns web2py execution environment for application (a), controller (c),
function (f).
If import_models is True the exec all application models into the
environment.
extra_request allows you to pass along any extra variables to the request
object before your models get executed. This was mainly done to support
web2py_utils.test_runner, however you can use it with any wrapper scripts
that need access to the web2py environment.
"""
request = Request({})
response = Response()
session = Session()
request.application = a
# Populate the dummy environment with sensible defaults.
if not dir:
request.folder = os.path.join('applications', a)
else:
request.folder = dir
request.controller = c or 'default'
request.function = f or 'index'
response.view = '%s/%s.html' % (request.controller,
request.function)
if global_settings.cmd_options:
ip = global_settings.cmd_options.ip
port = global_settings.cmd_options.port
request.is_shell = global_settings.cmd_options.shell is not None
request.is_scheduler = global_settings.cmd_options.scheduler is not None
else:
ip, port = '127.0.0.1', '8000'
request.env.http_host = '%s:%s' % (ip, port)
request.env.remote_addr = '127.0.0.1'
request.env.web2py_runtime_gae = global_settings.web2py_runtime_gae
for k, v in extra_request.items():
request[k] = v
path_info = '/%s/%s/%s' % (a, c, f)
if request.args:
path_info = '%s/%s' % (path_info, '/'.join(request.args))
if request.vars:
vars = ['%s=%s' % (k, v) if v else '%s' % k
for (k, v) in iteritems(request.vars)]
path_info = '%s?%s' % (path_info, '&'.join(vars))
request.env.path_info = path_info
# Monkey patch so credentials checks pass.
def check_credentials(request, other_application='admin'):
return True
fileutils.check_credentials = check_credentials
environment = build_environment(request, response, session)
if import_models:
try:
run_models_in(environment)
except RestrictedError as e:
sys.stderr.write(e.traceback + '\n')
sys.exit(1)
environment['__name__'] = '__main__'
return environment
