def _try_store_in_file(self, request, response): try: if (not response.session_id or self._forget or self._unchanged(response)): # self.clear_session_cookies() self.save_session_id_cookie() return False if response.session_new or not response.session_file: # Tests if the session sub-folder exists, if not, create it session_folder = os.path.dirname(response.session_filename) if not os.path.exists(session_folder): os.mkdir(session_folder) response.session_file = recfile.open(response.session_filename, 'wb') portalocker.lock(response.session_file, portalocker.LOCK_EX) response.session_locked = True if response.session_file: session_pickled = response.session_pickled or pickle.dumps( self, pickle.HIGHEST_PROTOCOL) response.session_file.write(session_pickled) response.session_file.truncate() finally: self._close(response) self.save_session_id_cookie() return True
def _try_store_in_file(self, request, response): try: if (not response.session_id or not response.session_filename or self._forget or self._unchanged(response)): # self.clear_session_cookies() return False else: if response.session_new or not response.session_file: # Tests if the session sub-folder exists, if not, create it session_folder = os.path.dirname(response.session_filename) if not os.path.exists(session_folder): os.mkdir(session_folder) response.session_file = recfile.open(response.session_filename, 'wb') portalocker.lock(response.session_file, portalocker.LOCK_EX) response.session_locked = True if response.session_file: session_pickled = response.session_pickled or pickle.dumps(self, pickle.HIGHEST_PROTOCOL) response.session_file.write(session_pickled) response.session_file.truncate() return True finally: self._close(response) self.save_session_id_cookie()
def write_hosts_deny(denied_hosts): f = open(deny_file, 'w') portalocker.lock(f, portalocker.LOCK_EX) for key, val in denied_hosts.items(): if time.time() - val[1] < expiration_failed_logins: line = '%s %s %s\n' % (key, val[0], val[1]) f.write(line) portalocker.unlock(f) f.close()
def write_hosts_deny(denied_hosts): f = open(deny_file, 'w') portalocker.lock(f, portalocker.LOCK_EX) for key, val in denied_hosts.items(): if time.time()-val[1] < expiration_failed_logins: line = '%s %s %s\n' % (key, val[0], val[1]) f.write(line) portalocker.unlock(f) f.close()
def wait_portalock(self, val_file): """ Wait for the process file lock. """ while True: try: portalocker.lock(val_file, portalocker.LOCK_EX) break except: time.sleep(self.file_lock_time_wait)
def release(self): """ this function writes into cron.master the time when cron job was completed """ if not self.master.closed: portalocker.lock(self.master, portalocker.LOCK_EX) logger.debug('WEB2PY CRON: Releasing cron lock') self.master.seek(0) (start, stop) = cPickle.load(self.master) if start == self.now: # if this is my lock self.master.seek(0) cPickle.dump((self.now, time.time()), self.master) portalocker.unlock(self.master) self.master.close()
def release(self): """ this function writes into cron.msater the time when cron job was completed """ if not self.master.closed: portalocker.lock(self.master,portalocker.LOCK_EX) logging.debug('WEB2PY CRON: Releasing cron lock') self.master.seek(0) (start, stop) = cPickle.load(self.master) if start == self.now: # if this is my lock self.master.seek(0) cPickle.dump((self.now,time.time()),self.master) portalocker.unlock(self.master) self.master.close()
def read_hosts_deny(): import datetime hosts = {} if os.path.exists(deny_file): hosts = {} f = open(deny_file, 'r') portalocker.lock(f, portalocker.LOCK_SH) for line in f.readlines(): if not line.strip() or line.startswith('#'): continue fields = line.strip().split() if len(fields) > 2: hosts[fields[0].strip()] = ( # ip int(fields[1].strip()), # n attemps int(fields[2].strip()) # last attempts ) portalocker.unlock(f) f.close() return hosts
def open_read(self): f_name=self.f_name try: if os.path.isfile(f_name): # no file.json - create it md='r+' else: md='w' f = open(f_name, md) self.f_obj = f portalocker.lock(f, portalocker.LOCK_EX ) if md=='r+': tmp= f.read() self.log = (tmp and js.loads(tmp)) or {} else: self.log = {} except : self.error['open_read'] = sys.exc_info()[0] return self.log
def acquire(self, startup=False): """ returns the time when the lock is acquired or None if cron already running lock is implemented by writing a pickle (start, stop) in cron.master start is time when cron job starts and stop is time when cron completed stop == 0 if job started but did not yet complete if a cron job started within less than 60 seconds, acquire returns None if a cron job started before 60 seconds and did not stop, a warning is issue "Stale cron.master detected" """ if portalocker.LOCK_EX == None: logger.warning('WEB2PY CRON: Disabled because no file locking') return None self.master = open(self.path, 'rb+') try: ret = None portalocker.lock(self.master, portalocker.LOCK_EX) try: (start, stop) = cPickle.load(self.master) except: (start, stop) = (0, 1) if startup or self.now - start > 59.99: ret = self.now if not stop: # this happens if previous cron job longer than 1 minute logger.warning('WEB2PY CRON: Stale cron.master detected') logger.debug('WEB2PY CRON: Acquiring lock') self.master.seek(0) cPickle.dump((self.now, 0), self.master) finally: portalocker.unlock(self.master) if not ret: # do this so no need to release self.master.close() return ret
def acquire(self,startup=False): """ returns the time when the lock is acquired or None if cron already runing lock is implemnted by writing a pickle (start, stop) in cron.master start is time when cron job starts and stop is time when cron completed stop == 0 if job started but did not yet complete if a cron job started within less than 60 secods, acquire returns None if a cron job started before 60 seconds and did not stop, a warning is issue "Stale cron.master detected" """ if portalocker.LOCK_EX == None: logging.warning('WEB2PY CRON: Disabled because no file locking') return None self.master = open(self.path,'rb+') try: ret = None portalocker.lock(self.master,portalocker.LOCK_EX) try: (start, stop) = cPickle.load(self.master) except: (start, stop) = (0, 1) if startup or self.now - start >= 60: ret = self.now if not stop: # this happens if previous cron job longer than 1 minute logging.warning('WEB2PY CRON: Stale cron.master detected') logging.debug('WEB2PY CRON: Acquiring lock') self.master.seek(0) cPickle.dump((self.now,0),self.master) finally: portalocker.unlock(self.master) if not ret: # do this so no need to release self.master.close() return ret
def connect(self, request=None, response=None, db=None, tablename='web2py_session', masterapp=None, migrate=True, separate=None, check_client=False, cookie_key=None, cookie_expires=None, compression_level=None ): """ Used in models, allows to customize Session handling Args: request: the request object response: the response object db: to store/retrieve sessions in db (a table is created) tablename(str): table name masterapp(str): points to another's app sessions. This enables a "SSO" environment among apps migrate: passed to the underlying db separate: with True, creates a folder with the 2 initials of the session id. Can also be a function, e.g. :: separate=lambda(session_name): session_name[-2:] check_client: if True, sessions can only come from the same ip cookie_key(str): secret for cookie encryption cookie_expires: sets the expiration of the cookie compression_level(int): 0-9, sets zlib compression on the data before the encryption """ from gluon.dal import Field request = request or current.request response = response or current.response masterapp = masterapp or request.application cookies = request.cookies self._unlock(response) response.session_masterapp = masterapp response.session_id_name = 'session_id_%s' % masterapp.lower() response.session_data_name = 'session_data_%s' % masterapp.lower() response.session_cookie_expires = cookie_expires response.session_client = str(request.client).replace(':', '.') current._session_cookie_key = cookie_key response.session_cookie_compression_level = compression_level # check if there is a session_id in cookies try: old_session_id = cookies[response.session_id_name].value except KeyError: old_session_id = None response.session_id = old_session_id # if we are supposed to use cookie based session data if cookie_key: response.session_storage_type = 'cookie' elif db: response.session_storage_type = 'db' else: response.session_storage_type = 'file' # why do we do this? # because connect may be called twice, by web2py and in models. # the first time there is no db yet so it should do nothing if (global_settings.db_sessions is True or masterapp in global_settings.db_sessions): return if response.session_storage_type == 'cookie': # check if there is session data in cookies if response.session_data_name in cookies: session_cookie_data = cookies[response.session_data_name].value else: session_cookie_data = None if session_cookie_data: data = secure_loads(session_cookie_data, cookie_key, compression_level=compression_level) if data: self.update(data) response.session_id = True # else if we are supposed to use file based sessions elif response.session_storage_type == 'file': response.session_new = False response.session_file = None # check if the session_id points to a valid sesion filename if response.session_id: if not regex_session_id.match(response.session_id): response.session_id = None else: response.session_filename = \ os.path.join(up(request.folder), masterapp, 'sessions', response.session_id) try: response.session_file = \ recfile.open(response.session_filename, 'rb+') portalocker.lock(response.session_file, portalocker.LOCK_EX) response.session_locked = True self.update(pickle.load(response.session_file)) response.session_file.seek(0) oc = response.session_filename.split('/')[-1].split('-')[0] if check_client and response.session_client != oc: raise Exception("cookie attack") except: response.session_id = None if not response.session_id: uuid = web2py_uuid() response.session_id = '%s-%s' % (response.session_client, uuid) separate = separate and (lambda session_name: session_name[-2:]) if separate: prefix = separate(response.session_id) response.session_id = '%s/%s' % (prefix, response.session_id) response.session_filename = \ os.path.join(up(request.folder), masterapp, 'sessions', response.session_id) response.session_new = True # else the session goes in db elif response.session_storage_type == 'db': if global_settings.db_sessions is not True: global_settings.db_sessions.add(masterapp) # if had a session on file alreday, close it (yes, can happen) if response.session_file: self._close(response) # if on GAE tickets go also in DB if settings.global_settings.web2py_runtime_gae: request.tickets_db = db if masterapp == request.application: table_migrate = migrate else: table_migrate = False tname = tablename + '_' + masterapp table = db.get(tname, None) # Field = db.Field if table is None: db.define_table( tname, Field('locked', 'boolean', default=False), Field('client_ip', length=64), Field('created_datetime', 'datetime', default=request.now), Field('modified_datetime', 'datetime'), Field('unique_key', length=64), Field('session_data', 'blob'), migrate=table_migrate, ) table = db[tname] # to allow for lazy table response.session_db_table = table if response.session_id: # Get session data out of the database try: (record_id, unique_key) = response.session_id.split(':') record_id = long(record_id) except (TypeError, ValueError): record_id = None # Select from database if record_id: row = table(record_id, unique_key=unique_key) # Make sure the session data exists in the database if row: # rows[0].update_record(locked=True) # Unpickle the data session_data = pickle.loads(row.session_data) self.update(session_data) response.session_new = False else: record_id = None if record_id: response.session_id = '%s:%s' % (record_id, unique_key) response.session_db_unique_key = unique_key response.session_db_record_id = record_id else: response.session_id = None response.session_new = True # if there is no session id yet, we'll need to create a # new session else: response.session_new = True # set the cookie now if you know the session_id so user can set # cookie attributes in controllers/models # cookie will be reset later # yet cookie may be reset later # Removed comparison between old and new session ids - should send # the cookie all the time if isinstance(response.session_id, str): response.cookies[response.session_id_name] = response.session_id response.cookies[response.session_id_name]['path'] = '/' if cookie_expires: response.cookies[response.session_id_name]['expires'] = \ cookie_expires.strftime(FMT) session_pickled = pickle.dumps(self, pickle.HIGHEST_PROTOCOL) response.session_hash = hashlib.md5(session_pickled).hexdigest() if self.flash: (response.flash, self.flash) = (self.flash, None)
def ccache(): form = FORM( P(TAG.BUTTON("Clear CACHE?", _type="submit", _name="yes", _value="yes")), P(TAG.BUTTON("Clear RAM", _type="submit", _name="ram", _value="ram")), P(TAG.BUTTON("Clear DISK", _type="submit", _name="disk", _value="disk")), ) if form.accepts(request.vars, session): clear_ram = False clear_disk = False session.flash = "" if request.vars.yes: clear_ram = clear_disk = True if request.vars.ram: clear_ram = True if request.vars.disk: clear_disk = True if clear_ram: cache.ram.clear() session.flash += "Ram Cleared " if clear_disk: cache.disk.clear() session.flash += "Disk Cleared" redirect(URL(r=request)) try: from guppy import hpy hp = hpy() except ImportError: hp = False import shelve import os import copy import time import math from gluon import portalocker ram = { 'entries': 0, 'bytes': 0, 'objects': 0, 'hits': 0, 'misses': 0, 'ratio': 0, 'oldest': time.time(), 'keys': [] } disk = copy.copy(ram) total = copy.copy(ram) disk['keys'] = [] total['keys'] = [] def GetInHMS(seconds): hours = math.floor(seconds / 3600) seconds -= hours * 3600 minutes = math.floor(seconds / 60) seconds -= minutes * 60 seconds = math.floor(seconds) return (hours, minutes, seconds) for key, value in cache.ram.storage.items(): if isinstance(value, dict): ram['hits'] = value['hit_total'] - value['misses'] ram['misses'] = value['misses'] try: ram['ratio'] = ram['hits'] * 100 / value['hit_total'] except (KeyError, ZeroDivisionError): ram['ratio'] = 0 else: if hp: ram['bytes'] += hp.iso(value[1]).size ram['objects'] += hp.iso(value[1]).count ram['entries'] += 1 if value[0] < ram['oldest']: ram['oldest'] = value[0] ram['keys'].append((key, GetInHMS(time.time() - value[0]))) locker = open(os.path.join(request.folder, 'cache/cache.lock'), 'a') portalocker.lock(locker, portalocker.LOCK_EX) disk_storage = shelve.open(os.path.join(request.folder, 'cache/cache.shelve')) try: for key, value in disk_storage.items(): if isinstance(value, dict): disk['hits'] = value['hit_total'] - value['misses'] disk['misses'] = value['misses'] try: disk['ratio'] = disk['hits'] * 100 / value['hit_total'] except (KeyError, ZeroDivisionError): disk['ratio'] = 0 else: if hp: disk['bytes'] += hp.iso(value[1]).size disk['objects'] += hp.iso(value[1]).count disk['entries'] += 1 if value[0] < disk['oldest']: disk['oldest'] = value[0] disk['keys'].append((key, GetInHMS(time.time() - value[0]))) finally: portalocker.unlock(locker) locker.close() disk_storage.close() total['entries'] = ram['entries'] + disk['entries'] total['bytes'] = ram['bytes'] + disk['bytes'] total['objects'] = ram['objects'] + disk['objects'] total['hits'] = ram['hits'] + disk['hits'] total['misses'] = ram['misses'] + disk['misses'] total['keys'] = ram['keys'] + disk['keys'] try: total['ratio'] = total['hits'] * 100 / (total['hits'] + total['misses']) except (KeyError, ZeroDivisionError): total['ratio'] = 0 if disk['oldest'] < ram['oldest']: total['oldest'] = disk['oldest'] else: total['oldest'] = ram['oldest'] ram['oldest'] = GetInHMS(time.time() - ram['oldest']) disk['oldest'] = GetInHMS(time.time() - disk['oldest']) total['oldest'] = GetInHMS(time.time() - total['oldest']) def key_table(keys): return TABLE( TR(TD(B('Key')), TD(B('Time in Cache (h:m:s)'))), *[TR(TD(k[0]), TD('%02d:%02d:%02d' % k[1])) for k in keys], **dict(_class='cache-keys', _style="border-collapse: separate; border-spacing: .5em;")) ram['keys'] = key_table(ram['keys']) disk['keys'] = key_table(disk['keys']) total['keys'] = key_table(total['keys']) return dict(form=form, total=total, ram=ram, disk=disk, object_stats=hp != False)
print >> sys.stderr, msg if errors: # Report errors and stop. actionrequired = T("ACTION REQUIRED") prefix = "\n%s: " % actionrequired msg = prefix + prefix.join(errors) print >> sys.stderr, msg htmlprefix = "\n<br /><b>%s</b>: " % actionrequired html = "<errors>" + htmlprefix + htmlprefix.join( errors) + "\n</errors>" raise HTTP(500, body=html) # Create or update the canary file. from gluon import portalocker canary = open("applications/%s/models/0000_update_check.py" % appname, "w") portalocker.lock(canary, portalocker.LOCK_EX) statement = "CANARY_UPDATE_CHECK_ID = %s" % CURRENT_UPDATE_CHECK_ID canary.write(statement) canary.close() # ----------------------------------------------------------------------------- import os try: # Python 2.7 from collections import OrderedDict except: # Python 2.6 from gluon.contrib.simplejson.ordered_dict import OrderedDict from gluon import current
ram['ratio'] = ram['hits'] * 100 / value['hit_total'] except (KeyError, ZeroDivisionError): ram['ratio'] = 0 else: if hp: ram['bytes'] += hp.iso(value[1]).size ram['objects'] += hp.iso(value[1]).count ram['entries'] += 1 if value[0] < ram['oldest']: ram['oldest'] = value[0] ram['keys'].append((key, GetInHMS(time.time() - value[0]))) folder = os.path.join(request.folder,'cache') if not os.path.exists(folder): os.mkdir(folder) locker = open(os.path.join(folder, 'cache.lock'), 'a') portalocker.lock(locker, portalocker.LOCK_EX) disk_storage = shelve.open( os.path.join(folder, 'cache.shelve')) try: for key, value in disk_storage.items(): ======= if is_gae: gae_stats = cache.ram.client.get_stats() try: gae_stats['ratio'] = ((gae_stats['hits'] * 100) / (gae_stats['hits'] + gae_stats['misses'])) except ZeroDivisionError: gae_stats['ratio'] = T("?") gae_stats['oldest'] = GetInHMS(time.time() - gae_stats['oldest_item_age']) total.update(gae_stats) else:
def system(): import os form = FORM( P(TAG.BUTTON("Clear Sessions?", _type="submit", _id = "sessions", _name="sessions", _value="sessions")), P(TAG.BUTTON("Clear Errors?", _type="submit", _id = "errors", _name="errors", _value="errors")), P(TAG.BUTTON("Clear RAM Cache", _type="submit", _id = "ram", _name="ram", _value="ram")), P(TAG.BUTTON("Clear Disk Cache", _type="submit", _id = "disk", _name="disk", _value="disk")), P(TAG.BUTTON("Clear All Cache?", _type="submit", _id = "cache", _name="cache", _value="cache")) ) if form.accepts(request.vars, session): if request.vars.sessions: sessions_path = os.path.join(request.folder, 'sessions') for file_path in os.listdir(sessions_path): session_path = os.path.join(request.folder, 'sessions', file_path) os.unlink(session_path) redirect(URL(a=request.application, c='default', f='index')) if request.vars.errors: errors_path = os.path.join(request.folder, 'errors') for file_path in os.listdir(errors_path): error_path = os.path.join(request.folder, 'errors', file_path) os.unlink(error_path) clear_ram = False clear_disk = False if request.vars.cache: clear_ram = clear_disk = True if request.vars.ram: clear_ram = True if request.vars.disk: clear_disk = True if clear_ram: cache.ram.clear() if clear_disk: cache.disk.clear() try: from guppy import hpy; hp=hpy() except ImportError: hp = False import shelve, os, copy, time, math from gluon import portalocker ram = { 'bytes': 0, 'objects': 0, 'hits': 0, 'misses': 0, 'ratio': 0, 'oldest': time.time() } disk = copy.copy(ram) total = copy.copy(ram) for key, value in cache.ram.storage.items(): if isinstance(value, dict): ram['hits'] = value['hit_total'] - value['misses'] ram['misses'] = value['misses'] try: ram['ratio'] = ram['hits'] * 100 / value['hit_total'] except (KeyError, ZeroDivisionError): ram['ratio'] = 0 else: if hp: ram['bytes'] += hp.iso(value[1]).size ram['objects'] += hp.iso(value[1]).count if value[0] < ram['oldest']: ram['oldest'] = value[0] locker = open(os.path.join(request.folder, 'cache/cache.lock'), 'a') portalocker.lock(locker, portalocker.LOCK_EX) disk_storage = shelve.open( os.path.join(request.folder, 'cache/cache.shelve')) for key, value in disk_storage.items(): if isinstance(value, dict): disk['hits'] = value['hit_total'] - value['misses'] disk['misses'] = value['misses'] try: disk['ratio'] = disk['hits'] * 100 / value['hit_total'] except (KeyError, ZeroDivisionError): disk['ratio'] = 0 else: if hp: disk['bytes'] += hp.iso(value[1]).size disk['objects'] += hp.iso(value[1]).count if value[0] < disk['oldest']: disk['oldest'] = value[0] portalocker.unlock(locker) locker.close() disk_storage.close() total['bytes'] = ram['bytes'] + disk['bytes'] total['objects'] = ram['objects'] + disk['objects'] total['hits'] = ram['hits'] + disk['hits'] total['misses'] = ram['misses'] + disk['misses'] try: total['ratio'] = total['hits'] * 100 / (total['hits'] + total['misses']) except (KeyError, ZeroDivisionError): total['ratio'] = 0 if disk['oldest'] < ram['oldest']: total['oldest'] = disk['oldest'] else: total['oldest'] = ram['oldest'] def GetInHMS(seconds): hours = math.floor(seconds / 3600) seconds -= hours * 3600 minutes = math.floor(seconds / 60) seconds -= minutes * 60 seconds = math.floor(seconds) return (hours, minutes, seconds) ram['oldest'] = GetInHMS(time.time() - ram['oldest']) disk['oldest'] = GetInHMS(time.time() - disk['oldest']) total['oldest'] = GetInHMS(time.time() - total['oldest']) return dict(form=form, total=total, ram=ram, disk=disk)
def connect(self, request=None, response=None, db=None, tablename='web2py_session', masterapp=None, migrate=True, separate=None, check_client=False, cookie_key=None, cookie_expires=None, compression_level=None): """ Used in models, allows to customize Session handling Args: request: the request object response: the response object db: to store/retrieve sessions in db (a table is created) tablename(str): table name masterapp(str): points to another's app sessions. This enables a "SSO" environment among apps migrate: passed to the underlying db separate: with True, creates a folder with the 2 initials of the session id. Can also be a function, e.g. :: separate=lambda(session_name): session_name[-2:] check_client: if True, sessions can only come from the same ip cookie_key(str): secret for cookie encryption cookie_expires: sets the expiration of the cookie compression_level(int): 0-9, sets zlib compression on the data before the encryption """ from gluon.dal import Field request = request or current.request response = response or current.response masterapp = masterapp or request.application cookies = request.cookies self._unlock(response) response.session_masterapp = masterapp response.session_id_name = 'session_id_%s' % masterapp.lower() response.session_data_name = 'session_data_%s' % masterapp.lower() response.session_cookie_expires = cookie_expires response.session_client = str(request.client).replace(':', '.') current._session_cookie_key = cookie_key response.session_cookie_compression_level = compression_level # check if there is a session_id in cookies try: old_session_id = cookies[response.session_id_name].value except KeyError: old_session_id = None response.session_id = old_session_id # if we are supposed to use cookie based session data if cookie_key: response.session_storage_type = 'cookie' elif db: response.session_storage_type = 'db' else: response.session_storage_type = 'file' # why do we do this? # because connect may be called twice, by web2py and in models. # the first time there is no db yet so it should do nothing if (global_settings.db_sessions is True or masterapp in global_settings.db_sessions): return if response.session_storage_type == 'cookie': # check if there is session data in cookies if response.session_data_name in cookies: session_cookie_data = cookies[response.session_data_name].value else: session_cookie_data = None if session_cookie_data: data = secure_loads(session_cookie_data, cookie_key, compression_level=compression_level) if data: self.update(data) response.session_id = True # else if we are supposed to use file based sessions elif response.session_storage_type == 'file': response.session_new = False response.session_file = None # check if the session_id points to a valid sesion filename if response.session_id: if not regex_session_id.match(response.session_id): response.session_id = None else: response.session_filename = \ os.path.join(up(request.folder), masterapp, 'sessions', response.session_id) try: response.session_file = \ recfile.open(response.session_filename, 'rb+') portalocker.lock(response.session_file, portalocker.LOCK_EX) response.session_locked = True self.update(pickle.load(response.session_file)) response.session_file.seek(0) oc = response.session_filename.split('/')[-1].split( '-')[0] if check_client and response.session_client != oc: raise Exception("cookie attack") except: response.session_id = None if not response.session_id: uuid = web2py_uuid() response.session_id = '%s-%s' % (response.session_client, uuid) separate = separate and ( lambda session_name: session_name[-2:]) if separate: prefix = separate(response.session_id) response.session_id = '%s/%s' % (prefix, response.session_id) response.session_filename = \ os.path.join(up(request.folder), masterapp, 'sessions', response.session_id) response.session_new = True # else the session goes in db elif response.session_storage_type == 'db': if global_settings.db_sessions is not True: global_settings.db_sessions.add(masterapp) # if had a session on file alreday, close it (yes, can happen) if response.session_file: self._close(response) # if on GAE tickets go also in DB if settings.global_settings.web2py_runtime_gae: request.tickets_db = db if masterapp == request.application: table_migrate = migrate else: table_migrate = False tname = tablename + '_' + masterapp table = db.get(tname, None) # Field = db.Field if table is None: db.define_table( tname, Field('locked', 'boolean', default=False), Field('client_ip', length=64), Field('created_datetime', 'datetime', default=request.now), Field('modified_datetime', 'datetime'), Field('unique_key', length=64), Field('session_data', 'blob'), migrate=table_migrate, ) table = db[tname] # to allow for lazy table response.session_db_table = table if response.session_id: # Get session data out of the database try: (record_id, unique_key) = response.session_id.split(':') record_id = long(record_id) except (TypeError, ValueError): record_id = None # Select from database if record_id: row = table(record_id, unique_key=unique_key) # Make sure the session data exists in the database if row: # rows[0].update_record(locked=True) # Unpickle the data session_data = pickle.loads(row.session_data) self.update(session_data) response.session_new = False else: record_id = None if record_id: response.session_id = '%s:%s' % (record_id, unique_key) response.session_db_unique_key = unique_key response.session_db_record_id = record_id else: response.session_id = None response.session_new = True # if there is no session id yet, we'll need to create a # new session else: response.session_new = True # set the cookie now if you know the session_id so user can set # cookie attributes in controllers/models # cookie will be reset later # yet cookie may be reset later # Removed comparison between old and new session ids - should send # the cookie all the time if isinstance(response.session_id, str): response.cookies[response.session_id_name] = response.session_id response.cookies[response.session_id_name]['path'] = '/' if cookie_expires: response.cookies[response.session_id_name]['expires'] = \ cookie_expires.strftime(FMT) session_pickled = pickle.dumps(self, pickle.HIGHEST_PROTOCOL) response.session_hash = hashlib.md5(session_pickled).hexdigest() if self.flash: (response.flash, self.flash) = (self.flash, None)
def ccache(): form = FORM( P(TAG.BUTTON("Clear CACHE?", _type="submit", _name="yes", _value="yes")), P(TAG.BUTTON("Clear RAM", _type="submit", _name="ram", _value="ram")), P(TAG.BUTTON("Clear DISK", _type="submit", _name="disk", _value="disk")), ) if form.accepts(request.vars, session): clear_ram = False clear_disk = False session.flash = "" if request.vars.yes: clear_ram = clear_disk = True if request.vars.ram: clear_ram = True if request.vars.disk: clear_disk = True if clear_ram: cache.ram.clear() session.flash += "Ram Cleared " if clear_disk: cache.disk.clear() session.flash += "Disk Cleared" redirect(URL(r=request)) try: from guppy import hpy; hp=hpy() except ImportError: hp = False import shelve, os, copy, time, math from gluon import portalocker ram = { 'bytes': 0, 'objects': 0, 'hits': 0, 'misses': 0, 'ratio': 0, 'oldest': time.time() } disk = copy.copy(ram) total = copy.copy(ram) for key, value in cache.ram.storage.items(): if isinstance(value, dict): ram['hits'] = value['hit_total'] - value['misses'] ram['misses'] = value['misses'] try: ram['ratio'] = ram['hits'] * 100 / value['hit_total'] except (KeyError, ZeroDivisionError): ram['ratio'] = 0 else: if hp: ram['bytes'] += hp.iso(value[1]).size ram['objects'] += hp.iso(value[1]).count if value[0] < ram['oldest']: ram['oldest'] = value[0] locker = open(os.path.join(request.folder, 'cache/cache.lock'), 'a') portalocker.lock(locker, portalocker.LOCK_EX) disk_storage = shelve.open(os.path.join(request.folder, 'cache/cache.shelve')) try: for key, value in disk_storage.items(): if isinstance(value, dict): disk['hits'] = value['hit_total'] - value['misses'] disk['misses'] = value['misses'] try: disk['ratio'] = disk['hits'] * 100 / value['hit_total'] except (KeyError, ZeroDivisionError): disk['ratio'] = 0 else: if hp: disk['bytes'] += hp.iso(value[1]).size disk['objects'] += hp.iso(value[1]).count if value[0] < disk['oldest']: disk['oldest'] = value[0] finally: portalocker.unlock(locker) locker.close() disk_storage.close() total['bytes'] = ram['bytes'] + disk['bytes'] total['objects'] = ram['objects'] + disk['objects'] total['hits'] = ram['hits'] + disk['hits'] total['misses'] = ram['misses'] + disk['misses'] try: total['ratio'] = total['hits'] * 100 / (total['hits'] + total['misses']) except (KeyError, ZeroDivisionError): total['ratio'] = 0 if disk['oldest'] < ram['oldest']: total['oldest'] = disk['oldest'] else: total['oldest'] = ram['oldest'] def GetInHMS(seconds): hours = math.floor(seconds / 3600) seconds -= hours * 3600 minutes = math.floor(seconds / 60) seconds -= minutes * 60 seconds = math.floor(seconds) return (hours, minutes, seconds) ram['oldest'] = GetInHMS(time.time() - ram['oldest']) disk['oldest'] = GetInHMS(time.time() - disk['oldest']) total['oldest'] = GetInHMS(time.time() - total['oldest']) return dict(form=form, total=total, ram=ram, disk=disk)
def lock(name=''): from gluon.globals import current locker = open(os.path.join(current.request.folder, 'debug_%s.lock' % name), 'a') portalocker.lock(locker, portalocker.LOCK_EX) return locker
def ccache(): form = FORM( P( TAG.BUTTON(T("Clear CACHE?"), _type="submit", _name="yes", _value="yes")), P(TAG.BUTTON(T("Clear RAM"), _type="submit", _name="ram", _value="ram")), P( TAG.BUTTON(T("Clear DISK"), _type="submit", _name="disk", _value="disk")), ) if form.accepts(request.vars, session): clear_ram = False clear_disk = False session.flash = "" if request.vars.yes: clear_ram = clear_disk = True if request.vars.ram: clear_ram = True if request.vars.disk: clear_disk = True if clear_ram: cache.ram.clear() session.flash += T("Ram Cleared") if clear_disk: cache.disk.clear() session.flash += T("Disk Cleared") redirect(URL(r=request)) try: from guppy import hpy hp = hpy() except ImportError: hp = False import shelve, os, copy, time, math from gluon import portalocker ram = { 'entries': 0, 'bytes': 0, 'objects': 0, 'hits': 0, 'misses': 0, 'ratio': 0, 'oldest': time.time(), 'keys': [] } disk = copy.copy(ram) total = copy.copy(ram) disk['keys'] = [] total['keys'] = [] def GetInHMS(seconds): hours = math.floor(seconds / 3600) seconds -= hours * 3600 minutes = math.floor(seconds / 60) seconds -= minutes * 60 seconds = math.floor(seconds) return (hours, minutes, seconds) for key, value in cache.ram.storage.items(): if isinstance(value, dict): ram['hits'] = value['hit_total'] - value['misses'] ram['misses'] = value['misses'] try: ram['ratio'] = ram['hits'] * 100 / value['hit_total'] except (KeyError, ZeroDivisionError): ram['ratio'] = 0 else: if hp: ram['bytes'] += hp.iso(value[1]).size ram['objects'] += hp.iso(value[1]).count ram['entries'] += 1 if value[0] < ram['oldest']: ram['oldest'] = value[0] ram['keys'].append((key, GetInHMS(time.time() - value[0]))) locker = open(os.path.join(request.folder, 'cache/cache.lock'), 'a') portalocker.lock(locker, portalocker.LOCK_EX) disk_storage = shelve.open( os.path.join(request.folder, 'cache/cache.shelve')) try: for key, value in disk_storage.items(): if isinstance(value, dict): disk['hits'] = value['hit_total'] - value['misses'] disk['misses'] = value['misses'] try: disk['ratio'] = disk['hits'] * 100 / value['hit_total'] except (KeyError, ZeroDivisionError): disk['ratio'] = 0 else: if hp: disk['bytes'] += hp.iso(value[1]).size disk['objects'] += hp.iso(value[1]).count disk['entries'] += 1 if value[0] < disk['oldest']: disk['oldest'] = value[0] disk['keys'].append((key, GetInHMS(time.time() - value[0]))) finally: portalocker.unlock(locker) locker.close() disk_storage.close() total['entries'] = ram['entries'] + disk['entries'] total['bytes'] = ram['bytes'] + disk['bytes'] total['objects'] = ram['objects'] + disk['objects'] total['hits'] = ram['hits'] + disk['hits'] total['misses'] = ram['misses'] + disk['misses'] total['keys'] = ram['keys'] + disk['keys'] try: total['ratio'] = total['hits'] * 100 / (total['hits'] + total['misses']) except (KeyError, ZeroDivisionError): total['ratio'] = 0 if disk['oldest'] < ram['oldest']: total['oldest'] = disk['oldest'] else: total['oldest'] = ram['oldest'] ram['oldest'] = GetInHMS(time.time() - ram['oldest']) disk['oldest'] = GetInHMS(time.time() - disk['oldest']) total['oldest'] = GetInHMS(time.time() - total['oldest']) def key_table(keys): return TABLE( TR(TD(B(T('Key'))), TD(B(T('Time in Cache (h:m:s)')))), *[TR(TD(k[0]), TD('%02d:%02d:%02d' % k[1])) for k in keys], **dict(_class='cache-keys', _style="border-collapse: separate; border-spacing: .5em;")) ram['keys'] = key_table(ram['keys']) disk['keys'] = key_table(disk['keys']) total['keys'] = key_table(total['keys']) return dict(form=form, total=total, ram=ram, disk=disk, object_stats=hp != False)
msg = prefix + prefix.join(warnings) print >> sys.stderr, msg if errors: # Report errors and stop. actionrequired = T("ACTION REQUIRED") prefix = "\n%s: " % actionrequired msg = prefix + prefix.join(errors) print >> sys.stderr, msg htmlprefix = "\n<br /><b>%s</b>: " % actionrequired html = "<errors>" + htmlprefix + htmlprefix.join(errors) + "\n</errors>" raise HTTP(500, body=html) # Create or update the canary file. from gluon import portalocker canary = open("applications/%s/models/0000_update_check.py" % appname, "w") portalocker.lock(canary, portalocker.LOCK_EX) statement = "CANARY_UPDATE_CHECK_ID = %s" % CURRENT_UPDATE_CHECK_ID canary.write(statement) canary.close() # ----------------------------------------------------------------------------- import os try: # Python 2.7 from collections import OrderedDict except: # Python 2.6 from gluon.contrib.simplejson.ordered_dict import OrderedDict from gluon import current
def ccache(): form = FORM( P(TAG.BUTTON("Clear CACHE?", _type="submit", _name="yes", _value="yes")), P(TAG.BUTTON("Clear RAM", _type="submit", _name="ram", _value="ram")), P(TAG.BUTTON("Clear DISK", _type="submit", _name="disk", _value="disk")), ) if form.accepts(request.vars, session): clear_ram = False clear_disk = False session.flash = "" if request.vars.yes: clear_ram = clear_disk = True if request.vars.ram: clear_ram = True if request.vars.disk: clear_disk = True if clear_ram: cache.ram.clear() session.flash += "Ram Cleared " if clear_disk: cache.disk.clear() session.flash += "Disk Cleared" redirect(URL(r=request)) try: from guppy import hpy; hp=hpy() except: hp = False import shelve, os, copy, time, math from gluon import portalocker ram = { 'bytes': 0, 'objects': 0, 'hits': 0, 'misses': 0, 'ratio': 0, 'oldest': time.time() } disk = copy.copy(ram) total = copy.copy(ram) for key, value in cache.ram.storage.items(): if isinstance(value, dict): ram['hits'] = value['hit_total'] - value['misses'] ram['misses'] = value['misses'] try: ram['ratio'] = ram['hits'] * 100 / value['hit_total'] except: ram['ratio'] = 0 else: if hp: ram['bytes'] += hp.iso(value[1]).size ram['objects'] += hp.iso(value[1]).count if value[0] < ram['oldest']: ram['oldest'] = value[0] locker = open(os.path.join(request.folder, 'cache/cache.lock'), 'a') portalocker.lock(locker, portalocker.LOCK_EX) disk_storage = shelve.open( os.path.join(request.folder, 'cache/cache.shelve')) for key, value in disk_storage.items(): if isinstance(value, dict): disk['hits'] = value['hit_total'] - value['misses'] disk['misses'] = value['misses'] try: disk['ratio'] = disk['hits'] * 100 / value['hit_total'] except: disk['ratio'] = 0 else: if hp: disk['bytes'] += hp.iso(value[1]).size disk['objects'] += hp.iso(value[1]).count if value[0] < disk['oldest']: disk['oldest'] = value[0] portalocker.unlock(locker) locker.close() disk_storage.close() total['bytes'] = ram['bytes'] + disk['bytes'] total['objects'] = ram['objects'] + disk['objects'] total['hits'] = ram['hits'] + disk['hits'] total['misses'] = ram['misses'] + disk['misses'] total['ratio'] = (ram['ratio'] + disk['ratio']) / 2 if disk['oldest'] < ram['oldest']: total['oldest'] = disk['oldest'] else: total['oldest'] = ram['oldest'] def GetInHMS(seconds): hours = math.floor(seconds / 3600) seconds -= hours * 3600 minutes = math.floor(seconds / 60) seconds -= minutes * 60 seconds = math.floor(seconds) return (hours, minutes, seconds) ram['oldest'] = GetInHMS(time.time() - ram['oldest']) disk['oldest'] = GetInHMS(time.time() - disk['oldest']) total['oldest'] = GetInHMS(time.time() - total['oldest']) return dict(form=form, total=total, ram=ram, disk=disk)
def ccache(): cache.ram.initialize() cache.disk.initialize() form = FORM( P(TAG.BUTTON(T("Clear CACHE?"), _type="submit", _name="yes", _value="yes")), P(TAG.BUTTON(T("Clear RAM"), _type="submit", _name="ram", _value="ram")), P(TAG.BUTTON(T("Clear DISK"), _type="submit", _name="disk", _value="disk")), ) if form.accepts(request.vars, session): clear_ram = False clear_disk = False session.flash = "" if request.vars.yes: clear_ram = clear_disk = True if request.vars.ram: clear_ram = True if request.vars.disk: clear_disk = True if clear_ram: cache.ram.clear() session.flash += T("Ram Cleared") if clear_disk: cache.disk.clear() session.flash += T("Disk Cleared") redirect(URL(r=request)) try: from guppy import hpy hp = hpy() except ImportError: hp = False import shelve import os import copy import time import math from gluon import portalocker ram = { "entries": 0, "bytes": 0, "objects": 0, "hits": 0, "misses": 0, "ratio": 0, "oldest": time.time(), "keys": [], } disk = copy.copy(ram) total = copy.copy(ram) disk["keys"] = [] total["keys"] = [] def GetInHMS(seconds): hours = math.floor(seconds / 3600) seconds -= hours * 3600 minutes = math.floor(seconds / 60) seconds -= minutes * 60 seconds = math.floor(seconds) return (hours, minutes, seconds) for key, value in cache.ram.storage.iteritems(): if isinstance(value, dict): ram["hits"] = value["hit_total"] - value["misses"] ram["misses"] = value["misses"] try: ram["ratio"] = ram["hits"] * 100 / value["hit_total"] except (KeyError, ZeroDivisionError): ram["ratio"] = 0 else: if hp: ram["bytes"] += hp.iso(value[1]).size ram["objects"] += hp.iso(value[1]).count ram["entries"] += 1 if value[0] < ram["oldest"]: ram["oldest"] = value[0] ram["keys"].append((key, GetInHMS(time.time() - value[0]))) folder = os.path.join(request.folder, "cache") if not os.path.exists(folder): os.mkdir(folder) locker = open(os.path.join(folder, "cache.lock"), "a") portalocker.lock(locker, portalocker.LOCK_EX) disk_storage = shelve.open(os.path.join(folder, "cache.shelve")) try: for key, value in disk_storage.items(): if isinstance(value, dict): disk["hits"] = value["hit_total"] - value["misses"] disk["misses"] = value["misses"] try: disk["ratio"] = disk["hits"] * 100 / value["hit_total"] except (KeyError, ZeroDivisionError): disk["ratio"] = 0 else: if hp: disk["bytes"] += hp.iso(value[1]).size disk["objects"] += hp.iso(value[1]).count disk["entries"] += 1 if value[0] < disk["oldest"]: disk["oldest"] = value[0] disk["keys"].append((key, GetInHMS(time.time() - value[0]))) finally: portalocker.unlock(locker) locker.close() disk_storage.close() total["entries"] = ram["entries"] + disk["entries"] total["bytes"] = ram["bytes"] + disk["bytes"] total["objects"] = ram["objects"] + disk["objects"] total["hits"] = ram["hits"] + disk["hits"] total["misses"] = ram["misses"] + disk["misses"] total["keys"] = ram["keys"] + disk["keys"] try: total["ratio"] = total["hits"] * 100 / (total["hits"] + total["misses"]) except (KeyError, ZeroDivisionError): total["ratio"] = 0 if disk["oldest"] < ram["oldest"]: total["oldest"] = disk["oldest"] else: total["oldest"] = ram["oldest"] ram["oldest"] = GetInHMS(time.time() - ram["oldest"]) disk["oldest"] = GetInHMS(time.time() - disk["oldest"]) total["oldest"] = GetInHMS(time.time() - total["oldest"]) def key_table(keys): return TABLE( TR(TD(B(T("Key"))), TD(B(T("Time in Cache (h:m:s)")))), *[TR(TD(k[0]), TD("%02d:%02d:%02d" % k[1])) for k in keys], **dict(_class="cache-keys", _style="border-collapse: separate; border-spacing: .5em;") ) ram["keys"] = key_table(ram["keys"]) disk["keys"] = key_table(disk["keys"]) total["keys"] = key_table(total["keys"]) return dict(form=form, total=total, ram=ram, disk=disk, object_stats=hp != False)