def _try_store_in_file(self, request, response): try: if (not response.session_id or not response.session_filename or self._forget or self._unchanged(response)): # self.clear_session_cookies() return False else: if response.session_new or not response.session_file: # Tests if the session sub-folder exists, if not, create it session_folder = os.path.dirname(response.session_filename) if not os.path.exists(session_folder): os.mkdir(session_folder) response.session_file = recfile.open(response.session_filename, 'wb') portalocker.lock(response.session_file, portalocker.LOCK_EX) response.session_locked = True if response.session_file: session_pickled = response.session_pickled or pickle.dumps(self, pickle.HIGHEST_PROTOCOL) response.session_file.write(session_pickled) response.session_file.truncate() return True finally: self._close(response) self.save_session_id_cookie()
def _try_store_in_file(self, request, response): try: if (not response.session_id or not response.session_filename or self._forget or self._unchanged(response)): # self.clear_session_cookies() return False else: if response.session_new or not response.session_file: # Tests if the session sub-folder exists, if not, create it session_folder = os.path.dirname(response.session_filename) if not os.path.exists(session_folder): os.mkdir(session_folder) response.session_file = recfile.open( response.session_filename, 'wb') portalocker.lock(response.session_file, portalocker.LOCK_EX) response.session_locked = True if response.session_file: session_pickled = response.session_pickled or pickle.dumps( self, pickle.HIGHEST_PROTOCOL) response.session_file.write(session_pickled) response.session_file.truncate() return True finally: self._close(response) self.save_session_id_cookie()
def write_hosts_deny(denied_hosts): f = open(deny_file, 'w') portalocker.lock(f, portalocker.LOCK_EX) for key, val in denied_hosts.items(): if time.time() - val[1] < expiration_failed_logins: line = '%s %s %s\n' % (key, val[0], val[1]) f.write(line) portalocker.unlock(f) f.close()
def wait_portalock(self, val_file): """ Wait for the process file lock. """ while True: try: portalocker.lock(val_file, portalocker.LOCK_EX) break except: time.sleep(self.file_lock_time_wait)
def release(self): """ Writes into cron.master the time when cron job was completed """ if not self.master.closed: portalocker.lock(self.master, portalocker.LOCK_EX) logger.debug('WEB2PY CRON: Releasing cron lock') self.master.seek(0) (start, stop) = pickle.load(self.master) if start == self.now: # if this is my lock self.master.seek(0) pickle.dump((self.now, time.time()), self.master) portalocker.unlock(self.master) self.master.close()
def test_write_locked(self): def worker(fh): time.sleep(2) fh.close() f = open("test.txt", mode="wb") lock(f, LOCK_EX) t1 = threading.Thread(target=worker, args=(f, )) t1.start() start = int(time.time()) write_locked("test.txt", to_bytes("test ok")) end = int(time.time()) t1.join() with open("test.txt") as g: content = g.read() # it took at least 2 seconds to read self.assertTrue(end - start >= 2) self.assertEqual(content, "test ok")
def read_hosts_deny(): import datetime hosts = {} if os.path.exists(deny_file): hosts = {} f = open(deny_file, 'r') portalocker.lock(f, portalocker.LOCK_SH) for line in f.readlines(): if not line.strip() or line.startswith('#'): continue fields = line.strip().split() if len(fields) > 2: hosts[fields[0].strip()] = ( # ip int(fields[1].strip()), # n attemps int(fields[2].strip()) # last attempts ) portalocker.unlock(f) f.close() return hosts
def acquire(self, startup=False): """ Returns the time when the lock is acquired or None if cron already running lock is implemented by writing a pickle (start, stop) in cron.master start is time when cron job starts and stop is time when cron completed stop == 0 if job started but did not yet complete if a cron job started within less than 60 seconds, acquire returns None if a cron job started before 60 seconds and did not stop, a warning is issue "Stale cron.master detected" """ if sys.platform == 'win32': locktime = 59.5 else: locktime = 59.99 if portalocker.LOCK_EX is None: logger.warning('cron disabled because no file locking') return None self.master = fileutils.open_file(self.path, 'rb+') try: ret = None portalocker.lock(self.master, portalocker.LOCK_EX) try: (start, stop) = pickle.load(self.master) except: start = 0 stop = 1 if startup or self.now - start > locktime: ret = self.now if not stop: # this happens if previous cron job longer than 1 minute logger.warning('stale cron.master detected') logger.debug('acquiring lock') self.master.seek(0) pickle.dump((self.now, 0), self.master) self.master.flush() finally: portalocker.unlock(self.master) if not ret: # do this so no need to release self.master.close() return ret
def acquire(self, startup=False): """ Returns the time when the lock is acquired or None if cron already running lock is implemented by writing a pickle (start, stop) in cron.master start is time when cron job starts and stop is time when cron completed stop == 0 if job started but did not yet complete if a cron job started within less than 60 seconds, acquire returns None if a cron job started before 60 seconds and did not stop, a warning is issued ("Stale cron.master detected") """ if sys.platform == 'win32': locktime = 59.5 else: locktime = 59.99 if portalocker.LOCK_EX is None: self.logger.warning('cron disabled because no file locking') return None self.master = fileutils.open_file(self.path, 'rb+') ret = None try: portalocker.lock(self.master, portalocker.LOCK_EX) try: (start, stop) = pickle.load(self.master) except: start = 0 stop = 1 if startup or self.now - start > locktime: ret = self.now if not stop: # this happens if previous cron job longer than 1 minute self.logger.warning('stale cron.master detected') self.logger.debug('acquiring lock') self.master.seek(0) pickle.dump((self.now, 0), self.master) self.master.flush() finally: portalocker.unlock(self.master) if not ret: # do this so no need to release self.master.close() return ret
def test_write_locked(self): def worker(fh): time.sleep(2) fh.close() f = open('test.txt', mode='wb') lock(f, LOCK_EX) t1 = threading.Thread(target=worker, args=(f, )) t1.start() start = int(time.time()) write_locked('test.txt', to_bytes('test ok')) end = int(time.time()) t1.join() with open('test.txt') as g: content = g.read() # it took at least 2 seconds to read self.assertTrue(end - start >= 2) self.assertEqual(content, 'test ok')
def test_lock_unlock(self): def worker1(fh): time.sleep(2) unlock(fh) def worker2(fh): time.sleep(2) fh.close() f = open('test.txt', mode='wb') lock(f, LOCK_EX) f.write(to_bytes('test ok')) t1 = threading.Thread(target=worker1, args=(f, )) t1.start() start = int(time.time()) content = read_locked('test.txt') end = int(time.time()) t1.join() f.close() # it took at least 2 seconds to read # although nothing is there until .close() self.assertTrue(end - start >= 2) self.assertEqual(content, to_bytes('')) content = read_locked('test.txt') self.assertEqual(content, to_bytes('test ok')) f = LockedFile('test.txt', mode='wb') f.write(to_bytes('test ok')) t1 = threading.Thread(target=worker2, args=(f, )) t1.start() start = int(time.time()) content = read_locked('test.txt') end = int(time.time()) t1.join() # it took at least 2 seconds to read # content is there because we called close() self.assertTrue(end - start >= 2) self.assertEqual(content, to_bytes('test ok'))
def connect(self, request=None, response=None, db=None, tablename='web2py_session', masterapp=None, migrate=True, separate=None, check_client=False, cookie_key=None, cookie_expires=None, compression_level=None ): """ Used in models, allows to customize Session handling Args: request: the request object response: the response object db: to store/retrieve sessions in db (a table is created) tablename(str): table name masterapp(str): points to another's app sessions. This enables a "SSO" environment among apps migrate: passed to the underlying db separate: with True, creates a folder with the 2 initials of the session id. Can also be a function, e.g. :: separate=lambda(session_name): session_name[-2:] check_client: if True, sessions can only come from the same ip cookie_key(str): secret for cookie encryption cookie_expires: sets the expiration of the cookie compression_level(int): 0-9, sets zlib compression on the data before the encryption """ from gluon.dal import Field request = request or current.request response = response or current.response masterapp = masterapp or request.application cookies = request.cookies self._unlock(response) response.session_masterapp = masterapp response.session_id_name = 'session_id_%s' % masterapp.lower() response.session_data_name = 'session_data_%s' % masterapp.lower() response.session_cookie_expires = cookie_expires response.session_client = str(request.client).replace(':', '.') current._session_cookie_key = cookie_key response.session_cookie_compression_level = compression_level # check if there is a session_id in cookies try: old_session_id = cookies[response.session_id_name].value except KeyError: old_session_id = None response.session_id = old_session_id # if we are supposed to use cookie based session data if cookie_key: response.session_storage_type = 'cookie' elif db: response.session_storage_type = 'db' else: response.session_storage_type = 'file' # why do we do this? # because connect may be called twice, by web2py and in models. # the first time there is no db yet so it should do nothing if (global_settings.db_sessions is True or masterapp in global_settings.db_sessions): return if response.session_storage_type == 'cookie': # check if there is session data in cookies if response.session_data_name in cookies: session_cookie_data = cookies[response.session_data_name].value else: session_cookie_data = None if session_cookie_data: data = secure_loads(session_cookie_data, cookie_key, compression_level=compression_level) if data: self.update(data) response.session_id = True # else if we are supposed to use file based sessions elif response.session_storage_type == 'file': response.session_new = False response.session_file = None # check if the session_id points to a valid sesion filename if response.session_id: if not regex_session_id.match(response.session_id): response.session_id = None else: response.session_filename = \ os.path.join(up(request.folder), masterapp, 'sessions', response.session_id) try: response.session_file = \ recfile.open(response.session_filename, 'rb+') portalocker.lock(response.session_file, portalocker.LOCK_EX) response.session_locked = True self.update(pickle.load(response.session_file)) response.session_file.seek(0) oc = response.session_filename.split('/')[-1].split('-')[0] if check_client and response.session_client != oc: raise Exception("cookie attack") except: response.session_id = None if not response.session_id: uuid = web2py_uuid() response.session_id = '%s-%s' % (response.session_client, uuid) separate = separate and (lambda session_name: session_name[-2:]) if separate: prefix = separate(response.session_id) response.session_id = '%s/%s' % (prefix, response.session_id) response.session_filename = \ os.path.join(up(request.folder), masterapp, 'sessions', response.session_id) response.session_new = True # else the session goes in db elif response.session_storage_type == 'db': if global_settings.db_sessions is not True: global_settings.db_sessions.add(masterapp) # if had a session on file alreday, close it (yes, can happen) if response.session_file: self._close(response) # if on GAE tickets go also in DB if settings.global_settings.web2py_runtime_gae: request.tickets_db = db if masterapp == request.application: table_migrate = migrate else: table_migrate = False tname = tablename + '_' + masterapp table = db.get(tname, None) # Field = db.Field if table is None: db.define_table( tname, Field('locked', 'boolean', default=False), Field('client_ip', length=64), Field('created_datetime', 'datetime', default=request.now), Field('modified_datetime', 'datetime'), Field('unique_key', length=64), Field('session_data', 'blob'), migrate=table_migrate, ) table = db[tname] # to allow for lazy table response.session_db_table = table if response.session_id: # Get session data out of the database try: (record_id, unique_key) = response.session_id.split(':') record_id = long(record_id) except (TypeError, ValueError): record_id = None # Select from database if record_id: row = table(record_id, unique_key=unique_key) # Make sure the session data exists in the database if row: # rows[0].update_record(locked=True) # Unpickle the data session_data = pickle.loads(row.session_data) self.update(session_data) response.session_new = False else: record_id = None if record_id: response.session_id = '%s:%s' % (record_id, unique_key) response.session_db_unique_key = unique_key response.session_db_record_id = record_id else: response.session_id = None response.session_new = True # if there is no session id yet, we'll need to create a # new session else: response.session_new = True # set the cookie now if you know the session_id so user can set # cookie attributes in controllers/models # cookie will be reset later # yet cookie may be reset later # Removed comparison between old and new session ids - should send # the cookie all the time if isinstance(response.session_id, str): response.cookies[response.session_id_name] = response.session_id response.cookies[response.session_id_name]['path'] = '/' if cookie_expires: response.cookies[response.session_id_name]['expires'] = \ cookie_expires.strftime(FMT) session_pickled = pickle.dumps(self, pickle.HIGHEST_PROTOCOL) response.session_hash = hashlib.md5(session_pickled).hexdigest() if self.flash: (response.flash, self.flash) = (self.flash, None)
def connect(self, request=None, response=None, db=None, tablename='web2py_session', masterapp=None, migrate=True, separate=None, check_client=False, cookie_key=None, cookie_expires=None, compression_level=None): """ Used in models, allows to customize Session handling Args: request: the request object response: the response object db: to store/retrieve sessions in db (a table is created) tablename(str): table name masterapp(str): points to another's app sessions. This enables a "SSO" environment among apps migrate: passed to the underlying db separate: with True, creates a folder with the 2 initials of the session id. Can also be a function, e.g. :: separate=lambda(session_name): session_name[-2:] check_client: if True, sessions can only come from the same ip cookie_key(str): secret for cookie encryption cookie_expires: sets the expiration of the cookie compression_level(int): 0-9, sets zlib compression on the data before the encryption """ from gluon.dal import Field request = request or current.request response = response or current.response masterapp = masterapp or request.application cookies = request.cookies self._unlock(response) response.session_masterapp = masterapp response.session_id_name = 'session_id_%s' % masterapp.lower() response.session_data_name = 'session_data_%s' % masterapp.lower() response.session_cookie_expires = cookie_expires response.session_client = str(request.client).replace(':', '.') current._session_cookie_key = cookie_key response.session_cookie_compression_level = compression_level # check if there is a session_id in cookies try: old_session_id = cookies[response.session_id_name].value except KeyError: old_session_id = None response.session_id = old_session_id # if we are supposed to use cookie based session data if cookie_key: response.session_storage_type = 'cookie' elif db: response.session_storage_type = 'db' else: response.session_storage_type = 'file' # why do we do this? # because connect may be called twice, by web2py and in models. # the first time there is no db yet so it should do nothing if (global_settings.db_sessions is True or masterapp in global_settings.db_sessions): return if response.session_storage_type == 'cookie': # check if there is session data in cookies if response.session_data_name in cookies: session_cookie_data = cookies[response.session_data_name].value else: session_cookie_data = None if session_cookie_data: data = secure_loads(session_cookie_data, cookie_key, compression_level=compression_level) if data: self.update(data) response.session_id = True # else if we are supposed to use file based sessions elif response.session_storage_type == 'file': response.session_new = False response.session_file = None # check if the session_id points to a valid sesion filename if response.session_id: if not regex_session_id.match(response.session_id): response.session_id = None else: response.session_filename = \ os.path.join(up(request.folder), masterapp, 'sessions', response.session_id) try: response.session_file = \ recfile.open(response.session_filename, 'rb+') portalocker.lock(response.session_file, portalocker.LOCK_EX) response.session_locked = True self.update(pickle.load(response.session_file)) response.session_file.seek(0) oc = response.session_filename.split('/')[-1].split( '-')[0] if check_client and response.session_client != oc: raise Exception("cookie attack") except: response.session_id = None if not response.session_id: uuid = web2py_uuid() response.session_id = '%s-%s' % (response.session_client, uuid) separate = separate and ( lambda session_name: session_name[-2:]) if separate: prefix = separate(response.session_id) response.session_id = '%s/%s' % (prefix, response.session_id) response.session_filename = \ os.path.join(up(request.folder), masterapp, 'sessions', response.session_id) response.session_new = True # else the session goes in db elif response.session_storage_type == 'db': if global_settings.db_sessions is not True: global_settings.db_sessions.add(masterapp) # if had a session on file alreday, close it (yes, can happen) if response.session_file: self._close(response) # if on GAE tickets go also in DB if settings.global_settings.web2py_runtime_gae: request.tickets_db = db if masterapp == request.application: table_migrate = migrate else: table_migrate = False tname = tablename + '_' + masterapp table = db.get(tname, None) # Field = db.Field if table is None: db.define_table( tname, Field('locked', 'boolean', default=False), Field('client_ip', length=64), Field('created_datetime', 'datetime', default=request.now), Field('modified_datetime', 'datetime'), Field('unique_key', length=64), Field('session_data', 'blob'), migrate=table_migrate, ) table = db[tname] # to allow for lazy table response.session_db_table = table if response.session_id: # Get session data out of the database try: (record_id, unique_key) = response.session_id.split(':') record_id = long(record_id) except (TypeError, ValueError): record_id = None # Select from database if record_id: row = table(record_id, unique_key=unique_key) # Make sure the session data exists in the database if row: # rows[0].update_record(locked=True) # Unpickle the data session_data = pickle.loads(row.session_data) self.update(session_data) response.session_new = False else: record_id = None if record_id: response.session_id = '%s:%s' % (record_id, unique_key) response.session_db_unique_key = unique_key response.session_db_record_id = record_id else: response.session_id = None response.session_new = True # if there is no session id yet, we'll need to create a # new session else: response.session_new = True # set the cookie now if you know the session_id so user can set # cookie attributes in controllers/models # cookie will be reset later # yet cookie may be reset later # Removed comparison between old and new session ids - should send # the cookie all the time if isinstance(response.session_id, str): response.cookies[response.session_id_name] = response.session_id response.cookies[response.session_id_name]['path'] = '/' if cookie_expires: response.cookies[response.session_id_name]['expires'] = \ cookie_expires.strftime(FMT) session_pickled = pickle.dumps(self, pickle.HIGHEST_PROTOCOL) response.session_hash = hashlib.md5(session_pickled).hexdigest() if self.flash: (response.flash, self.flash) = (self.flash, None)