def test_LockedFile(self):
f = LockedFile("test.txt", mode="wb")
f.write(to_bytes("test ok"))
f.close()
f = LockedFile("test.txt", mode="rb")
self.assertEqual(f.read(), to_bytes("test ok"))
f.close()
def test_LockedFile(self):
f = LockedFile('test.txt', mode='wb')
f.write(to_bytes('test ok'))
f.close()
f = LockedFile('test.txt', mode='rb')
self.assertEqual(f.read(), to_bytes('test ok'))
f.close()
def test_LockedFile(self):
f = LockedFile('test.txt', mode='wb')
f.write(to_bytes('test ok'))
f.close()
f = LockedFile('test.txt', mode='rb')
self.assertEqual(f.read(), to_bytes('test ok'))
f.close()
def secure_loads(data, encryption_key, hash_key=None, compression_level=None):
encryption_key = to_bytes(encryption_key)
data = to_native(data)
if ':' not in data:
return None
if not hash_key:
hash_key = sha1(encryption_key).hexdigest()
signature, encrypted_data = data.split(':', 1)
encrypted_data = to_bytes(encrypted_data)
actual_signature = hmac.new(to_bytes(hash_key), encrypted_data,
hashlib.md5).hexdigest()
if not compare(signature, actual_signature):
return None
key = pad(encryption_key)[:32]
encrypted_data = base64.urlsafe_b64decode(encrypted_data)
IV, encrypted_data = encrypted_data[:16], encrypted_data[16:]
cipher, _ = AES_new(key, IV=IV)
try:
data = cipher.decrypt(encrypted_data)
data = data.rstrip(b' ')
if compression_level:
data = zlib.decompress(data)
return pickle.loads(data)
except Exception as e:
return None
def secure_dumps(data, encryption_key, hash_key=None, compression_level=None):
encryption_key = to_bytes(encryption_key)
if not hash_key:
hash_key = sha1(encryption_key).hexdigest()
dump = pickle.dumps(data, pickle.HIGHEST_PROTOCOL)
if compression_level:
dump = zlib.compress(dump, compression_level)
key = pad(encryption_key)[:32]
cipher, IV = AES_new(key)
encrypted_data = base64.urlsafe_b64encode(IV + cipher.encrypt(pad(dump)))
signature = to_bytes(hmac.new(to_bytes(hash_key), encrypted_data, hashlib.md5).hexdigest())
return signature + b':' + encrypted_data
def secure_dumps(data, encryption_key, hash_key=None, compression_level=None):
encryption_key = to_bytes(encryption_key)
if not hash_key:
hash_key = sha1(encryption_key).hexdigest()
dump = pickle.dumps(data, pickle.HIGHEST_PROTOCOL)
if compression_level:
dump = zlib.compress(dump, compression_level)
key = pad(encryption_key)[:32]
cipher, IV = AES_new(key)
encrypted_data = base64.urlsafe_b64encode(IV + cipher.encrypt(pad(dump)))
signature = to_bytes(
hmac.new(to_bytes(hash_key), encrypted_data, hashlib.md5).hexdigest())
return signature + b':' + encrypted_data
def test_readline(self):
f = LockedFile("test.txt", "wb")
f.write(to_bytes("abc\n"))
f.write(to_bytes("123\n"))
f.close()
f = LockedFile("test.txt", "rb")
rl = f.readline()
self.assertTrue(to_bytes("abc") in rl)
rl = f.readline()
self.assertTrue(to_bytes("123") in rl)
f.close()
f = LockedFile("test.txt", "rb")
rls = f.readlines()
f.close()
self.assertEqual(len(rls), 2)
def test_readline(self):
f = LockedFile('test.txt', 'wb')
f.write(to_bytes('abc\n'))
f.write(to_bytes('123\n'))
f.close()
f = LockedFile('test.txt', 'rb')
rl = f.readline()
self.assertTrue(to_bytes('abc') in rl)
rl = f.readline()
self.assertTrue(to_bytes('123') in rl)
f.close()
f = LockedFile('test.txt', 'rb')
rls = f.readlines()
f.close()
self.assertEqual(len(rls), 2)
def test_readline(self):
f = LockedFile('test.txt', 'wb')
f.write(to_bytes('abc\n'))
f.write(to_bytes('123\n'))
f.close()
f = LockedFile('test.txt', 'rb')
rl = f.readline()
self.assertTrue(to_bytes('abc') in rl)
rl = f.readline()
self.assertTrue(to_bytes('123') in rl)
f.close()
f = LockedFile('test.txt', 'rb')
rls = f.readlines()
f.close()
self.assertEqual(len(rls), 2)
def test_read_locked(self):
def worker(fh):
time.sleep(2)
fh.close()
f = LockedFile("test.txt", mode="wb")
f.write(to_bytes("test ok"))
t1 = threading.Thread(target=worker, args=(f, ))
t1.start()
start = int(time.time())
content = read_locked("test.txt")
end = int(time.time())
t1.join()
# it took at least 2 seconds to read
self.assertTrue(end - start >= 2)
self.assertEqual(content, to_bytes("test ok"))
def load(self):
self.local.session_cookie_name = "%s_session" % request.app_name
self.local.changed = False
self.local.secure = request.url.startswith("https")
self.local.data = {}
raw_token = request.get_cookie(
self.local.session_cookie_name
) or request.query.get("_session_token")
if not raw_token and request.method in ("POST", "PUT", "DELETE"):
raw_token = (request.forms and request.forms.get("_session_token")) or (
request.json and request.json and request.json.get("_session_token")
)
if raw_token:
token_data = to_bytes(raw_token)
try:
if self.storage:
json_data = self.storage.get(token_data)
if json_data:
self.local.data = json.loads(json_data)
else:
self.local.data = jwt.decode(
token_data, self.secret, algorithms=[self.algorithm]
)
if self.expiration is not None and self.storage is None:
assert self.local.data["timestamp"] > time.time() - int(
self.expiration
)
assert self.local.data.get("secure") == self.local.secure
except Exception:
pass
if not "uuid" in self.local.data:
self.clear()
def test_openmultiple(self):
t0 = time.time()
def worker1():
start = time.time()
f1 = LockedFile('test.txt', mode='ab')
time.sleep(2)
f1.write(to_bytes("%s\t%s\n" % (start, time.time())))
f1.close()
f = LockedFile('test.txt', mode='wb')
f.write(to_bytes(''))
f.close()
th = []
for x in range(10):
t1 = threading.Thread(target=worker1)
th.append(t1)
t1.start()
for t in th:
t.join()
with open('test.txt') as g:
content = g.read()
results = [line.strip().split('\t') for line in content.split('\n') if line]
# all started at more or less the same time
starts = [1 for line in results if float(line[0])-t0<1]
ends = [line[1] for line in results]
self.assertEqual(sum(starts), len(starts))
# end - start is at least 2
for line in results:
self.assertTrue(float(line[1]) - float(line[0]) >= 2)
# ends are not the same
self.assertTrue(len(ends) == len(ends))
def test_openmultiple(self):
t0 = time.time()
def worker1():
start = time.time()
f1 = LockedFile('test.txt', mode='ab')
time.sleep(2)
f1.write(to_bytes("%s\t%s\n" % (start, time.time())))
f1.close()
f = LockedFile('test.txt', mode='wb')
f.write(to_bytes(''))
f.close()
th = []
for x in range(10):
t1 = threading.Thread(target=worker1)
th.append(t1)
t1.start()
for t in th:
t.join()
with open('test.txt') as g:
content = g.read()
results = [line.strip().split('\t') for line in content.split('\n') if line]
# all started at more or less the same time
starts = [1 for line in results if float(line[0])-t0<1]
ends = [line[1] for line in results]
self.assertEqual(sum(starts), len(starts))
# end - start is at least 2
for line in results:
self.assertTrue(float(line[1]) - float(line[0]) >= 2)
# ends are not the same
self.assertTrue(len(ends) == len(ends))
def load(self):
self.local.session_cookie_name = '%s_session' % request.app_name
self.local.changed = False
self.local.secure = request.url.startswith('https')
self.local.data = {}
raw_token = (request.get_cookie(self.local.session_cookie_name)
or request.query.get('_session_token'))
if not raw_token and request.method in ('POST', 'PUT', 'DELETE'):
raw_token = ((request.forms
and request.forms.get('_session_token'))
or (request.json and request.json
and request.json.get('_session_token')))
if raw_token:
token_data = to_bytes(raw_token)
try:
if self.storage:
json_data = self.storage.get(token_data)
if json_data:
self.local.data = json.loads(json_data)
else:
self.local.data = jwt.decode(token_data,
self.secret,
algorithms=[self.algorithm])
if self.expiration is not None and self.storage is None:
assert self.local.data['timestamp'] > time.time() - int(
self.expiration)
assert self.local.data.get('secure') == self.local.secure
except Exception:
pass
if not 'uuid' in self.local.data:
self.local.changed = True
self.local.data['uuid'] = str(uuid.uuid4())
self.local.data['secure'] = self.local.secure
def load(self):
self.local.session_cookie_name = '%s_session' % request.app_name
cookie_data = _compat.to_bytes(
request.get_cookie(self.local.session_cookie_name))
self.local.changed = False
self.local.secure = request.url.startswith('https')
self.local.data = {}
if cookie_data:
try:
if self.storage:
json_data = self.storage.get(cookie_data)
if json_data:
self.local.data = json.loads(json_data)
else:
self.local.data = jwt.decode(cookie_data,
self.secret,
algorithms=[self.algorithm])
if self.expiration is not None and self.storage is None:
assert self.local.data['timestamp'] > time.time() - int(
self.expiration)
assert self.local.data.get('secure') == self.local.secure
except (jwt.exceptions.InvalidSignatureError, AssertionError,
ValueError):
pass
if not 'uuid' in self.local.data:
self.local.changed = True
self.local.data['uuid'] = _compat.to_native(str(uuid.uuid4()))
self.local.data['secure'] = self.local.secure
def test_fpdf(self):
""" Basic PDF test and sanity checks """
self.assertEqual(
fpdf.FPDF_VERSION, pyfpdf.FPDF_VERSION, 'version mistmatch')
self.assertEqual(fpdf.FPDF, pyfpdf.FPDF, 'class mistmatch')
pdf = fpdf.FPDF()
pdf.add_page()
pdf.compress = False
pdf.set_font('Arial', '', 14)
pdf.ln(10)
pdf.write(5, 'hello world')
pdf_out = pdf.output('', 'S')
self.assertTrue(to_bytes(fpdf.FPDF_VERSION) in pdf_out, 'version string')
self.assertTrue(to_bytes('hello world') in pdf_out, 'sample message')
def test_read_locked(self):
def worker(fh):
time.sleep(2)
fh.close()
f = LockedFile('test.txt', mode='wb')
f.write(to_bytes('test ok'))
t1 = threading.Thread(target=worker, args=(f, ))
t1.start()
start = int(time.time())
content = read_locked('test.txt')
end = int(time.time())
t1.join()
# it took at least 2 seconds to read
self.assertTrue(end - start >= 2)
self.assertEqual(content, to_bytes('test ok'))
def test_lock_unlock(self):
def worker1(fh):
time.sleep(2)
unlock(fh)
def worker2(fh):
time.sleep(2)
fh.close()
f = open('test.txt', mode='wb')
lock(f, LOCK_EX)
f.write(to_bytes('test ok'))
t1 = threading.Thread(target=worker1, args=(f, ))
t1.start()
start = int(time.time())
content = read_locked('test.txt')
end = int(time.time())
t1.join()
f.close()
# it took at least 2 seconds to read
# although nothing is there until .close()
self.assertTrue(end - start >= 2)
self.assertEqual(content, to_bytes(''))
content = read_locked('test.txt')
self.assertEqual(content, to_bytes('test ok'))
f = LockedFile('test.txt', mode='wb')
f.write(to_bytes('test ok'))
t1 = threading.Thread(target=worker2, args=(f, ))
t1.start()
start = int(time.time())
content = read_locked('test.txt')
end = int(time.time())
t1.join()
# it took at least 2 seconds to read
# content is there because we called close()
self.assertTrue(end - start >= 2)
self.assertEqual(content, to_bytes('test ok'))
def test_lock_unlock(self):
def worker1(fh):
time.sleep(2)
unlock(fh)
def worker2(fh):
time.sleep(2)
fh.close()
f = open('test.txt', mode='wb')
lock(f, LOCK_EX)
f.write(to_bytes('test ok'))
t1 = threading.Thread(target=worker1, args=(f, ))
t1.start()
start = int(time.time())
content = read_locked('test.txt')
end = int(time.time())
t1.join()
f.close()
# it took at least 2 seconds to read
# although nothing is there until .close()
self.assertTrue(end - start >= 2)
self.assertEqual(content, to_bytes(''))
content = read_locked('test.txt')
self.assertEqual(content, to_bytes('test ok'))
f = LockedFile('test.txt', mode='wb')
f.write(to_bytes('test ok'))
t1 = threading.Thread(target=worker2, args=(f, ))
t1.start()
start = int(time.time())
content = read_locked('test.txt')
end = int(time.time())
t1.join()
# it took at least 2 seconds to read
# content is there because we called close()
self.assertTrue(end - start >= 2)
self.assertEqual(content, to_bytes('test ok'))
def simple_hash(text, key='', salt='', digest_alg='md5'):
"""
Generates hash with the given text using the specified
digest hashing algorithm
"""
text = to_bytes(text)
key = to_bytes(key)
salt = to_bytes(salt)
if not digest_alg:
raise RuntimeError("simple_hash with digest_alg=None")
elif not isinstance(digest_alg, str): # manual approach
h = digest_alg(text + key + salt)
elif digest_alg.startswith('pbkdf2'): # latest and coolest!
iterations, keylen, alg = digest_alg[7:-1].split(',')
return to_native(pbkdf2_hex(text, salt, int(iterations),
int(keylen), get_digest(alg)))
elif key: # use hmac
digest_alg = get_digest(digest_alg)
h = hmac.new(key + salt, text, digest_alg)
else: # compatible with third party systems
h = get_digest(digest_alg)()
h.update(text + salt)
return h.hexdigest()
def simple_hash(text, key='', salt='', digest_alg='md5'):
"""
Generates hash with the given text using the specified
digest hashing algorithm
"""
text = to_bytes(text)
key = to_bytes(key)
salt = to_bytes(salt)
if not digest_alg:
raise RuntimeError("simple_hash with digest_alg=None")
elif not isinstance(digest_alg, str): # manual approach
h = digest_alg(text + key + salt)
elif digest_alg.startswith('pbkdf2'): # latest and coolest!
iterations, keylen, alg = digest_alg[7:-1].split(',')
return to_native(
pbkdf2_hex(text, salt, int(iterations), int(keylen),
get_digest(alg)))
elif key: # use hmac
digest_alg = get_digest(digest_alg)
h = hmac.new(key + salt, text, digest_alg)
else: # compatible with third party systems
h = get_digest(digest_alg)()
h.update(text + salt)
return h.hexdigest()
def secure_loads(data, encryption_key, hash_key=None, compression_level=None):
encryption_key = to_bytes(encryption_key)
data = to_native(data)
if ':' not in data:
return None
if not hash_key:
hash_key = sha1(encryption_key).hexdigest()
signature, encrypted_data = data.split(':', 1)
encrypted_data = to_bytes(encrypted_data)
actual_signature = hmac.new(to_bytes(hash_key), encrypted_data, hashlib.md5).hexdigest()
if not compare(signature, actual_signature):
return None
key = pad(encryption_key)[:32]
encrypted_data = base64.urlsafe_b64decode(encrypted_data)
IV, encrypted_data = encrypted_data[:16], encrypted_data[16:]
cipher, _ = AES_new(key, IV=IV)
try:
data = cipher.decrypt(encrypted_data)
data = data.rstrip(b' ')
if compression_level:
data = zlib.decompress(data)
return pickle.loads(data)
except Exception as e:
return None
def test_write_locked(self):
def worker(fh):
time.sleep(2)
fh.close()
f = open("test.txt", mode="wb")
lock(f, LOCK_EX)
t1 = threading.Thread(target=worker, args=(f, ))
t1.start()
start = int(time.time())
write_locked("test.txt", to_bytes("test ok"))
end = int(time.time())
t1.join()
with open("test.txt") as g:
content = g.read()
# it took at least 2 seconds to read
self.assertTrue(end - start >= 2)
self.assertEqual(content, "test ok")
def test_write_locked(self):
def worker(fh):
time.sleep(2)
fh.close()
f = open('test.txt', mode='wb')
lock(f, LOCK_EX)
t1 = threading.Thread(target=worker, args=(f, ))
t1.start()
start = int(time.time())
write_locked('test.txt', to_bytes('test ok'))
end = int(time.time())
t1.join()
with open('test.txt') as g:
content = g.read()
# it took at least 2 seconds to read
self.assertTrue(end - start >= 2)
self.assertEqual(content, 'test ok')
def load(self):
self.local.session_cookie_name = '%s_session' % request.app_name
cookie_data = _compat.to_bytes(request.get_cookie(self.local.session_cookie_name))
self.local.changed = False
self.local.secure = request.url.startswith('https')
self.local.data = {}
if cookie_data:
try:
if self.storage:
json_data = self.storage.get(cookie_data)
if json_data:
self.local.data = json.loads(json_data)
else:
self.local.data = jwt.decode(cookie_data, self.secret, algorithms=[self.algorithm])
if self.expiration is not None and self.storage is None:
assert self.local.data['timestamp'] > time.time() - int(self.expiration)
assert self.local.data.get('secure') == self.local.secure
except (jwt.exceptions.InvalidSignatureError, AssertionError, ValueError):
pass
if not 'uuid' in self.local.data:
self.local.changed = True
self.local.data['uuid'] = str(uuid.uuid4())
self.local.data['secure'] = self.local.secure
def md5_hash(text):
""" Generates a md5 hash with the given text """
return md5(to_bytes(text)).hexdigest()
def upper_fun(s):
return to_bytes(to_unicode(s).upper())
def title_fun(s):
return to_bytes(to_unicode(s).title())
def pbkdf2_hex(data, salt, iterations=1000, keylen=24, hashfunc=None):
hashfunc = hashfunc or sha1
hmac = hashlib.pbkdf2_hmac(hashfunc().name, to_bytes(data),
to_bytes(salt), iterations, keylen)
return binascii.hexlify(hmac)
def post(self, url, data=None, cookies=None,
headers=None, auth=None, method='auto'):
self.url = self.app + url
# if this POST form requires a postback do it
if data and '_formname' in data and self.postbacks and \
self.history and self.history[-1][1] != self.url:
# to bypass the web2py CSRF need to get formkey
# before submitting the form
self.get(url, cookies=cookies, headers=headers, auth=auth)
# unless cookies are specified, recycle cookies
if cookies is None:
cookies = self.cookies
cookies = cookies or {}
headers = headers or {}
cj = cookielib.CookieJar()
args = [
urllib2.HTTPCookieProcessor(cj),
urllib2.HTTPHandler(debuglevel=0)
]
# if required do basic auth
if auth:
auth_handler = urllib2.HTTPBasicAuthHandler()
auth_handler.add_password(**auth)
args.append(auth_handler)
opener = urllib2.build_opener(*args)
# copy headers from dict to list of key,value
headers_list = []
for key, value in iteritems(self.default_headers):
if not key in headers:
headers[key] = value
for key, value in iteritems(headers):
if isinstance(value, (list, tuple)):
for v in value:
headers_list.append((key, v))
else:
headers_list.append((key, value))
# move cookies to headers
for key, value in iteritems(cookies):
headers_list.append(('Cookie', '%s=%s' % (key, value)))
# add headers to request
for key, value in headers_list:
opener.addheaders.append((key, str(value)))
# assume everything is ok and make http request
error = None
try:
if isinstance(data, str):
self.method = 'POST' if method=='auto' else method
elif isinstance(data, dict):
self.method = 'POST' if method=='auto' else method
# if there is only one form, set _formname automatically
if not '_formname' in data and len(self.forms) == 1:
data['_formname'] = self.forms.keys()[0]
# if there is no formkey but it is known, set it
if '_formname' in data and not '_formkey' in data and \
data['_formname'] in self.forms:
data['_formkey'] = self.forms[data['_formname']]
# time the POST request
data = urlencode(data, doseq=True)
else:
self.method = 'GET' if method=='auto' else method
data = None
t0 = time.time()
self.response = opener.open(self.url, to_bytes(data))
self.time = time.time() - t0
except urllib2.HTTPError as er:
error = er
# catch HTTP errors
self.time = time.time() - t0
self.response = er
if hasattr(self.response, 'getcode'):
self.status = self.response.getcode()
else:#python2.5
self.status = None
self.text = to_native(self.response.read())
# In PY3 self.response.headers are case sensitive
self.headers = dict()
for h in self.response.headers:
self.headers[h.lower()] = self.response.headers[h]
# treat web2py tickets as special types of errors
if error is not None:
if 'web2py_error' in self.headers:
raise RuntimeError(self.headers['web2py_error'])
else:
raise error
# parse headers into cookies
self.cookies = {}
if 'set-cookie' in self.headers:
for item in self.headers['set-cookie'].split(','):
key, value = item[:item.find(';')].split('=')
self.cookies[key.strip()] = value.strip()
# check is a new session id has been issued, symptom of broken session
if self.session_regex is not None:
for cookie, value in iteritems(self.cookies):
match = self.session_regex.match(cookie)
if match:
name = match.group('name')
if name in self.sessions and self.sessions[name] != value:
print(RuntimeError('Changed session ID %s' % name))
self.sessions[name] = value
# find all forms and formkeys in page
self.forms = {}
for match in FORM_REGEX.finditer(to_native(self.text)):
self.forms[match.group('formname')] = match.group('formkey')
# log this request
self.history.append((self.method, self.url, self.status, self.time))
def title_fun(s):
return to_bytes(to_unicode(s).title())
def cap_fun(s):
return to_bytes(to_unicode(s).capitalize())
def post(self,
url,
data=None,
cookies=None,
headers=None,
auth=None,
method='auto'):
self.url = self.app + url
# if this POST form requires a postback do it
if data and '_formname' in data and self.postbacks and \
self.history and self.history[-1][1] != self.url:
# to bypass the web2py CSRF need to get formkey
# before submitting the form
self.get(url, cookies=cookies, headers=headers, auth=auth)
# unless cookies are specified, recycle cookies
if cookies is None:
cookies = self.cookies
cookies = cookies or {}
headers = headers or {}
cj = cookielib.CookieJar()
args = [
urllib2.HTTPCookieProcessor(cj),
urllib2.HTTPHandler(debuglevel=0)
]
# if required do basic auth
if auth:
auth_handler = urllib2.HTTPBasicAuthHandler()
auth_handler.add_password(**auth)
args.append(auth_handler)
opener = urllib2.build_opener(*args)
# copy headers from dict to list of key,value
headers_list = []
for key, value in iteritems(self.default_headers):
if not key in headers:
headers[key] = value
for key, value in iteritems(headers):
if isinstance(value, (list, tuple)):
for v in value:
headers_list.append((key, v))
else:
headers_list.append((key, value))
# move cookies to headers
for key, value in iteritems(cookies):
headers_list.append(('Cookie', '%s=%s' % (key, value)))
# add headers to request
for key, value in headers_list:
opener.addheaders.append((key, str(value)))
# assume everything is ok and make http request
error = None
try:
if isinstance(data, str):
self.method = 'POST' if method == 'auto' else method
elif isinstance(data, dict):
self.method = 'POST' if method == 'auto' else method
# if there is only one form, set _formname automatically
if not '_formname' in data and len(self.forms) == 1:
data['_formname'] = self.forms.keys()[0]
# if there is no formkey but it is known, set it
if '_formname' in data and not '_formkey' in data and \
data['_formname'] in self.forms:
data['_formkey'] = self.forms[data['_formname']]
# time the POST request
data = urlencode(data, doseq=True)
else:
self.method = 'GET' if method == 'auto' else method
data = None
t0 = time.time()
self.response = opener.open(self.url, to_bytes(data))
self.time = time.time() - t0
except urllib2.HTTPError as er:
error = er
# catch HTTP errors
self.time = time.time() - t0
self.response = er
if hasattr(self.response, 'getcode'):
self.status = self.response.getcode()
else: #python2.5
self.status = None
self.text = to_native(self.response.read())
# In PY3 self.response.headers are case sensitive
self.headers = dict()
for h in self.response.headers:
self.headers[h.lower()] = self.response.headers[h]
# treat web2py tickets as special types of errors
if error is not None:
if 'web2py_error' in self.headers:
raise RuntimeError(self.headers['web2py_error'])
else:
raise error
# parse headers into cookies
self.cookies = {}
if 'set-cookie' in self.headers:
for item in self.headers['set-cookie'].split(','):
key, value = item[:item.find(';')].split('=')
self.cookies[key.strip()] = value.strip()
# check is a new session id has been issued, symptom of broken session
if self.session_regex is not None:
for cookie, value in iteritems(self.cookies):
match = self.session_regex.match(cookie)
if match:
name = match.group('name')
if name in self.sessions and self.sessions[name] != value:
print(RuntimeError('Changed session ID %s' % name))
self.sessions[name] = value
# find all forms and formkeys in page
self.forms = {}
for match in FORM_REGEX.finditer(to_native(self.text)):
self.forms[match.group('formname')] = match.group('formkey')
# log this request
self.history.append((self.method, self.url, self.status, self.time))
def key_filter_in_windows(key):
"""
Windows doesn't allow \ / : * ? "< > | in filenames.
To go around this encode the keys with base32.
"""
return to_native(base64.b32encode(to_bytes(key)))
def key_filter_in_windows(key):
"""
Windows doesn't allow \ / : * ? "< > | in filenames.
To go around this encode the keys with base32.
"""
return to_native(base64.b32encode(to_bytes(key)))
def test_routes_args(self):
'''
Test URL args parsing/generation
'''
data = r'''routes_in = [
('/robots.txt', '/welcome/static/robots.txt'),
('/favicon.ico', '/welcome/static/favicon.ico'),
('/admin$anything', '/admin$anything'),
('.*:https?://(.*\\.)?domain1.com:$method /', '/app1/default'),
('.*:https?://(.*\\.)?domain1.com:$method /static/$anything',
'/app1/static/$anything'),
('.*:https?://(.*\\.)?domain1.com:$method /appadmin/$anything',
'/app1/appadmin/$anything'),
('.*:https?://(.*\\.)?domain1.com:$method /$anything',
'/app1/default/$anything'),
('.*:https?://(.*\\.)?domain2.com:$method /', '/app2/default'),
('.*:https?://(.*\\.)?domain2.com:$method /static/$anything',
'/app2/static/$anything'),
('.*:https?://(.*\\.)?domain2.com:$method /appadmin/$anything',
'/app2/appadmin/$anything'),
('.*:https?://(.*\\.)?domain2.com:$method /$anything',
'/app2/default/$anything'),
('.*:https?://(.*\\.)?domain3.com:$method /', '/app3/defcon3'),
('.*:https?://(.*\\.)?domain3.com:$method /static/$anything',
'/app3/static/$anything'),
('.*:https?://(.*\\.)?domain3.com:$method /appadmin/$anything',
'/app3/appadmin/$anything'),
('.*:https?://(.*\\.)?domain3.com:$method /$anything',
'/app3/defcon3/$anything'),
('/', '/welcome/default'),
('/welcome/default/$anything', '/welcome/default/$anything'),
('/welcome/$anything', '/welcome/default/$anything'),
('/static/$anything', '/welcome/static/$anything'),
('/appadmin/$anything', '/welcome/appadmin/$anything'),
('/$anything', '/welcome/default/$anything'),
]
routes_out = [
('/welcome/static/$anything', '/static/$anything'),
('/welcome/appadmin/$anything', '/appadmin/$anything'),
('/welcome/default/$anything', '/$anything'),
('/app1/static/$anything', '/static/$anything'),
('/app1/appadmin/$anything', '/appadmin/$anything'),
('/app1/default/$anything', '/$anything'),
('/app2/static/$anything', '/static/$anything'),
('/app2/appadmin/$anything', '/appadmin/$anything'),
('/app2/default/$anything', '/$anything'),
('/app3/static/$anything', '/static/$anything'),
('/app3/appadmin/$anything', '/appadmin/$anything'),
('/app3/defcon3/$anything', '/$anything')
]
'''
load(data=data)
self.assertEqual(
filter_url('http://domain.com/welcome/default/f/arg1'),
"/welcome/default/f ['arg1']")
self.assertEqual(
filter_url('http://domain.com/welcome/default/f/arg1/'),
"/welcome/default/f ['arg1']")
self.assertEqual(
filter_url('http://domain.com/welcome/default/f/arg1//'),
"/welcome/default/f ['arg1', '']")
self.assertEqual(
filter_url('http://domain.com/welcome/default/f//arg1'),
"/welcome/default/f ['', 'arg1']")
self.assertEqual(
filter_url('http://domain.com/welcome/default/f/arg1/arg2'),
"/welcome/default/f ['arg1', 'arg2']")
self.assertEqual(
filter_url('http://domain.com/welcome/default/f/arg1//arg2'),
"/welcome/default/f ['arg1', '', 'arg2']")
self.assertEqual(
filter_url('http://domain.com/welcome/default/f/arg1//arg3/'),
"/welcome/default/f ['arg1', '', 'arg3']")
self.assertEqual(
filter_url('http://domain.com/welcome/default/f/arg1//arg3//'),
"/welcome/default/f ['arg1', '', 'arg3', '']")
self.assertEqual(
filter_url('http://domain.com/welcome/default/f', out=True), "/f")
self.assertEqual(regex_filter_out('/welcome/default/f'), "/f")
self.assertEqual(str(URL(a='welcome', c='default', f='f', args=None)),
"/f")
self.assertEqual(
str(URL(a='welcome', c='default', f='f', args=['arg1'])),
"/f/arg1")
self.assertEqual(
str(URL(a='welcome', c='default', f='f', args=['arg1', ''])),
"/f/arg1//")
self.assertEqual(
str(URL(a='welcome', c='default', f='f',
args=['arg1', '', 'arg3'])), "/f/arg1//arg3")
self.assertEqual(
str(URL(a='welcome', c='default', f='f', args=['ar g'])),
"/f/ar%20g")
self.assertEqual(
str(URL(a='welcome', c='default', f='f', args=['årg'])),
"/f/%C3%A5rg")
self.assertEqual(URL(a='welcome', c='default', f='fünc'), "/fünc")
self.assertEqual(to_bytes(URL(a='welcome', c='default', f='fünc')),
b"/f\xc3\xbcnc")
def __init__(self,
table,
record=None,
readonly=False,
deletable=True,
formstyle=FormStyleDefault,
dbio=True,
keep_values=False,
form_name=False,
hidden=None,
csrf_uuid=None):
if isinstance(table, list):
dbio = False
# mimic a table from a list of fields without calling define_table
form_name = form_name or 'none'
for field in table:
field.tablename = getattr(field, 'tablename', form_name)
if isinstance(record, (int, str)):
record_id = int(str(record))
self.record = table[record_id]
else:
self.record = record
self.table = table
self.readonly = readonly
self.deletable = deletable and not readonly and self.record
self.formstyle = formstyle
self.dbio = dbio
self.keep_values = True if keep_values or self.record else False
self.csrf_uuid = csrf_uuid and csrf_uuid
self.vars = {}
self.errors = {}
self.submitted = False
self.deleted = False
self.accepted = False
self.form_name = form_name or table._tablename
self.hidden = hidden
self.formkey = None
self.cached_helper = None
if readonly or request.method == 'GET':
if self.record:
self.vars = self.record
else:
post_vars = request.forms
self.submitted = True
process = False
# we only a process a form if it is POST and the formkey matches (correct formname and crsf)
# notice we never expose the crsf uuid, we only use to sign the form uuid
if request.method == 'POST':
if csrf_uuid:
code, signature = post_vars['_formkey'].split('/')
expected = hmac.new(to_bytes(csrf_uuid),
to_bytes(self.form_name + '/' +
code)).hexdigest()
if signature == expected:
process = True
elif post_vars.get('_formkey') == self.form_name:
process = True
if process:
if not post_vars.get('_delete'):
for field in self.table:
if field.writable:
value = post_vars.get(field.name)
# FIX THIS deal with set_self_id before validate
(value, error) = field.validate(value)
if field.type == 'upload':
delete = post_vars.get('_delete_' + field.name)
if value is not None and hasattr(
value, 'file'):
value = field.store(
value.file, value.filename,
field.uploadfolder)
elif self.record and not delete:
value = self.record.get(field.name)
else:
value = None
self.vars[field.name] = value
if error:
self.errors[field.name] = error
if self.record:
self.vars['id'] = self.record.id
if not self.errors:
self.accepted = True
if dbio:
self.update_or_insert()
elif dbio:
self.deleted = True
self.record.delete_record()
# store key for future CSRF
if csrf_uuid:
code = str(uuid.uuid4())
signature = hmac.new(to_bytes(csrf_uuid),
to_bytes(self.form_name + '/' +
code)).hexdigest()
self.formkey = '%s/%s' % (code, signature)
else:
self.formkey = self.form_name
def upper_fun(s):
return to_bytes(to_unicode(s).upper())
def test_routes_args(self):
'''
Test URL args parsing/generation
'''
data = r'''routes_in = [
('/robots.txt', '/welcome/static/robots.txt'),
('/favicon.ico', '/welcome/static/favicon.ico'),
('/admin$anything', '/admin$anything'),
('.*:https?://(.*\\.)?domain1.com:$method /', '/app1/default'),
('.*:https?://(.*\\.)?domain1.com:$method /static/$anything',
'/app1/static/$anything'),
('.*:https?://(.*\\.)?domain1.com:$method /appadmin/$anything',
'/app1/appadmin/$anything'),
('.*:https?://(.*\\.)?domain1.com:$method /$anything',
'/app1/default/$anything'),
('.*:https?://(.*\\.)?domain2.com:$method /', '/app2/default'),
('.*:https?://(.*\\.)?domain2.com:$method /static/$anything',
'/app2/static/$anything'),
('.*:https?://(.*\\.)?domain2.com:$method /appadmin/$anything',
'/app2/appadmin/$anything'),
('.*:https?://(.*\\.)?domain2.com:$method /$anything',
'/app2/default/$anything'),
('.*:https?://(.*\\.)?domain3.com:$method /', '/app3/defcon3'),
('.*:https?://(.*\\.)?domain3.com:$method /static/$anything',
'/app3/static/$anything'),
('.*:https?://(.*\\.)?domain3.com:$method /appadmin/$anything',
'/app3/appadmin/$anything'),
('.*:https?://(.*\\.)?domain3.com:$method /$anything',
'/app3/defcon3/$anything'),
('/', '/welcome/default'),
('/welcome/default/$anything', '/welcome/default/$anything'),
('/welcome/$anything', '/welcome/default/$anything'),
('/static/$anything', '/welcome/static/$anything'),
('/appadmin/$anything', '/welcome/appadmin/$anything'),
('/$anything', '/welcome/default/$anything'),
]
routes_out = [
('/welcome/static/$anything', '/static/$anything'),
('/welcome/appadmin/$anything', '/appadmin/$anything'),
('/welcome/default/$anything', '/$anything'),
('/app1/static/$anything', '/static/$anything'),
('/app1/appadmin/$anything', '/appadmin/$anything'),
('/app1/default/$anything', '/$anything'),
('/app2/static/$anything', '/static/$anything'),
('/app2/appadmin/$anything', '/appadmin/$anything'),
('/app2/default/$anything', '/$anything'),
('/app3/static/$anything', '/static/$anything'),
('/app3/appadmin/$anything', '/appadmin/$anything'),
('/app3/defcon3/$anything', '/$anything')
]
'''
load(data=data)
self.assertEqual(
filter_url('http://domain.com/welcome/default/f/arg1'),
"/welcome/default/f ['arg1']")
self.assertEqual(
filter_url('http://domain.com/welcome/default/f/arg1/'),
"/welcome/default/f ['arg1']")
self.assertEqual(
filter_url('http://domain.com/welcome/default/f/arg1//'),
"/welcome/default/f ['arg1', '']")
self.assertEqual(
filter_url('http://domain.com/welcome/default/f//arg1'),
"/welcome/default/f ['', 'arg1']")
self.assertEqual(
filter_url('http://domain.com/welcome/default/f/arg1/arg2'),
"/welcome/default/f ['arg1', 'arg2']")
self.assertEqual(
filter_url('http://domain.com/welcome/default/f/arg1//arg2'),
"/welcome/default/f ['arg1', '', 'arg2']")
self.assertEqual(
filter_url('http://domain.com/welcome/default/f/arg1//arg3/'),
"/welcome/default/f ['arg1', '', 'arg3']")
self.assertEqual(
filter_url('http://domain.com/welcome/default/f/arg1//arg3//'),
"/welcome/default/f ['arg1', '', 'arg3', '']")
self.assertEqual(
filter_url('http://domain.com/welcome/default/f', out=True), "/f")
self.assertEqual(regex_filter_out('/welcome/default/f'), "/f")
self.assertEqual(
str(URL(a='welcome', c='default', f='f', args=None)), "/f")
self.assertEqual(str(
URL(a='welcome', c='default', f='f', args=['arg1'])), "/f/arg1")
self.assertEqual(str(URL(
a='welcome', c='default', f='f', args=['arg1', ''])), "/f/arg1//")
self.assertEqual(str(URL(a='welcome', c='default', f='f',
args=['arg1', '', 'arg3'])), "/f/arg1//arg3")
self.assertEqual(str(
URL(a='welcome', c='default', f='f', args=['ar g'])), "/f/ar%20g")
self.assertEqual(str(URL(
a='welcome', c='default', f='f', args=['årg'])), "/f/%C3%A5rg")
self.assertEqual(
URL(a='welcome', c='default', f='fünc'), "/fünc")
self.assertEqual(
to_bytes(URL(a='welcome', c='default', f='fünc')), b"/f\xc3\xbcnc")
def key_filter_out_windows(key):
"""
We need to decode the keys so regex based removal works.
"""
return to_native(base64.b32decode(to_bytes(key)))
def worker1():
start = time.time()
f1 = LockedFile('test.txt', mode='ab')
time.sleep(2)
f1.write(to_bytes("%s\t%s\n" % (start, time.time())))
f1.close()
def cap_fun(s):
return to_bytes(to_unicode(s).capitalize())
def pbkdf2_hex(data, salt, iterations=1000, keylen=24, hashfunc=None):
hashfunc = hashfunc or sha1
hmac = hashlib.pbkdf2_hmac(hashfunc().name, to_bytes(data),
to_bytes(salt), iterations, keylen)
return binascii.hexlify(hmac)
def __init__(self,
table,
record=None,
readonly=False,
deletable=True,
formstyle=FormStyleDefault,
dbio=True,
keepvalues=False,
formname=False,
hidden=None,
csrf_uuid=None):
if isinstance(table, list):
dbio = False
# mimic a table from a list of fields without calling define_table
formname = formname or 'none'
for field in table: field.tablename = getattr(field,'tablename',formname)
if isinstance(record, (int, str)):
record_id = int(str(record))
self.record = table[record_id]
else:
self.record = record
self.table = table
self.readonly = readonly
self.deletable = deletable and not readonly and self.record
self.formstyle = formstyle
self.dbio = dbio
self.keepvalues = True if keepvalues or self.record else False
self.csrf_uuid = csrf_uuid and csrf_uuid
self.vars = {}
self.errors = {}
self.submitted = False
self.deleted = False
self.accepted = False
self.formname = formname or table._tablename
self.hidden = hidden
self.formkey = None
self.cached_helper = None
if readonly or request.method=='GET':
if self.record:
self.vars = self.record
else:
post_vars = request.forms
self.submitted = True
process = False
if request.method == 'POST':
if csrf_uuid:
a, b = post_vars['_formkey'].split('/')
if b == hmac.new(to_bytes(csrf_uuid), to_bytes(a)).hexdigest():
process = True
if process:
if not post_vars.get('_delete'):
for field in self.table:
if field.writable:
value = post_vars.get(field.name)
# FIX THIS deal with set_self_id before validate
(value, error) = field.validate(value)
if field.type == 'upload':
delete = post_vars.get('_delete_'+field.name)
if value is not None and hasattr(value,'file'):
value = field.store(value.file,
value.filename,
field.uploadfolder)
elif self.record and not delete:
value = self.record.get(field.name)
else:
value = None
self.vars[field.name] = value
if error:
self.errors[field.name] = error
if self.record:
self.vars['id'] = self.record.id
if not self.errors:
self.accepted = True
if dbio:
self.update_or_insert()
elif dbio:
self.deleted = True
self.record.delete_record()
# store key for future CSRF
if csrf_uuid:
a = str(uuid.uuid4())
self.formkey = '%s/%s' % (a, hmac.new(to_bytes(csrf_uuid), to_bytes(a)).hexdigest())
def worker1():
start = time.time()
f1 = LockedFile("test.txt", mode="ab")
time.sleep(2)
f1.write(to_bytes("%s\t%s\n" % (start, time.time())))
f1.close()
def md5_hash(text):
""" Generates a md5 hash with the given text """
return md5(to_bytes(text)).hexdigest()
def key_filter_out_windows(key):
"""
We need to decode the keys so regex based removal works.
"""
return to_native(base64.b32decode(to_bytes(key)))
def worker1():
start = int(time.time())
f1 = LockedFile('test.txt', mode='ab')
time.sleep(2)
f1.write(to_bytes("%s\t%s\n" % (start, int(time.time()))))
f1.close()
