def decorated_task(task):
tasks.append(adict(
task=task,
seconds=number_of_seconds,
last=0,
name=task.__name__,
))
return task
def __delitem__(self, hash_key):
'''
Deletes item from db:
del db.user[email]
TODO: range_key support:
del db.message[topic][date]
'''
self.layer2.delete_item(item=adict(table=self, hash_key=hash_key, range_key=None))
def __getitem__(self, hash_key):
'''
Gets item from db:
user = db.user[email]
TODO: range_key support:
message = db.message[topic][date]
TODO: Option to control consistent_read.
'''
return adict(self.get_item(hash_key=hash_key, consistent_read=True))
def code():
with _local.db_conn.cursor(cursor_factory=RealDictCursor) as cursor:
if _log:
_log((sql, values))
cursor.execute(sql, values)
# Use "INSERT ... RETURNING" instead of "cursor.lastrowid":
# http://pythonhosted.org/psycopg2/cursor.html#cursor.lastrowid
affected = cursor.rowcount
try:
rows = [adict(row) for row in cursor.fetchall()] if affected else []
except ProgrammingError as e:
if 'no results to fetch' in repr(e):
rows = []
else:
raise
return rows, affected
def __init__(self, task_id, mode='train'):
self.mode = mode
raw_train, raw_test = get_raw_babi(task_id)
self.QA = adict()
self.QA.VOCAB = {'<PAD>': 0}
self.QA.IVOCAB = {0: '<PAD>'}
self.count_id = 1
self.tree_dict = {}
self.train, self.count_id, tree_dict1 = self.get_indexed_qa(
raw_train, self.count_id)
self.tree_dict.update(tree_dict1)
self.valid = [
self.train[i][int(-len(self.train[i]) / 10):] for i in range(3)
]
self.train = [
self.train[i][:int(9 * len(self.train[i]) / 10)] for i in range(3)
]
self.test, self.count_id, tree_dict2 = self.get_indexed_qa(
raw_test, self.count_id)
self.tree_dict.update(tree_dict2)
def db(sql, *values, **params):
"""
Queries DB and returns result.
@param str sql - SQL with %s placehodlers.
Do NOT add quotes or IN-brackets around %s.
E.g. db('...WHERE `name` IN %s', [value1, value2])
@param tuple values - Values for %s placehodlers.
@param dict params:
str charset - E.g. "utf8mb4" for tables with Emoji.
@return adict:
list(adict) rows - All rows in result.
adict|NoneType row - First row, if any.
int affected - Number of rows affected.
"""
charset = params.get('charset', db_config.charset)
def code():
if _local.db_conn.charset != charset:
_local.db_conn.set_charset(charset)
with _local.db_conn.cursor() as cursor:
with Timeout(db_config.query_timeout):
cursor.execute(sql, values)
return cursor.fetchall(), cursor.rowcount
rows, affected = db_transaction(code)
# It's just "rows, affected = code()" if already inside a transaction.
return adict(
rows=rows if isinstance(rows, list) else list(
rows
), # E.g. "DictCursorMixin" creates "list" from "tuple" only if it is not empty.
row=rows[0] if rows else None,
affected=affected,
)
def db(sql, *values, **params):
"""
Queries DB and returns result.
@param str sql - SQL with %s placehodlers.
Do NOT add quotes or IN-brackets around %s.
E.g. db('...WHERE `name` IN %s', [value1, value2])
@param tuple values - Values for %s placehodlers.
@param dict params:
str charset - E.g. "utf8mb4" for tables with Emoji.
@return adict:
list(adict) rows - All rows in result.
adict|NoneType row - First row, if any.
int affected - Number of rows affected.
"""
charset = params.get('charset', db_config.charset)
def code():
if _local.db_conn.charset != charset:
_local.db_conn.set_charset(charset)
with _local.db_conn.cursor() as cursor:
with Timeout(db_config.query_timeout):
cursor.execute(sql, values)
return cursor.fetchall(), cursor.rowcount
rows, affected = db_transaction(code)
# It's just "rows, affected = code()" if already inside a transaction.
return adict(
rows=rows if isinstance(rows, list) else list(rows), # E.g. "DictCursorMixin" creates "list" from "tuple" only if it is not empty.
row=rows[0] if rows else None,
affected=affected,
)
def __init__(self, field_name, state, explain=''):
error = adict(field=field_name, state=state)
if explain:
error.explain = explain
super(BadRequest, self).__init__(400, error)
def test():
pool_size = 2
db_config(name='test', user='******', password='******', pool_size=pool_size, pool_block=pool_size) # Use your database credentials!
### async, reconnect
'''
[0.0 seconds] Testing async pool (connections: 2), reconnect and retry:
[0.0 seconds] Greenlet 139815404809104 tries to "SELECT pg_sleep(5)".
[0.0 seconds] Greenlet 139815404810064 tries to "SELECT pg_sleep(5)".
[0.0 seconds] Greenlet 139815404808464 tries to "SELECT pg_sleep(5)".
[0.0 seconds] Greenlet 139815401537616 tries to "SELECT pg_sleep(5)".
[3.16 seconds] Restarted PostgreSQL in 3.16118597984 seconds.
# 4.18 seconds - Reconnected to PostgreSQL, trying to sleep again.
[9.18 seconds] Greenlet 139815404809104 stopped sleeping.
[9.18 seconds] Greenlet 139815404810064 stopped sleeping.
# Pool of 2 connections is ready for pending queries.
[14.18 seconds] Greenlet 139815404808464 stopped sleeping.
[14.18 seconds] Greenlet 139815401537616 stopped sleeping.
[14.18 seconds] All greenlets are done.
'''
import gevent, time
from subprocess import Popen
from thread import get_ident
sleep_seconds = 5
start = time.time()
def log(text):
print('[{seconds} seconds] {text}'.format(seconds=round(time.time() - start, 2), text=text))
log('Testing async pool (connections: {pool_size}), reconnect and retry:'.format(pool_size=pool_size))
def restart_postgresql():
start = time.time()
Popen('sudo service postgresql restart', shell=True).communicate()
log('Restarted PostgreSQL in {seconds} seconds.'.format(seconds=(time.time() - start)))
def sleep_at_postgresql():
greenlet_id = get_ident()
log('Greenlet {greenlet_id} tries to "SELECT pg_sleep({seconds})".'.format(greenlet_id=greenlet_id, seconds=sleep_seconds))
db('SELECT pg_sleep(%s)', sleep_seconds)
log('Greenlet {greenlet_id} stopped sleeping.'.format(greenlet_id=greenlet_id))
gevent.joinall([
gevent.spawn(sleep_at_postgresql)
for _ in xrange(pool_size * 2)
] + [
gevent.spawn(restart_postgresql),
])
log('All greenlets are done.')
### other
print('\nTesting other features...')
### db
db('DROP TABLE IF EXISTS "test_item"')
db('''
CREATE TABLE "test_item" (
"id" serial NOT NULL PRIMARY KEY,
"parent_id" integer
)
''')
### db_insert
item1_id = db_insert('test_item',
_return='id',
)
item2_id = db_insert('test_item',
parent_id=item1_id,
_return='id',
)
### db_insert result, db_transaction, db_rollback, db rows iteration
raised = False
try:
def code():
assert db('SELECT "id" FROM "test_item" WHERE id = %s', item2_id).row.id == item2_id
db('DELETE FROM "test_item" WHERE "id" = %s', item2_id)
assert db('SELECT "id" FROM "test_item" WHERE id = %s', item2_id).row is None
raise db_rollback
db_transaction(code)
except db_rollback:
raised = True
assert raised
items = db('SELECT "id" FROM "test_item" WHERE "id" IN %s', (item1_id, item2_id))
assert set(item.id for item in items) == set([item1_id, item2_id])
### db_transaction result, db_update
def code():
return db('SELECT "parent_id" FROM "test_item" WHERE "id" = %s', item2_id).row.parent_id
assert db_transaction(code) == item1_id
assert db_update('test_item',
parent_id=None,
where=dict(id=item2_id),
) == 1
assert db('SELECT "parent_id" FROM "test_item" WHERE "id" = %s', item2_id).row.parent_id is None
### db_update "IN %s"
db_update('test_item',
parent_id=0,
where=dict(id=(item1_id, item2_id)),
)
items = db('SELECT "parent_id" FROM "test_item" WHERE "id" IN %s', (item1_id, item2_id))
assert [item.parent_id for item in items] == [0, 0]
### "cannot run inside a transaction block"
db('DROP TYPE IF EXISTS "my_type"')
db("""CREATE TYPE "my_type" AS ENUM ('aaa', 'bbb')""")
raised = False
try:
db("""ALTER TYPE "my_type" ADD VALUE 'my_value'""")
except Exception as e:
raised = True
assert 'cannot run inside a transaction block' in repr(e)
assert raised
db("""ALTER TYPE "my_type" ADD VALUE 'my_value'""", autocommit=True)
### escape_like
assert escape_like('\? item_percent = 42%') == r'%\\? item\_percent = 42\%%'
### log
state = adict(logged=False)
def log(query):
sql, values = query
assert sql == 'UPDATE "test_item" SET "parent_id" = %s WHERE "id" = %s'
assert values == (42, 1)
state.logged = True
db_config(name=_db_config['database'], user=_db_config['user'], password=_db_config['password'], pool_size=_db_pool.qsize(), log=log)
db_update('test_item', parent_id=42, where=dict(id=item1_id))
assert state.logged
print('OK')
### adict cursor
class AdictCursorMixin(pymysql.cursors.DictCursorMixin):
dict_type = adict
class AdictCursor(AdictCursorMixin, pymysql.cursors.Cursor):
"""A cursor which returns results as a dict with attr access to keys."""
### db_config
db_config = adict(
host='127.0.0.1',
port=3306,
user='******',
password='******',
database='test',
charset='utf8',
cursor_class=AdictCursor,
query_timeout=55, # Less than default web timeout of 60 seconds, but greater than default "innodb_lock_wait_timeout" of 50 seconds.
pool_size=10,
_pool=None,
)
### db_init
def db_init():
db_config._pool = Queue()
for _ in xrange(db_config.pool_size):
db_config._pool.put(_create_db_conn())
### _create_db_conn
from traceback import format_exc
import sys
### crit_defaults
crit_defaults = adict(
subject='CRIT', # '{service_name} {host}:{port} CRIT'
plugins=[], # [critbot.plugins.syslog.plugin(), ...]
crit_in_crit=sys.stderr.write, # logging.getLogger('critbot').critical - if you enabled "syslog" plugin, else your_logger.critical
stop_spam_file=adict( # Stop spam from multiple processes at the same host.
enabled=False,
path=(
'/run/lock/critbot' if sys.platform.startswith('linux') # RAM, no disk IO.
else '/tmp/critbot' # Mac OS X, etc.
),
),
mc=adict( # Memcached to stop spam from multiple hosts.
enabled=False,
servers=['127.0.0.1'], # http://sendapatch.se/projects/pylibmc/reference.html#pylibmc.Client
client_config=adict(binary=True, behaviors=dict(tcp_nodelay=True, distribution='consistent')),
pool_size=10,
pool=None, # Is initialized on first crit from updated crit_defaults and to avoid useless slowdown in small scripts.
),
)
### utf8
def utf8(value):
return (
def app(environ, start_response):
status = response = raw_response = ''
try:
action = routes.get(environ['PATH_INFO'])
if not action:
raise ApiError(404)
raw_response = hasattr(action, 'raw_response')
#### raw environ
if hasattr(action, 'raw_environ'):
if raw_response:
response = action(environ, start_response)
else:
response = action(environ)
#### normal request
else:
if environ['REQUEST_METHOD'] != 'POST': # See README.md.
raise ApiError(501)
request = environ['wsgi.input'].read()
try:
request = json.loads(request) if request else {}
if not isinstance(request, dict):
raise ValueError
except ValueError:
raise ApiError(400, 'Request content should be JSON Object') # See README.md.
request = adict(request)
if raw_response:
response = action(request, start_response)
else:
response = action(request)
#### normal response
if not raw_response:
if response is None:
response = {}
if not isinstance(response, dict):
raise Exception('Response content should be JSON Object') # See README.md.
status = status_by_code[200]
#### Expected error, no logging.
except ApiError as e:
status = status_by_code[e.status_code]
response = dict(error=(e.error or status))
#### Unexpected error, traceback is logged.
except:
print_exc() # To sys.stderr.
status = status_by_code[500]
response = dict(error=status) # Server error details are saved to log and not disclosed to a client.
#### Response.
finally:
#### raw response
if raw_response:
return response
#### normal response
start_response(status, headers=[
('Content-Type', 'application/json'),
])
return [
json.dumps(response),
]
#### import
from adict import adict, ajson
import json, requests
#### config
config = adict(
host='127.0.0.1',
port=8888,
version='v0',
) # Can be updated from a caller script.
#### test
def test(target, action, request_json, status_code, response_json, method='POST'):
url = 'http://{host}:{port}/api/{version}/{target}/{action}'.format(target=target, action=action, **config)
request_json = json.dumps(request_json) if request_json else None
print('{method} {url} {request_json} --> {status_code} {response_json}'.format(**locals()))
response = requests.request(method, url, data=request_json)
assert response.status_code == status_code, (response.status_code, status_code, response.text)
response.ajson = ajson(response.json())
assert response.ajson == response_json, (response.text, response_json)
# NOTE: response.text is used instead of response.ajson|json() to easily compare without extra u'...'-s.
return response
dict_type = adict
class AdictCursor(AdictCursorMixin, pymysql.cursors.Cursor):
"""A cursor which returns results as a dict with attr access to keys."""
### db_config
db_config = adict(
host='127.0.0.1',
port=3306,
user='******',
password='******',
database='test',
charset='utf8',
cursor_class=AdictCursor,
query_timeout=
55, # Less than default web timeout of 60 seconds, but greater than default "innodb_lock_wait_timeout" of 50 seconds.
pool_size=10,
_pool=None,
)
### db_init
def db_init():
db_config._pool = Queue()
for _ in xrange(db_config.pool_size):
db_config._pool.put(_create_db_conn())
